From 8c3c5a3c0d36671c1e096adac216adf4a848078c Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 11 Aug 2026 00:27:41 -0700 Subject: [PATCH] feat: add native OpenAI streaming Signed-off-by: Ajay Thorve --- README.md | 4 +- adapters/common/README.md | 30 + .../nemo_fabric_adapters/common/lifecycle.py | 373 ++++++- crates/fabric-core/src/error.rs | 16 + crates/fabric-core/src/lib.rs | 10 +- crates/fabric-core/src/runtime.rs | 746 +++++++++++++- crates/fabric-core/src/schema.rs | 90 +- crates/fabric-python/src/lib.rs | 28 +- docs/adapter-contract/conformance.md | 12 +- docs/adapter-contract/execution.md | 100 +- docs/index.yml | 4 +- .../api/python-library-reference/index.md | 2 + .../nemo_fabric.openai_streaming.md | 54 + .../nemo_fabric.runtime.md | 33 +- .../nemo_fabric.streaming.md | 4 +- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../config/enum-relayatifstorageconfig.mdx | 28 +- .../config/enum-relayatofsinkconfig.mdx | 24 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../error/enum-fabricerror.mdx | 170 ++-- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/fn-version.mdx | 2 +- .../nemo-fabric-core/index.mdx | 15 + ...-openai-chat-completions-chunk-profile.mdx | 14 + .../runtime/constant-openai-stream-host.mdx | 14 + ...onstant-openai-stream-protocol-version.mdx | 14 + .../runtime/enum-errorstage.mdx | 2 +- .../enum-openaichatcompletionchunkobject.mdx | 108 ++ .../runtime/enum-openaistreamhost.mdx | 108 ++ .../runtime/enum-openaistreamprofile.mdx | 108 ++ .../enum-openaistreamprotocolversion.mdx | 108 ++ .../runtime/enum-openaistreamrecord.mdx | 142 +++ .../runtime/enum-runstatus.mdx | 2 +- .../runtime/fn-invoke-openai-stream.mdx | 14 + .../runtime/fn-invoke-runtime.mdx | 2 +- .../runtime/fn-prepare-environment.mdx | 2 +- .../nemo-fabric-core/runtime/fn-run-plan.mdx | 2 +- .../runtime/fn-start-runtime.mdx | 2 +- .../runtime/fn-stop-runtime.mdx | 2 +- .../nemo-fabric-core/runtime/index.mdx | 20 +- .../struct-openaichatcompletionchunk.mdx | 118 +++ ...struct-openaichatcompletionchunkchoice.mdx | 110 ++ .../struct-openaichatcompletionchunkdelta.mdx | 114 +++ .../runtime/struct-openaistreaminvocation.mdx | 102 ++ .../runtime/struct-openaistreamsink.mdx | 122 +++ .../runtime/struct-openaistreamtransport.mdx | 98 ++ .../runtime/struct-runrequest.mdx | 2 +- .../runtime/struct-runresult.mdx | 2 +- .../runtime/struct-runtimecontext.mdx | 2 +- .../runtime/struct-runtimehandle.mdx | 2 +- .../struct-runtimetelemetrycontext.mdx | 2 +- .../runtime/struct-telemetryref.mdx | 2 +- .../schema/enum-schemaname.mdx | 16 +- .../nemo-fabric-core/schema/index.mdx | 2 +- docs/sdk/python.mdx | 56 +- python/src/nemo_fabric/__init__.py | 2 + python/src/nemo_fabric/_native.pyi | 6 + python/src/nemo_fabric/openai_streaming.py | 695 +++++++++++++ python/src/nemo_fabric/runtime.py | 110 +- python/src/nemo_fabric/streaming.py | 4 +- schemas/SCHEMA.md | 16 +- .../openai-stream-invocation.schema.json | 368 +++++++ .../legacy/openai-stream-record.schema.json | 246 +++++ scripts/docs/enhance_python_api_reference.py | 1 + .../docs/generate_rust_library_reference.py | 5 +- scripts/generate_api_docs.sh | 11 +- skills/nemo-fabric-build-adapter/SKILL.md | 33 +- skills/nemo-fabric-integrate/SKILL.md | 45 +- .../references/sdk-api-inventory.md | 26 +- .../test_adapters_common_lifecycle.py | 423 ++++++++ tests/docs/test_python_api_docs.py | 3 + .../adapters/hermes-shim/fabric-adapter.json | 3 + .../hermes_shim/adapter.py | 44 + tests/python/test_native_sdk.py | 24 + tests/python/test_openai_streaming.py | 941 ++++++++++++++++++ 78 files changed, 5949 insertions(+), 228 deletions(-) create mode 100644 docs/reference/api/python-library-reference/nemo_fabric.openai_streaming.md create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/constant-openai-chat-completions-chunk-profile.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/constant-openai-stream-host.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/constant-openai-stream-protocol-version.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx create mode 100644 python/src/nemo_fabric/openai_streaming.py create mode 100644 schemas/adapter-contract/legacy/openai-stream-invocation.schema.json create mode 100644 schemas/adapter-contract/legacy/openai-stream-record.schema.json create mode 100644 tests/python/test_openai_streaming.py diff --git a/README.md b/README.md index 5ff881a4..adfee6d0 100644 --- a/README.md +++ b/README.md @@ -233,8 +233,8 @@ Use the following resources to learn about NeMo Fabric: - [Example Notebooks](examples/notebooks/README.md) provide a guided tour of the Python SDK. - [Python SDK guide](docs/sdk/python.mdx): typed configuration, planning, - diagnostics, requests, multi-turn runtimes, NeMo Relay streaming, - parallelism, results, and errors. + diagnostics, requests, multi-turn runtimes, native OpenAI streaming, NeMo + Relay streaming, parallelism, results, and errors. - [Experimentation CLI](docs/experimentation/cli.mdx): presets, maintained examples, editable application scaffolds, and explicit non-goals. - [Getting Started overview](docs/about-nemo-fabric/overview.mdx): interface diff --git a/adapters/common/README.md b/adapters/common/README.md index a455e7f9..3d3ee43a 100644 --- a/adapters/common/README.md +++ b/adapters/common/README.md @@ -44,6 +44,36 @@ class AdapterRuntime: lifecycle.serve(AdapterRuntime) ``` +If the adapter descriptor declares `capabilities.streaming`, the runtime must +also implement native OpenAI Chat Completions streaming: + +```python +class AdapterRuntime: + async def invoke_openai_stream(self, payload, emit): + async for chunk in self.client.stream(payload["request"]["input"]): + await emit(chunk) + return {"answer": "..."} +``` + +The optional method must have the signature +`async invoke_openai_stream(payload, emit)`. Execute the target exactly once, +await `emit(chunk)` only for mappings in the +`openai.chat_completions.chunk/v1` profile, and return one JSON-compatible +terminal outcome. Each chunk requires non-empty `id` and `model` strings, a +nonnegative integer `created`, the exact `chat.completion.chunk` object +discriminator, and structurally valid `choices`. An empty chunk stream is +valid. + +The SDK owns the authenticated loopback HTTP transport with chunked NDJSON +framing. The common host validates its credentials and framing, removes the +transport from the adapter payload, and supplies `emit`. Do not write chunks to +stdout or log stream credentials. This method is not used for Relay-backed +`Runtime.invoke_stream()`, which continues to execute ordinary `invoke`. +Process bindings that implement the wire protocol directly must follow the +generated +[`openai-stream-record.schema.json`](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/adapter-contract/legacy/openai-stream-record.schema.json) +chunk and explicit-end envelopes. + Adapters whose descriptor sets `config.input` to `agent_config` can ask the host to validate the southbound contract before `start`: diff --git a/adapters/common/src/nemo_fabric_adapters/common/lifecycle.py b/adapters/common/src/nemo_fabric_adapters/common/lifecycle.py index 2fb23730..a39cc70f 100644 --- a/adapters/common/src/nemo_fabric_adapters/common/lifecycle.py +++ b/adapters/common/src/nemo_fabric_adapters/common/lifecycle.py @@ -16,6 +16,7 @@ from collections.abc import Mapping from contextlib import contextmanager from contextlib import redirect_stdout +from contextlib import suppress from dataclasses import dataclass from typing import Any from typing import Protocol @@ -37,6 +38,16 @@ async def stop(self) -> None: RuntimeFactory = Callable[[], AdapterRuntime] ConfigLoader = Callable[[Any], Any] +OpenAIChunkEmitter = Callable[[Mapping[str, Any]], Awaitable[None]] + +_OPENAI_STREAM_CONNECT_TIMEOUT = 10.0 +_OPENAI_STREAM_HOST = "127.0.0.1" +_OPENAI_STREAM_PATH = "/openai-stream" +_OPENAI_STREAM_PROTOCOL = "fabric.openai_stream/v1alpha1" +_OPENAI_STREAM_PROFILE = "openai.chat_completions.chunk/v1" +_OPENAI_STREAM_RECORD_LIMIT = 1024 * 1024 +_UINT32_MAX = (1 << 32) - 1 +_UINT64_MAX = (1 << 64) - 1 class LifecycleError(Exception): @@ -61,6 +72,308 @@ class _AdapterCallError(LifecycleError): """Failure raised while executing an adapter runtime method.""" +async def _close_stream_writer(writer: asyncio.StreamWriter) -> None: + writer.close() + with suppress(OSError): + await writer.wait_closed() + + +class _OpenAIStreamWriter: + """Write correlated OpenAI chunks to one SDK-owned HTTP endpoint.""" + + def __init__( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + sink: dict[str, Any], + ) -> None: + self._reader = reader + self._writer = writer + self._sink = sink + self._sequence = 0 + self._finished = False + self._write_lock = asyncio.Lock() + + @classmethod + async def connect( + cls, + payload: dict[str, Any], + ) -> tuple[_OpenAIStreamWriter, dict[str, Any]]: + sink, adapter_payload = _validated_openai_stream_payload(payload) + reader: asyncio.StreamReader | None = None + writer: asyncio.StreamWriter | None = None + connected = False + try: + reader, writer = await asyncio.wait_for( + asyncio.open_connection(sink["host"], sink["port"]), + _OPENAI_STREAM_CONNECT_TIMEOUT, + ) + request = ( + f"POST {_OPENAI_STREAM_PATH} HTTP/1.1\r\n" + f"Host: {_OPENAI_STREAM_HOST}:{sink['port']}\r\n" + f"Authorization: Bearer {sink['token']}\r\n" + "Content-Type: application/x-ndjson\r\n" + "Transfer-Encoding: chunked\r\n" + "Expect: 100-continue\r\n" + "Connection: close\r\n\r\n" + ) + writer.write(request.encode("ascii")) + await writer.drain() + status = await asyncio.wait_for( + _read_http_response(reader), + _OPENAI_STREAM_CONNECT_TIMEOUT, + ) + if status != 100: + raise LifecycleError( + "lifecycle_stream_transport_failed", + "OpenAI stream listener rejected the adapter connection", + ) + connected = True + except LifecycleError: + raise + except Exception as error: + raise LifecycleError( + "lifecycle_stream_transport_failed", + "Adapter could not connect to the OpenAI stream listener", + ) from error + finally: + if writer is not None and not connected: + await _close_stream_writer(writer) + assert reader is not None and writer is not None + return cls(reader, writer, sink), adapter_payload + + async def emit(self, chunk: Mapping[str, Any]) -> None: + if not isinstance(chunk, Mapping): + raise LifecycleError( + "lifecycle_invalid_openai_stream_event", + "OpenAI stream events must be mappings", + ) + event = _validated_openai_chunk(dict(chunk)) + async with self._write_lock: + if self._finished: + raise LifecycleError( + "lifecycle_stream_transport_failed", + "Adapter cannot emit after finishing the OpenAI event stream", + ) + await self._write_record("chunk", chunk=event) + + async def finish(self) -> None: + async with self._write_lock: + if self._finished: + return + self._finished = True + try: + await self._write_record("end") + self._writer.write(b"0\r\n\r\n") + await self._writer.drain() + status = await asyncio.wait_for( + _read_http_response(self._reader), + _OPENAI_STREAM_CONNECT_TIMEOUT, + ) + if status != 200: + raise LifecycleError( + "lifecycle_stream_transport_failed", + "OpenAI stream listener rejected the event stream", + ) + except LifecycleError: + raise + except Exception as error: + raise LifecycleError( + "lifecycle_stream_transport_failed", + "Adapter could not finish the OpenAI event stream", + ) from error + finally: + await _close_stream_writer(self._writer) + + async def _write_record( + self, + record_type: str, + *, + chunk: dict[str, Any] | None = None, + ) -> None: + record = { + "type": record_type, + "sequence": self._sequence, + "runtime_id": self._sink["runtime_id"], + "invocation_id": self._sink["invocation_id"], + "request_id": self._sink["request_id"], + } + if chunk is not None: + record["chunk"] = chunk + try: + encoded = ( + json.dumps( + record, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + b"\n" + ) + except (TypeError, ValueError) as error: + raise LifecycleError( + "lifecycle_invalid_openai_stream_event", + "OpenAI stream events must contain JSON-compatible values", + ) from error + if len(encoded) > _OPENAI_STREAM_RECORD_LIMIT: + raise LifecycleError( + "lifecycle_openai_stream_event_too_large", + "OpenAI stream event exceeds the 1 MiB record limit", + ) + self._writer.write(f"{len(encoded):X}\r\n".encode("ascii")) + self._writer.write(encoded) + self._writer.write(b"\r\n") + try: + await self._writer.drain() + except Exception as error: + raise LifecycleError( + "lifecycle_stream_transport_failed", + "Adapter lost the OpenAI stream listener connection", + ) from error + self._sequence += 1 + + +async def _read_http_response(reader: asyncio.StreamReader) -> int: + status_line = await reader.readline() + try: + _version, raw_status, _reason = status_line.decode("ascii").split(" ", 2) + status = int(raw_status) + except (UnicodeDecodeError, ValueError) as error: + raise LifecycleError( + "lifecycle_stream_transport_failed", + "OpenAI stream listener returned an invalid HTTP response", + ) from error + while True: + line = await reader.readline() + if line in (b"\r\n", b"\n"): + return status + if not line: + raise LifecycleError( + "lifecycle_stream_transport_failed", + "OpenAI stream listener closed an incomplete HTTP response", + ) + + +def _validated_openai_stream_payload( + payload: dict[str, Any], +) -> tuple[dict[str, Any], dict[str, Any]]: + sink = payload.get("stream") + context = payload.get("runtime_context") + if not isinstance(sink, dict) or not isinstance(context, dict): + raise LifecycleError( + "lifecycle_invalid_stream_sink", + "OpenAI stream invocation is missing its transport", + ) + expected = { + "protocol_version": _OPENAI_STREAM_PROTOCOL, + "profile": _OPENAI_STREAM_PROFILE, + "host": _OPENAI_STREAM_HOST, + } + if any(sink.get(key) != value for key, value in expected.items()): + raise LifecycleError( + "lifecycle_invalid_stream_sink", + "OpenAI stream transport uses an unsupported protocol or endpoint", + ) + port = sink.get("port") + token = sink.get("token") + if ( + isinstance(port, bool) + or not isinstance(port, int) + or not 0 < port <= 65535 + or not isinstance(token, str) + or not token + or "\r" in token + or "\n" in token + ): + raise LifecycleError( + "lifecycle_invalid_stream_sink", + "OpenAI stream transport credentials are invalid", + ) + for name in ("runtime_id", "invocation_id", "request_id"): + if not isinstance(sink.get(name), str) or sink[name] != context.get(name): + raise LifecycleError( + "lifecycle_invalid_stream_sink", + "OpenAI stream transport identity does not match the invocation", + ) + adapter_payload = {key: value for key, value in payload.items() if key != "stream"} + return sink, adapter_payload + + +def _validated_openai_chunk(value: dict[str, Any]) -> dict[str, Any]: + def invalid(message: str) -> LifecycleError: + return LifecycleError("lifecycle_invalid_openai_stream_event", message) + + if value.get("object") != "chat.completion.chunk": + raise invalid("OpenAI stream events must use object 'chat.completion.chunk'") + identifier = value.get("id") + model = value.get("model") + created = value.get("created") + choices = value.get("choices") + if not isinstance(identifier, str) or not identifier.strip(): + raise invalid( + "OpenAI stream event id must be a non-empty string containing " + "a non-whitespace character" + ) + if not isinstance(model, str) or not model.strip(): + raise invalid( + "OpenAI stream event model must be a non-empty string containing " + "a non-whitespace character" + ) + if ( + isinstance(created, bool) + or not isinstance(created, int) + or not 0 <= created <= _UINT64_MAX + ): + raise invalid("OpenAI stream event created must be an unsigned 64-bit integer") + if not isinstance(choices, list): + raise invalid("OpenAI stream event choices must be a list") + for choice in choices: + if not isinstance(choice, dict): + raise invalid("OpenAI stream choices must be mappings") + index = choice.get("index") + delta = choice.get("delta") + if ( + isinstance(index, bool) + or not isinstance(index, int) + or not 0 <= index <= _UINT32_MAX + ): + raise invalid("OpenAI stream choice index must be an unsigned 32-bit integer") + if not isinstance(delta, dict): + raise invalid("OpenAI stream choice delta must be a mapping") + for name in ("content", "refusal", "role"): + if name in delta and delta[name] is not None and not isinstance( + delta[name], str + ): + raise invalid( + f"OpenAI stream choice delta {name} must be a string or null" + ) + if "function_call" in delta and delta["function_call"] is not None: + if not isinstance(delta["function_call"], dict): + raise invalid( + "OpenAI stream choice delta function_call must be a mapping or null" + ) + if "tool_calls" in delta and delta["tool_calls"] is not None: + tool_calls = delta["tool_calls"] + if not isinstance(tool_calls, list) or not all( + isinstance(tool_call, dict) for tool_call in tool_calls + ): + raise invalid( + "OpenAI stream choice delta tool_calls must be a list of mappings or null" + ) + if "finish_reason" in choice and choice["finish_reason"] is not None: + if not isinstance(choice["finish_reason"], str): + raise invalid( + "OpenAI stream choice finish_reason must be a string or null" + ) + if "logprobs" in choice and choice["logprobs"] is not None: + if not isinstance(choice["logprobs"], dict): + raise invalid("OpenAI stream choice logprobs must be a mapping or null") + if "usage" in value and value["usage"] is not None: + if not isinstance(value["usage"], dict): + raise invalid("OpenAI stream event usage must be a mapping or null") + return value + + @dataclass class _HostState: runtime: AdapterRuntime | None = None @@ -110,7 +423,7 @@ def _response( def _runtime_id(operation: str, payload: dict[str, Any]) -> str | None: - if operation in {"start", "invoke"}: + if operation in {"start", "invoke", "invoke_openai_stream"}: value = (payload.get("runtime_context") or {}).get("runtime_id") else: value = payload.get("runtime_id") @@ -163,7 +476,7 @@ def _failure_response(operation: str, error: LifecycleError) -> dict[str, Any]: return _response( operation, error=_error( - operation, + "invoke" if operation == "invoke_openai_stream" else operation, error.code, error.message, retryable=error.retryable, @@ -182,7 +495,7 @@ async def _stop_after_eof(runtime: AdapterRuntime) -> None: def _validated_request( message: dict[str, Any], operation: str ) -> tuple[dict[str, Any], str]: - if operation not in {"start", "invoke", "stop"}: + if operation not in {"start", "invoke", "invoke_openai_stream", "stop"}: raise LifecycleError( "lifecycle_invalid_operation", "Unknown lifecycle operation", @@ -265,6 +578,45 @@ async def _handle_invoke( return _response("invoke", output=output) +async def _handle_invoke_openai_stream( + state: _HostState, + runtime: AdapterRuntime, + payload: dict[str, Any], +) -> dict[str, Any]: + if state.failed: + raise LifecycleError( + "lifecycle_runtime_failed", + "Lifecycle runtime cannot accept another invocation", + ) + invoke = getattr(runtime, "invoke_openai_stream", None) + if not callable(invoke): + raise LifecycleError( + "lifecycle_openai_stream_unsupported", + "Adapter runtime does not implement OpenAI streaming", + ) + writer, adapter_payload = await _OpenAIStreamWriter.connect(payload) + adapter_error: BaseException | None = None + output: Any = None + try: + with _invocation_environment(adapter_payload): + output = await _adapter_call( + "invoke_openai_stream", + lambda: invoke(adapter_payload, writer.emit), + ) + except BaseException as error: + adapter_error = error + try: + await writer.finish() + except BaseException as finish_error: + if adapter_error is not None: + traceback.print_exception(finish_error, file=sys.stderr) + raise adapter_error from finish_error + raise + if adapter_error is not None: + raise adapter_error + return _response("invoke_openai_stream", output=output) + + async def _handle_stop( state: _HostState, runtime: AdapterRuntime, @@ -295,6 +647,8 @@ async def _dispatch( runtime = _active_runtime(state, message_runtime_id) if operation == "invoke": return await _handle_invoke(state, runtime, payload) + if operation == "invoke_openai_stream": + return await _handle_invoke_openai_stream(state, runtime, payload) return await _handle_stop(state, runtime) @@ -307,13 +661,13 @@ def _encode_response( return json.dumps(response, sort_keys=True) except (TypeError, ValueError): traceback.print_exc(file=sys.stderr) - if operation == "invoke" and state.runtime is not None: + if operation in {"invoke", "invoke_openai_stream"} and state.runtime is not None: state.failed = True return json.dumps( _response( operation, error=_error( - operation, + "invoke" if operation == "invoke_openai_stream" else operation, "lifecycle_invalid_response", "Adapter returned a non-JSON lifecycle response", ), @@ -358,7 +712,7 @@ async def _serve( should_stop = operation == "stop" except LifecycleError as error: if ( - operation == "invoke" + operation in {"invoke", "invoke_openai_stream"} and state.runtime is not None and isinstance(error, _AdapterCallError) ): @@ -367,12 +721,15 @@ async def _serve( should_stop = should_stop or operation in {"start", "stop"} except Exception as error: traceback.print_exc(file=sys.stderr) - if operation == "invoke" and state.runtime is not None: + if ( + operation in {"invoke", "invoke_openai_stream"} + and state.runtime is not None + ): state.failed = True response = _response( operation, error=_error( - operation, + "invoke" if operation == "invoke_openai_stream" else operation, "lifecycle_invalid_request", "Invalid lifecycle request", ), diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index d51da5dd..4334b143 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -164,6 +164,22 @@ pub enum FabricError { /// Adapter kind. adapter_kind: AdapterKind, }, + /// A requested runtime capability is not implemented by the selected adapter. + #[error("adapter `{adapter_id}` does not support runtime capability `{capability}`")] + UnsupportedRuntimeCapability { + /// Selected adapter id or harness name. + adapter_id: String, + /// Requested capability. + capability: &'static str, + }, + /// The SDK-provided native streaming transport is invalid. + #[error("invalid OpenAI stream transport at `{field}`: {reason}")] + InvalidOpenAiStreamTransport { + /// Invalid transport field. + field: &'static str, + /// Validation failure without credential material. + reason: &'static str, + }, /// A persistent local-host lifecycle operation failed. #[error( "adapter lifecycle {operation} failed for runtime `{runtime_id}` ({code}): {message}{diagnostics_suffix}", diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index deb60895..f0bc69a6 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -37,9 +37,13 @@ pub use doctor::{DoctorCheck, DoctorReport, DoctorStatus, doctor_plan}; pub use error::{FabricError, Result}; pub use runtime::{ AdapterInvocation, ArtifactManifest, ArtifactRef, EnvironmentHandle, ErrorInfo, ErrorStage, - FabricEvent, InvocationHandle, RunRequest, RunResult, RunStatus, RuntimeContext, RuntimeHandle, - RuntimeTelemetryContext, TelemetryRef, invoke_runtime, prepare_environment, run_plan, - start_runtime, stop_runtime, + FabricEvent, InvocationHandle, OPENAI_CHAT_COMPLETIONS_CHUNK_PROFILE, OPENAI_STREAM_HOST, + OPENAI_STREAM_PROTOCOL_VERSION, OpenAiChatCompletionChunk, OpenAiChatCompletionChunkChoice, + OpenAiChatCompletionChunkDelta, OpenAiChatCompletionChunkObject, OpenAiStreamHost, + OpenAiStreamInvocation, OpenAiStreamProfile, OpenAiStreamProtocolVersion, OpenAiStreamRecord, + OpenAiStreamSink, OpenAiStreamTransport, RunRequest, RunResult, RunStatus, RuntimeContext, + RuntimeHandle, RuntimeTelemetryContext, TelemetryRef, invoke_openai_stream, invoke_runtime, + prepare_environment, run_plan, start_runtime, stop_runtime, }; pub use schema::{ SchemaName, generate_all_schemas, generate_schema, generate_schema_json, write_schema_snapshots, diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index a8c5fb71..77a2f16a 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -37,6 +37,13 @@ const LOCAL_HOST_INVOKE_TIMEOUT: Duration = Duration::from_secs(60 * 60); const LOCAL_HOST_STOP_TIMEOUT: Duration = Duration::from_secs(10); const LOCAL_HOST_EXIT_GRACE: Duration = Duration::from_secs(2); const LOCAL_HOST_DIAGNOSTIC_LIMIT: usize = 16 * 1024; +/// SDK-owned loopback host for adapter-native OpenAI streaming. +pub const OPENAI_STREAM_HOST: &str = "127.0.0.1"; + +/// Southbound protocol version for adapter-native OpenAI streaming. +pub const OPENAI_STREAM_PROTOCOL_VERSION: &str = "fabric.openai_stream/v1alpha1"; +/// OpenAI event profile supported by the initial native streaming contract. +pub const OPENAI_CHAT_COMPLETIONS_CHUNK_PROFILE: &str = "openai.chat_completions.chunk/v1"; #[cfg(not(windows))] const VENV_BIN_DIR: &str = "bin"; @@ -339,11 +346,225 @@ pub struct AdapterInvocation { pub request: RunRequest, } +/// SDK-owned loopback transport for one native OpenAI streaming invocation. +#[derive(Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct OpenAiStreamTransport { + /// Loopback TCP port owned by the SDK listener. + #[schemars(range(min = 1))] + pub port: u16, + /// Single-use bearer token used to authenticate the adapter connection. + #[schemars(length(min = 1), regex(pattern = r"^[^\r\n]*\S[^\r\n]*$"))] + pub token: String, +} + +impl std::fmt::Debug for OpenAiStreamTransport { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("OpenAiStreamTransport") + .field("port", &self.port) + .field("token", &"[REDACTED]") + .finish() + } +} + +/// Supported southbound native-streaming protocol version. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum OpenAiStreamProtocolVersion { + /// Initial authenticated loopback HTTP and chunked-NDJSON protocol. + #[serde(rename = "fabric.openai_stream/v1alpha1")] + V1Alpha1, +} + +/// Supported OpenAI-compatible chunk profile. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum OpenAiStreamProfile { + /// OpenAI Chat Completions chunk objects. + #[serde(rename = "openai.chat_completions.chunk/v1")] + ChatCompletionsChunkV1, +} + +/// Supported native-streaming listener host. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum OpenAiStreamHost { + /// SDK-owned IPv4 loopback listener. + #[serde(rename = "127.0.0.1")] + Ipv4Loopback, +} + +/// Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric. +#[derive(Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct OpenAiStreamSink { + /// Southbound stream protocol version. + pub protocol_version: OpenAiStreamProtocolVersion, + /// OpenAI event profile emitted on this stream. + pub profile: OpenAiStreamProfile, + /// Loopback host owned by the SDK listener. + pub host: OpenAiStreamHost, + /// Loopback TCP port owned by the SDK listener. + #[schemars(range(min = 1))] + pub port: u16, + /// Single-use bearer token. Adapters must not log or persist this value. + #[schemars(length(min = 1), regex(pattern = r"^[^\r\n]*\S[^\r\n]*$"))] + pub token: String, + /// Runtime id for stream correlation. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub runtime_id: String, + /// Invocation id for stream correlation. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub invocation_id: String, + /// Request id for stream correlation. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub request_id: String, +} + +impl std::fmt::Debug for OpenAiStreamSink { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("OpenAiStreamSink") + .field("protocol_version", &self.protocol_version) + .field("profile", &self.profile) + .field("host", &self.host) + .field("port", &self.port) + .field("token", &"[REDACTED]") + .field("runtime_id", &self.runtime_id) + .field("invocation_id", &self.invocation_id) + .field("request_id", &self.request_id) + .finish() + } +} + +/// One adapter-native OpenAI streaming invocation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct OpenAiStreamInvocation { + /// Invocation context generated by NeMo Fabric. + pub runtime_context: RuntimeContext, + /// Typed caller request for this invocation. + pub request: RunRequest, + /// Authenticated progressive-output sink for this invocation. + pub stream: OpenAiStreamSink, +} + +/// Exact OpenAI object discriminator accepted by the v1 chunk profile. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum OpenAiChatCompletionChunkObject { + /// OpenAI Chat Completions streaming chunk. + #[serde(rename = "chat.completion.chunk")] + ChatCompletionChunk, +} + +/// Incremental assistant message fields carried by one OpenAI choice. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct OpenAiChatCompletionChunkDelta { + /// Incremental text content. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + /// Incremental refusal content. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub refusal: Option, + /// Incremental message role. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + /// Legacy incremental function-call fields. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub function_call: Option>, + /// Incremental tool-call fields. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>>, + /// Additional OpenAI-compatible delta fields preserved by pass-through. + #[serde(flatten)] + pub extensions: BTreeMap, +} + +/// One choice within an OpenAI Chat Completions streaming chunk. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct OpenAiChatCompletionChunkChoice { + /// Choice index within the response. + #[schemars(range(max = u32::MAX))] + pub index: u32, + /// Incremental assistant message fields. + pub delta: OpenAiChatCompletionChunkDelta, + /// Terminal reason when this choice finishes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finish_reason: Option, + /// Incremental log-probability information. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logprobs: Option>, + /// Additional OpenAI-compatible choice fields preserved by pass-through. + #[serde(flatten)] + pub extensions: BTreeMap, +} + +/// OpenAI Chat Completions chunk accepted by the native streaming profile. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct OpenAiChatCompletionChunk { + /// Provider-generated response identifier. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub id: String, + /// Exact OpenAI streaming object discriminator. + pub object: OpenAiChatCompletionChunkObject, + /// Unix timestamp in seconds. + #[schemars(range(max = u64::MAX))] + pub created: u64, + /// Model identifier. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub model: String, + /// Incremental choices; an explicit usage-only chunk can contain none. + pub choices: Vec, + /// Optional token-usage data. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option>, + /// Additional OpenAI-compatible top-level fields preserved by pass-through. + #[serde(flatten)] + pub extensions: BTreeMap, +} + +/// One correlated NDJSON record on the adapter-native stream channel. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum OpenAiStreamRecord { + /// One OpenAI Chat Completions chunk. + Chunk { + /// Monotonic zero-based record sequence. + #[schemars(range(max = u64::MAX))] + sequence: u64, + /// Runtime id for stream correlation. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + runtime_id: String, + /// Invocation id for stream correlation. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + invocation_id: String, + /// Request id for stream correlation. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + request_id: String, + /// OpenAI-compatible chunk passed through to the consumer. + chunk: OpenAiChatCompletionChunk, + }, + /// Explicit successful end of the progressive event channel. + End { + /// Monotonic zero-based record sequence. + #[schemars(range(max = u64::MAX))] + sequence: u64, + /// Runtime id for stream correlation. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + runtime_id: String, + /// Invocation id for stream correlation. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + invocation_id: String, + /// Request id for stream correlation. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + request_id: String, + }, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] enum AdapterLifecycleOperation { Start, Invoke, + InvokeOpenaiStream, Stop, } @@ -352,6 +573,7 @@ impl AdapterLifecycleOperation { match self { Self::Start => "start", Self::Invoke => "invoke", + Self::InvokeOpenaiStream => "invoke_openai_stream", Self::Stop => "stop", } } @@ -359,7 +581,7 @@ impl AdapterLifecycleOperation { fn error_stage(self) -> ErrorStage { match self { Self::Start => ErrorStage::Start, - Self::Invoke => ErrorStage::Invoke, + Self::Invoke | Self::InvokeOpenaiStream => ErrorStage::Invoke, Self::Stop => ErrorStage::Stop, } } @@ -393,6 +615,7 @@ struct AdapterLifecycleStop { enum AdapterLifecycleRequestKind { Start(Box), Invoke(Box), + InvokeOpenaiStream(Box), Stop(AdapterLifecycleStop), } @@ -401,6 +624,7 @@ impl AdapterLifecycleRequestKind { match self { Self::Start(_) => AdapterLifecycleOperation::Start, Self::Invoke(_) => AdapterLifecycleOperation::Invoke, + Self::InvokeOpenaiStream(_) => AdapterLifecycleOperation::InvokeOpenaiStream, Self::Stop(_) => AdapterLifecycleOperation::Stop, } } @@ -448,6 +672,13 @@ trait RuntimeAdapter { runtime: &RuntimeHandle, request: RunRequest, ) -> Result; + fn invoke_openai_stream( + &self, + plan: &RunPlan, + runtime: &RuntimeHandle, + request: RunRequest, + transport: OpenAiStreamTransport, + ) -> Result; fn stop(&self, runtime: &RuntimeHandle) -> Result>; } @@ -589,6 +820,54 @@ pub fn invoke_runtime( }) } +/// Invoke a started harness runtime and pass through native OpenAI chat-completion chunks. +pub fn invoke_openai_stream( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: RunRequest, + transport: OpenAiStreamTransport, +) -> Result { + validate_adapter_compatibility(plan)?; + validate_runtime_handle(plan, runtime)?; + let descriptor_supports_streaming = plan + .adapter_descriptor + .as_ref() + .is_some_and(|adapter| adapter.descriptor.capabilities.streaming); + if !plan.capabilities.streaming || !descriptor_supports_streaming { + return Err(FabricError::UnsupportedRuntimeCapability { + adapter_id: adapter_id(plan).unwrap_or_else(|| harness(plan)), + capability: "streaming", + }); + } + validate_openai_stream_transport(&transport)?; + if uses_local_host(plan) { + return LocalHostAdapter.invoke_openai_stream(plan, runtime, request, transport); + } + Err(FabricError::UnsupportedRuntimeAdapter { + harness: harness(plan), + adapter_kind: adapter_kind(plan), + }) +} + +fn validate_openai_stream_transport(transport: &OpenAiStreamTransport) -> Result<()> { + if transport.port == 0 { + return Err(FabricError::InvalidOpenAiStreamTransport { + field: "port", + reason: "must be greater than zero", + }); + } + if transport.token.trim().is_empty() + || transport.token.contains('\r') + || transport.token.contains('\n') + { + return Err(FabricError::InvalidOpenAiStreamTransport { + field: "token", + reason: "must be a non-empty string without line breaks", + }); + } + Ok(()) +} + fn validate_adapter_compatibility(plan: &RunPlan) -> Result<()> { validate_adapter_config_compatibility( &plan.config, @@ -876,6 +1155,16 @@ impl RuntimeAdapter for LocalHostAdapter { run_local_host_adapter(plan, runtime, request) } + fn invoke_openai_stream( + &self, + plan: &RunPlan, + runtime: &RuntimeHandle, + request: RunRequest, + transport: OpenAiStreamTransport, + ) -> Result { + run_local_host_openai_stream_adapter(plan, runtime, request, transport) + } + fn stop(&self, runtime: &RuntimeHandle) -> Result> { let Some(host) = local_hosts().remove(&runtime.runtime_id) else { return Ok(vec![local_host_stop_event(runtime, true, false)]); @@ -935,30 +1224,77 @@ fn run_local_host_adapter( runtime: &RuntimeHandle, request: RunRequest, ) -> Result { - let timeout = match plan.config.runtime.timeout_seconds { - Some(seconds) if seconds <= 0.0 => { - return Err(FabricError::InvalidConfig { - field: "runtime.timeout_seconds".to_string(), - reason: "must be a finite number greater than zero".to_string(), - }); - } + run_local_host_adapter_with_timeout(plan, runtime, request, local_host_invoke_timeout(plan)?) +} + +fn run_local_host_openai_stream_adapter( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: RunRequest, + transport: OpenAiStreamTransport, +) -> Result { + run_local_host_invocation_with_timeout( + plan, + runtime, + request, + LocalHostInvocation::OpenAiStream(transport), + local_host_invoke_timeout(plan)?, + ) +} + +fn local_host_invoke_timeout(plan: &RunPlan) -> Result { + match plan.config.runtime.timeout_seconds { + Some(seconds) if seconds <= 0.0 => Err(FabricError::InvalidConfig { + field: "runtime.timeout_seconds".to_string(), + reason: "must be a finite number greater than zero".to_string(), + }), Some(seconds) => { Duration::try_from_secs_f64(seconds).map_err(|_| FabricError::InvalidConfig { field: "runtime.timeout_seconds".to_string(), reason: "must be a finite number greater than zero".to_string(), - })? + }) } - None => LOCAL_HOST_INVOKE_TIMEOUT, - }; - run_local_host_adapter_with_timeout(plan, runtime, request, timeout) + None => Ok(LOCAL_HOST_INVOKE_TIMEOUT), + } } fn run_local_host_adapter_with_timeout( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: RunRequest, + invoke_timeout: Duration, +) -> Result { + run_local_host_invocation_with_timeout( + plan, + runtime, + request, + LocalHostInvocation::Invoke, + invoke_timeout, + ) +} + +enum LocalHostInvocation { + Invoke, + OpenAiStream(OpenAiStreamTransport), +} + +impl LocalHostInvocation { + fn operation(&self) -> AdapterLifecycleOperation { + match self { + Self::Invoke => AdapterLifecycleOperation::Invoke, + Self::OpenAiStream(_) => AdapterLifecycleOperation::InvokeOpenaiStream, + } + } +} + +fn run_local_host_invocation_with_timeout( plan: &RunPlan, runtime: &RuntimeHandle, mut request: RunRequest, + invocation_kind: LocalHostInvocation, invoke_timeout: Duration, ) -> Result { + let operation = invocation_kind.operation(); if request.request_id.is_empty() { request.request_id = new_id("request"); } @@ -972,7 +1308,7 @@ fn run_local_host_adapter_with_timeout( .cloned() .ok_or_else(|| { lifecycle_error( - AdapterLifecycleOperation::Invoke, + operation, &runtime.runtime_id, "host_unavailable", "persistent local adapter host is not active", @@ -993,21 +1329,38 @@ fn run_local_host_adapter_with_timeout( &artifacts, relay_config.as_ref(), )?; - let mut persisted_invocation = adapter_invocation.clone(); - for value in persisted_invocation - .runtime_context - .environment - .env - .values_mut() - { - *value = "[REDACTED]".to_string(); - } - let adapter_payload = serde_json::to_string_pretty(&persisted_invocation) - .map_err(FabricError::SerializeJson)?; + let (lifecycle_request, adapter_payload) = match invocation_kind { + LocalHostInvocation::Invoke => { + let mut persisted = adapter_invocation.clone(); + redact_adapter_invocation(&mut persisted); + let payload = + serde_json::to_string_pretty(&persisted).map_err(FabricError::SerializeJson)?; + ( + AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Invoke(Box::new( + adapter_invocation, + ))), + payload, + ) + } + LocalHostInvocation::OpenAiStream(transport) => { + let streaming_invocation = OpenAiStreamInvocation { + stream: openai_stream_sink(runtime, &invocation, transport), + runtime_context: adapter_invocation.runtime_context, + request: adapter_invocation.request, + }; + let mut persisted = streaming_invocation.clone(); + redact_openai_stream_invocation(&mut persisted); + let payload = + serde_json::to_string_pretty(&persisted).map_err(FabricError::SerializeJson)?; + ( + AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::InvokeOpenaiStream( + Box::new(streaming_invocation), + )), + payload, + ) + } + }; let fabric_invocation = write_fabric_invocation(&fabric_home, &adapter_payload)?; - let lifecycle_request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Invoke( - Box::new(adapter_invocation), - )); match exchange_lifecycle_message( &mut host_guard, &runtime.runtime_id, @@ -1774,6 +2127,38 @@ fn adapter_invocation( }) } +fn redact_adapter_invocation(invocation: &mut AdapterInvocation) { + redact_runtime_context_environment(&mut invocation.runtime_context); +} + +fn redact_openai_stream_invocation(invocation: &mut OpenAiStreamInvocation) { + redact_runtime_context_environment(&mut invocation.runtime_context); + invocation.stream.token = "[REDACTED]".to_string(); +} + +fn redact_runtime_context_environment(context: &mut RuntimeContext) { + for value in context.environment.env.values_mut() { + *value = "[REDACTED]".to_string(); + } +} + +fn openai_stream_sink( + runtime: &RuntimeHandle, + invocation: &InvocationHandle, + transport: OpenAiStreamTransport, +) -> OpenAiStreamSink { + OpenAiStreamSink { + protocol_version: OpenAiStreamProtocolVersion::V1Alpha1, + profile: OpenAiStreamProfile::ChatCompletionsChunkV1, + host: OpenAiStreamHost::Ipv4Loopback, + port: transport.port, + token: transport.token, + runtime_id: runtime.runtime_id.clone(), + invocation_id: invocation.invocation_id.clone(), + request_id: invocation.request_id.clone(), + } +} + fn adapter_runtime_context( plan: &RunPlan, runtime: &RuntimeHandle, @@ -2452,6 +2837,8 @@ fn now_millis() -> u128 { #[cfg(test)] mod tests { use std::fs; + use std::io::Read; + use std::net::TcpListener; use super::*; use crate::config::{ResolveContext, resolve_run_plan_from_config}; @@ -2496,6 +2883,7 @@ mod tests { root.join("fake_host.py"), r#"import json import os +import socket import sys import time @@ -2521,6 +2909,52 @@ def failure(stage, code, message): "retryable": False, } +def read_http_response(stream): + status = int(stream.readline().decode("ascii").split(" ", 2)[1]) + while stream.readline() not in (b"\r\n", b"\n", b""): + pass + return status + +def write_openai_stream(sink, chunks): + with socket.create_connection((sink["host"], sink["port"]), timeout=2) as client: + request = ( + "POST /openai-stream HTTP/1.1\r\n" + f"Host: {sink['host']}:{sink['port']}\r\n" + f"Authorization: Bearer {sink['token']}\r\n" + "Content-Type: application/x-ndjson\r\n" + "Transfer-Encoding: chunked\r\n" + "Expect: 100-continue\r\n" + "Connection: close\r\n\r\n" + ) + client.sendall(request.encode("ascii")) + stream = client.makefile("rb") + if read_http_response(stream) != 100: + raise RuntimeError("stream listener rejected connection") + records = [ + { + "type": "chunk", + "sequence": index, + "runtime_id": sink["runtime_id"], + "invocation_id": sink["invocation_id"], + "request_id": sink["request_id"], + "chunk": chunk, + } + for index, chunk in enumerate(chunks) + ] + records.append({ + "type": "end", + "sequence": len(chunks), + "runtime_id": sink["runtime_id"], + "invocation_id": sink["invocation_id"], + "request_id": sink["request_id"], + }) + for record in records: + encoded = json.dumps(record, separators=(",", ":")).encode() + b"\n" + client.sendall(f"{len(encoded):X}\r\n".encode() + encoded + b"\r\n") + client.sendall(b"0\r\n\r\n") + if read_http_response(stream) != 200: + raise RuntimeError("stream listener rejected records") + for line in sys.stdin: message = json.loads(line) operation = message["operation"] @@ -2535,7 +2969,7 @@ for line in sys.stdin: print("host crashed intentionally", file=sys.stderr, flush=True) time.sleep(1) sys.exit(17) - elif operation == "invoke": + elif operation in {"invoke", "invoke_openai_stream"}: invocations += 1 if MODE == "invoke_stderr": print(f"diagnostic-{invocations}", file=sys.stderr, flush=True) @@ -2546,11 +2980,33 @@ for line in sys.stdin: response("invoke", error=failure("invoke", "fake_invoke", "invoke rejected")) continue invocation = message["payload"] - if set(invocation) != {"runtime_context", "request"}: - response("invoke", error=failure( + expected_fields = ( + {"runtime_context", "request", "stream"} + if operation == "invoke_openai_stream" + else {"runtime_context", "request"} + ) + if set(invocation) != expected_fields: + response(operation, error=failure( "invoke", "fake_invoke_shape", "invoke payload contains runtime config" )) continue + if operation == "invoke_openai_stream": + write_openai_stream(invocation["stream"], [ + { + "id": "chunk-1", + "object": "chat.completion.chunk", + "created": 0, + "model": "test-model", + "choices": [{"index": 0, "delta": {"content": "hel"}}], + }, + { + "id": "chunk-2", + "object": "chat.completion.chunk", + "created": 0, + "model": "test-model", + "choices": [{"index": 0, "delta": {"content": "lo"}}], + }, + ]) output = { "host_pid": os.getpid(), "invocation_count": invocations, @@ -2570,7 +3026,7 @@ for line in sys.stdin: "metadata": {"source": "fake-host"}, }, }) - response("invoke", output=output) + response(operation, output=output) elif operation == "stop": if MODE == "stop_failure": response("stop", error=failure("stop", "fake_stop", "stop rejected")) @@ -2617,6 +3073,76 @@ for line in sys.stdin: (root, plan) } + fn openai_stream_listener( + token: &str, + ) -> (OpenAiStreamTransport, thread::JoinHandle>) { + let listener = TcpListener::bind((OPENAI_STREAM_HOST, 0)).expect("bind stream listener"); + let port = listener.local_addr().expect("listener address").port(); + let expected_token = token.to_string(); + let capture = thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("accept stream connection"); + let mut reader = BufReader::new(stream.try_clone().expect("clone stream")); + let mut headers = Vec::new(); + loop { + let mut line = String::new(); + reader.read_line(&mut line).expect("read HTTP header"); + if line == "\r\n" { + break; + } + headers.push(line); + } + assert_eq!( + headers.first().map(String::as_str), + Some("POST /openai-stream HTTP/1.1\r\n") + ); + assert!(headers.iter().any(|header| { + header == &format!("Authorization: Bearer {expected_token}\r\n") + })); + stream + .write_all(b"HTTP/1.1 100 Continue\r\n\r\n") + .expect("accept stream request"); + stream.flush().expect("flush continue response"); + + let mut records = Vec::new(); + loop { + let mut size_line = String::new(); + reader.read_line(&mut size_line).expect("read chunk size"); + let size = usize::from_str_radix(size_line.trim(), 16).expect("hex chunk size"); + if size == 0 { + let mut terminator = String::new(); + reader + .read_line(&mut terminator) + .expect("read chunk terminator"); + assert_eq!(terminator, "\r\n"); + break; + } + let mut encoded = vec![0; size]; + reader.read_exact(&mut encoded).expect("read chunk body"); + let mut terminator = [0; 2]; + reader + .read_exact(&mut terminator) + .expect("read chunk terminator"); + assert_eq!(&terminator, b"\r\n"); + records.push( + serde_json::from_slice(encoded.strip_suffix(b"\n").unwrap_or(&encoded)) + .expect("parse stream record"), + ); + } + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .expect("complete stream request"); + stream.flush().expect("flush final response"); + records + }); + ( + OpenAiStreamTransport { + port, + token: token.to_string(), + }, + capture, + ) + } + #[test] fn adapter_config_input_selects_southbound_payload_without_changing_legacy_default() { let (root, mut plan) = local_host_plan("success"); @@ -2708,6 +3234,164 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn local_host_openai_stream_uses_side_channel_and_separate_terminal_result() { + let (root, mut plan) = local_host_plan("success"); + plan.capabilities.streaming = true; + plan.adapter_descriptor + .as_mut() + .expect("resolved descriptor") + .descriptor + .capabilities + .streaming = true; + let runtime = start_runtime(&plan).expect("start local host"); + let (transport, capture) = openai_stream_listener("stream-secret"); + + let result = invoke_openai_stream(&plan, &runtime, RunRequest::text("stream"), transport) + .expect("stream invocation"); + let records = capture.join().expect("stream capture"); + + assert_eq!(records.len(), 3); + assert_eq!(records[0]["type"], "chunk"); + assert_eq!(records[0]["sequence"], 0); + assert_eq!(records[0]["chunk"]["id"], "chunk-1"); + assert_eq!(records[1]["chunk"]["id"], "chunk-2"); + assert_eq!(records[2]["type"], "end"); + assert_eq!(records[2]["sequence"], 2); + for record in &records { + assert_eq!(record["runtime_id"], result.runtime_id); + assert_eq!(record["invocation_id"], result.invocation_id); + assert_eq!(record["request_id"], result.request_id); + } + assert_eq!(result.output["input"], "stream"); + let persisted = fs::read_to_string( + result.metadata["fabric_invocation"] + .as_str() + .expect("invocation path"), + ) + .expect("read persisted invocation"); + assert!(!persisted.contains("stream-secret")); + let persisted: Value = serde_json::from_str(&persisted).expect("parse invocation"); + assert_eq!(persisted["stream"]["token"], "[REDACTED]"); + assert_eq!( + persisted["runtime_context"]["environment"]["env"]["FABRIC_NORMALIZED_ENV"], + "[REDACTED]" + ); + assert_eq!( + persisted["stream"]["protocol_version"], + OPENAI_STREAM_PROTOCOL_VERSION + ); + assert_eq!( + persisted["stream"]["profile"], + OPENAI_CHAT_COMPLETIONS_CHUNK_PROFILE + ); + assert_eq!(persisted["stream"]["runtime_id"], result.runtime_id); + assert_eq!(persisted["stream"]["invocation_id"], result.invocation_id); + assert_eq!(persisted["stream"]["request_id"], result.request_id); + + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn openai_stream_rejects_missing_capability_before_execution() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start local host"); + + let error = invoke_openai_stream( + &plan, + &runtime, + RunRequest::text("stream"), + OpenAiStreamTransport { + port: 1, + token: "unused".to_string(), + }, + ) + .expect_err("streaming capability must be required"); + + assert!(matches!( + error, + FabricError::UnsupportedRuntimeCapability { + capability: "streaming", + .. + } + )); + let ordinary = invoke_runtime(&plan, &runtime, RunRequest::text("ordinary")) + .expect("unsupported stream must not execute or poison runtime"); + assert_eq!(ordinary.output["invocation_count"], 1); + + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn openai_stream_rejects_stale_plan_capability_before_execution() { + let (root, mut plan) = local_host_plan("success"); + plan.capabilities.streaming = true; + let runtime = start_runtime(&plan).expect("start local host"); + + let error = invoke_openai_stream( + &plan, + &runtime, + RunRequest::text("stream"), + OpenAiStreamTransport { + port: 1, + token: "unused".to_string(), + }, + ) + .expect_err("the resolved descriptor must also claim streaming"); + + assert!(matches!( + error, + FabricError::UnsupportedRuntimeCapability { + capability: "streaming", + .. + } + )); + let ordinary = invoke_runtime(&plan, &runtime, RunRequest::text("ordinary")) + .expect("unsupported stream must not execute or poison runtime"); + assert_eq!(ordinary.output["invocation_count"], 1); + + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn openai_stream_debug_output_redacts_the_bearer_token() { + let transport = OpenAiStreamTransport { + port: 1234, + token: "stream-secret".to_string(), + }; + let sink = OpenAiStreamSink { + protocol_version: OpenAiStreamProtocolVersion::V1Alpha1, + profile: OpenAiStreamProfile::ChatCompletionsChunkV1, + host: OpenAiStreamHost::Ipv4Loopback, + port: transport.port, + token: transport.token.clone(), + runtime_id: "runtime-1".to_string(), + invocation_id: "invocation-1".to_string(), + request_id: "request-1".to_string(), + }; + + assert!(!format!("{transport:?}").contains("stream-secret")); + assert!(!format!("{sink:?}").contains("stream-secret")); + } + + #[test] + fn openai_stream_transport_rejects_blank_or_header_unsafe_tokens() { + for token in ["", " ", "safe\r\ninjected"] { + let error = validate_openai_stream_transport(&OpenAiStreamTransport { + port: 1234, + token: token.to_string(), + }) + .expect_err("invalid token must be rejected"); + assert!(matches!( + error, + FabricError::InvalidOpenAiStreamTransport { field: "token", .. } + )); + } + } + #[test] fn process_adapter_uses_the_same_persistent_local_host_protocol() { let (root, mut plan) = local_host_plan("success"); diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index 3c9a045e..b68f37ce 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -13,7 +13,8 @@ use crate::config::{AdapterDescriptor, AgentConfig, FabricConfig, RunPlan}; use crate::error::{FabricError, Result}; use crate::runtime::{ AdapterInvocation, ArtifactManifest, EnvironmentHandle, ErrorInfo, FabricEvent, - InvocationHandle, RunRequest, RunResult, RuntimeContext, RuntimeHandle, + InvocationHandle, OpenAiStreamInvocation, OpenAiStreamRecord, RunRequest, RunResult, + RuntimeContext, RuntimeHandle, }; use crate::{AgentRunRequest, AgentRunResult}; @@ -34,6 +35,10 @@ pub enum SchemaName { RunPlan, /// Initialized-runtime invocation payload schema. AdapterInvocation, + /// Adapter-facing native OpenAI streaming invocation schema. + OpenAiStreamInvocation, + /// Adapter-native OpenAI streaming NDJSON record schema. + OpenAiStreamRecord, /// Runtime context schema. RuntimeContext, /// Environment handle schema. @@ -56,7 +61,7 @@ pub enum SchemaName { impl SchemaName { /// All public schemas in stable output order. - pub const ALL: [Self; 16] = [ + pub const ALL: [Self; 18] = [ Self::Agent, Self::AgentConfig, Self::AgentRunRequest, @@ -64,6 +69,8 @@ impl SchemaName { Self::AdapterDescriptor, Self::RunPlan, Self::AdapterInvocation, + Self::OpenAiStreamInvocation, + Self::OpenAiStreamRecord, Self::RuntimeContext, Self::EnvironmentHandle, Self::RuntimeHandle, @@ -85,6 +92,8 @@ impl SchemaName { Self::AdapterDescriptor => "adapter-descriptor", Self::RunPlan => "run-plan", Self::AdapterInvocation => "adapter-invocation", + Self::OpenAiStreamInvocation => "openai-stream-invocation", + Self::OpenAiStreamRecord => "openai-stream-record", Self::RuntimeContext => "runtime-context", Self::EnvironmentHandle => "environment-handle", Self::RuntimeHandle => "runtime-handle", @@ -111,9 +120,11 @@ impl SchemaName { | Self::AgentRunResult | Self::AdapterDescriptor | Self::RuntimeContext => PathBuf::from("adapter-contract").join(filename), - Self::AdapterInvocation => PathBuf::from("adapter-contract") - .join("legacy") - .join(filename), + Self::AdapterInvocation | Self::OpenAiStreamInvocation | Self::OpenAiStreamRecord => { + PathBuf::from("adapter-contract") + .join("legacy") + .join(filename) + } _ => PathBuf::from(filename), } } @@ -128,6 +139,10 @@ impl SchemaName { "adapter-descriptor" | "adapter_descriptor" => Ok(Self::AdapterDescriptor), "run-plan" | "run_plan" => Ok(Self::RunPlan), "adapter-invocation" | "adapter_invocation" => Ok(Self::AdapterInvocation), + "openai-stream-invocation" | "openai_stream_invocation" => { + Ok(Self::OpenAiStreamInvocation) + } + "openai-stream-record" | "openai_stream_record" => Ok(Self::OpenAiStreamRecord), "runtime-context" | "runtime_context" => Ok(Self::RuntimeContext), "environment-handle" | "environment_handle" => Ok(Self::EnvironmentHandle), "runtime-handle" | "runtime_handle" => Ok(Self::RuntimeHandle), @@ -158,6 +173,8 @@ pub fn generate_schema(schema: SchemaName) -> Result { SchemaName::AdapterDescriptor => to_value(schema_for!(AdapterDescriptor)), SchemaName::RunPlan => to_value(schema_for!(RunPlan)), SchemaName::AdapterInvocation => to_value(schema_for!(AdapterInvocation)), + SchemaName::OpenAiStreamInvocation => to_value(schema_for!(OpenAiStreamInvocation)), + SchemaName::OpenAiStreamRecord => to_value(schema_for!(OpenAiStreamRecord)), SchemaName::RuntimeContext => to_value(schema_for!(RuntimeContext)), SchemaName::EnvironmentHandle => to_value(schema_for!(EnvironmentHandle)), SchemaName::RuntimeHandle => to_value(schema_for!(RuntimeHandle)), @@ -273,6 +290,18 @@ mod tests { .join("legacy") .join(SchemaName::AdapterInvocation.filename()) ); + assert_eq!( + SchemaName::OpenAiStreamInvocation.relative_path(), + PathBuf::from("adapter-contract") + .join("legacy") + .join(SchemaName::OpenAiStreamInvocation.filename()) + ); + assert_eq!( + SchemaName::OpenAiStreamRecord.relative_path(), + PathBuf::from("adapter-contract") + .join("legacy") + .join(SchemaName::OpenAiStreamRecord.filename()) + ); assert_eq!( SchemaName::Agent.relative_path(), PathBuf::from(SchemaName::Agent.filename()) @@ -309,6 +338,57 @@ mod tests { ); } + #[test] + fn openai_stream_schemas_freeze_transport_and_record_invariants() { + let invocation = + generate_schema(SchemaName::OpenAiStreamInvocation).expect("schema generation"); + let sink = &invocation["$defs"]["OpenAiStreamSink"]; + assert_eq!(sink["properties"]["port"]["minimum"], 1); + assert_eq!(sink["properties"]["token"]["minLength"], 1); + assert_eq!( + sink["properties"]["token"]["pattern"], + r"^[^\r\n]*\S[^\r\n]*$" + ); + assert_eq!( + invocation["$defs"]["OpenAiStreamProtocolVersion"]["oneOf"][0]["const"], + crate::runtime::OPENAI_STREAM_PROTOCOL_VERSION + ); + assert_eq!( + invocation["$defs"]["OpenAiStreamProfile"]["oneOf"][0]["const"], + crate::runtime::OPENAI_CHAT_COMPLETIONS_CHUNK_PROFILE + ); + assert_eq!( + invocation["$defs"]["OpenAiStreamHost"]["oneOf"][0]["const"], + crate::runtime::OPENAI_STREAM_HOST + ); + + let record = generate_schema(SchemaName::OpenAiStreamRecord).expect("schema generation"); + assert_eq!(record["oneOf"][0]["properties"]["type"]["const"], "chunk"); + assert_eq!(record["oneOf"][1]["properties"]["type"]["const"], "end"); + assert_eq!(record["oneOf"][0]["additionalProperties"], false); + assert_eq!(record["oneOf"][1]["additionalProperties"], false); + assert_eq!( + record["oneOf"][0]["properties"]["sequence"]["maximum"], + u64::MAX + ); + assert_eq!( + record["oneOf"][1]["properties"]["sequence"]["maximum"], + u64::MAX + ); + assert_eq!( + record["$defs"]["OpenAiChatCompletionChunk"]["required"], + serde_json::json!(["id", "object", "created", "model", "choices"]) + ); + assert_eq!( + record["$defs"]["OpenAiChatCompletionChunk"]["properties"]["created"]["maximum"], + u64::MAX + ); + assert_eq!( + record["$defs"]["OpenAiChatCompletionChunkObject"]["oneOf"][0]["const"], + "chat.completion.chunk" + ); + } + #[test] fn adapter_descriptor_schema_rejects_empty_identifiers() { let schema = generate_schema(SchemaName::AdapterDescriptor).expect("schema generation"); diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 84b2aa2d..5812e3d1 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -10,8 +10,8 @@ use std::thread; use std::time::{Duration, Instant}; use nemo_fabric_core::{ - FabricConfig, ResolveContext, RunPlan, RunRequest, RuntimeHandle, doctor_plan, - resolve_diagnostic_plan_from_config_with_adapter_directories, + FabricConfig, OpenAiStreamTransport, ResolveContext, RunPlan, RunRequest, RuntimeHandle, + doctor_plan, resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config_with_adapter_directories, run_plan, }; use pyo3::exceptions::PyRuntimeError; @@ -144,6 +144,25 @@ fn invoke_runtime( to_json(&result) } +/// Invoke a previously started runtime with native OpenAI streaming. +#[pyfunction] +fn invoke_openai_stream( + py: Python<'_>, + plan_json: String, + runtime_json: String, + request_json: String, + transport_json: String, +) -> PyResult { + let plan = parse_run_plan(plan_json)?; + let runtime = parse_runtime_handle(runtime_json)?; + let request = parse_run_request(request_json)?; + let transport = parse_openai_stream_transport(transport_json)?; + let result = py + .detach(|| nemo_fabric_core::invoke_openai_stream(&plan, &runtime, request, transport)) + .map_err(to_py_error)?; + to_json(&result) +} + /// Stop a previously started runtime and return FabricEvent list JSON. #[pyfunction] fn stop_runtime(py: Python<'_>, plan_json: String, runtime_json: String) -> PyResult { @@ -163,6 +182,7 @@ fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(run_config, m)?)?; m.add_function(wrap_pyfunction!(start_runtime, m)?)?; m.add_function(wrap_pyfunction!(invoke_runtime, m)?)?; + m.add_function(wrap_pyfunction!(invoke_openai_stream, m)?)?; m.add_function(wrap_pyfunction!(stop_runtime, m)?)?; Ok(()) } @@ -305,3 +325,7 @@ fn parse_run_plan(contents: String) -> PyResult { fn parse_runtime_handle(contents: String) -> PyResult { serde_json::from_str(&contents).map_err(|error| PyRuntimeError::new_err(error.to_string())) } + +fn parse_openai_stream_transport(contents: String) -> PyResult { + serde_json::from_str(&contents).map_err(|error| PyRuntimeError::new_err(error.to_string())) +} diff --git a/docs/adapter-contract/conformance.md b/docs/adapter-contract/conformance.md index 552f2cdb..63f2e679 100644 --- a/docs/adapter-contract/conformance.md +++ b/docs/adapter-contract/conformance.md @@ -46,7 +46,8 @@ Test each descriptor claim separately: | Requirements | `doctor(...)` reports both satisfied and missing states. | | Telemetry output | Output is produced and correlated to the correct invocation. | | Relay-backed stream | Ordinary invoke completes while correlated ATOF reaches `Runtime.invoke_stream()`. | -| Cancellation, updates, or native streaming | Do not claim until the installed Fabric runtime exposes and tests the corresponding adapter operation. | +| Native OpenAI stream | The descriptor declares `capabilities.streaming`; `invoke_openai_stream` executes exactly once, emits only valid `chat.completion.chunk` mappings, and returns a separate terminal result. Test empty and multi-chunk streams, early consumer close, and invalid chunks. | +| Cancellation or updates | Do not claim until the installed Fabric runtime exposes and tests the corresponding adapter operation. | ## Minimum Test Matrix @@ -62,8 +63,13 @@ Run this minimum test matrix before publishing an adapter: stop. 8. Start failure, invoke transport failure, malformed output, and cleanup on EOF. -9. Two independent runtimes to check state isolation. -10. Secret-redaction checks for logs and persisted diagnostic payloads. +9. If native OpenAI streaming is claimed, test empty and multi-chunk streams, + a separate terminal result, early close without cancellation, one active + turn, malformed and oversized records, chunk-profile validation, monotonic + sequence and identity correlation, an explicit end record, unauthenticated + and surplus connections, and exactly one target invocation. +10. Two independent runtimes to check state isolation. +11. Secret-redaction checks for logs and persisted diagnostic payloads. Record unsupported optional capabilities explicitly rather than omitting them from release notes. Link test results to the exact adapter package and contract diff --git a/docs/adapter-contract/execution.md b/docs/adapter-contract/execution.md index 78a76dab..18095efb 100644 --- a/docs/adapter-contract/execution.md +++ b/docs/adapter-contract/execution.md @@ -20,15 +20,16 @@ The abstract lifecycle contract contains these operations: | `invoke(AgentRunRequest, RuntimeContext)` | Preview, not negotiated | Future typed invocation boundary. The current binding uses its legacy request envelope and JSON-compatible output. | | `stop(runtime_id)` | Required | Attempt to release all runtime resources, including after partial or failed execution. | | `invoke_stream(...)` | NeMo Fabric-provided | Run ordinary `invoke` while NeMo Relay supplies correlated ATOF to the consumer. | -| `invoke_openai_stream(...)` | Reserved optional surface | A future native pass-through can expose only a declared OpenAI-compatible event profile. Other native stream formats are outside the contract. | +| `invoke_openai_stream(...)` | Optional | Execute exactly one adapter invocation while emitting OpenAI Chat Completions chunks. The selected descriptor must declare `capabilities.streaming`. | | `cancel(...)` | Reserved optional surface | Request cancellation of an active invocation when a runtime binding implements it. | | `update(...)` | Reserved optional surface | Atomically apply declared updateable fields when a runtime binding implements it. | -The required ordering is one `start`, zero or more `invoke` operations, then -one `stop`, regardless of whether invoke uses the current binding or the future -typed boundary. The minimum profile permits only one active invocation in a -runtime. Adapters need not implement a queue or internal concurrency; consumers -start independent runtimes for parallel work. +The required ordering is one `start`, zero or more invocation operations, then +one `stop`, regardless of whether an invocation uses `invoke`, +`invoke_openai_stream`, or the future typed boundary. The minimum profile +permits only one active invocation in a runtime. Adapters need not implement a +queue or internal concurrency; consumers start independent runtimes for +parallel work. Each operation produces one terminal response. An invocation-level failure does not necessarily invalidate the runtime. A lifecycle or transport failure @@ -71,6 +72,59 @@ The adapter reads Relay configuration and environment from `RuntimeContext.telemetry` or uses the optional common adapter helpers. It does not invent a second stream protocol. +## Native OpenAI Streaming + +`Runtime.invoke_openai_stream()` exposes adapter-native progressive output for +an adapter whose descriptor declares `capabilities.streaming`. An adapter that +declares the capability must implement the optional operation. This capability +is narrower than a generic native stream: the adapter emits only the declared +`openai.chat_completions.chunk/v1` profile. Each mapping includes non-empty +`id` and `model` strings, a nonnegative integer `created`, the exact +`chat.completion.chunk` object discriminator, and structurally valid `choices`. +OpenAI Responses API events, target-native event objects, Server-Sent Events +framing, and terminal results are outside the progressive stream. + +One call executes exactly one adapter invocation. The stream can be empty, and +its terminal normalized `RunResult` remains separate and authoritative. Ending +iteration early does not cancel the invocation. The SDK drains the invocation +when the consumer closes the stream so that the runtime can safely accept its +next turn. + +NeMo Fabric owns the authenticated loopback HTTP transport, chunked NDJSON +framing, correlation, validation, buffering, and consumer lifecycle. Adapters +must not persist or log the bearer token. NeMo Fabric persists only redacted +transport metadata for invocation auditing. The adapter host continues to +reserve stdout for the single terminal lifecycle response. + +Bindings that implement the transport without the common Python host must +read the sink from the generated +[`openai-stream-invocation.schema.json`](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/adapter-contract/legacy/openai-stream-invocation.schema.json) +payload and follow the generated +[`openai-stream-record.schema.json`](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/adapter-contract/legacy/openai-stream-record.schema.json) +envelope for monotonic chunk records and the explicit end record. + +The `fabric.openai_stream/v1alpha1` wire sequence is fixed: + +1. Open one connection to the sink's loopback host and port. +2. Send `POST /openai-stream HTTP/1.1` with `Authorization: Bearer `, + `Content-Type: application/x-ndjson`, `Transfer-Encoding: chunked`, and + `Expect: 100-continue`. +3. Wait for HTTP `100 Continue` before executing the target invocation. +4. Send zero or more `chunk` records, starting at sequence zero, followed by + exactly one `end` record at the next sequence. Encode each record as one + newline-terminated JSON value in the chunked request body. +5. Send the zero-length HTTP chunk, wait for HTTP `200 OK`, then write the one + terminal lifecycle response to stdout. A rejection or incomplete end + sequence is a lifecycle transport failure, not a successful empty stream. + +The bearer token is single-use for that invocation. Do not retry or replay the +target after a transport failure. + +Native OpenAI streaming and Relay streaming are independent. An adapter can +support either, both, or neither. `Runtime.invoke_stream()` continues to execute +ordinary `invoke` while exposing raw ATOF from NeMo Relay; it does not call +`invoke_openai_stream`. + ## Current Python Host Binding `nemo-fabric-adapters-common` is optional. Python adapters can use its @@ -89,6 +143,25 @@ class ExampleRuntime: request = payload["request"] return {"answer": "..."} + async def invoke_openai_stream(self, payload, emit): + request = payload["request"] + await emit( + { + "id": "chunk-1", + "object": "chat.completion.chunk", + "created": 0, + "model": "example-model", + "choices": [ + { + "index": 0, + "delta": {"content": "..."}, + "finish_reason": None, + } + ], + } + ) + return {"answer": "..."} + async def stop(self): pass @@ -99,7 +172,9 @@ def main() -> None: The host validates the start `config` as `AgentConfig`, serializes operations, normalizes lifecycle failures, reserves stdout for its protocol, and attempts -cleanup on EOF. The adapter remains responsible for target-specific validation, +cleanup on EOF. For native OpenAI streaming, it validates and sends each chunk +through the Fabric-owned transport before writing one terminal lifecycle +response. The adapter remains responsible for target-specific validation, translation, state, and shutdown. The lifecycle table describes the typed adapter contract, not the Python method @@ -109,7 +184,10 @@ protocol envelope carries `RuntimeContext` and runtime identity. It calls `stop()` after resolving the runtime identity from that envelope. The current invoke payload contains `RuntimeContext` plus northbound -`RunRequest`, and accepts JSON-compatible output. `AgentRunRequest` and -`AgentRunResult` are preview-only and are not part of the negotiated contract. -Keep conversion at the edge of the adapter so adopting a future typed invoke -boundary does not affect target lifecycle code. +`RunRequest`, and accepts JSON-compatible output. The common host calls the +optional native stream method as `async invoke_openai_stream(payload, emit)`; +`payload` has the same adapter-visible invocation fields, without transport +credentials, and `emit` accepts only OpenAI Chat Completions chunk mappings. +`AgentRunRequest` and `AgentRunResult` are preview-only and are not part of the +negotiated contract. Keep conversion at the edge of the adapter so adopting a +future typed invoke boundary does not affect target lifecycle code. diff --git a/docs/index.yml b/docs/index.yml index c9bcaf87..f8068d05 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -72,8 +72,10 @@ navigation: path: ./reference/api/python-library-reference/nemo_fabric.client.md - page: Runtime path: ./reference/api/python-library-reference/nemo_fabric.runtime.md - - page: Streaming + - page: Relay Streaming path: ./reference/api/python-library-reference/nemo_fabric.streaming.md + - page: OpenAI Streaming + path: ./reference/api/python-library-reference/nemo_fabric.openai_streaming.md - page: Models path: ./reference/api/python-library-reference/nemo_fabric.models.md - page: Types diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index 717f2bea..5797a655 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -13,6 +13,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`nemo_fabric.client`](./nemo_fabric.client.md#module-nemo_fabricclient): Native Python client for resolving and running NVIDIA NeMo Fabric agents. - [`nemo_fabric.runtime`](./nemo_fabric.runtime.md#module-nemo_fabricruntime): Runtime lifecycle support for the NVIDIA NeMo Fabric Python SDK. - [`nemo_fabric.streaming`](./nemo_fabric.streaming.md#module-nemo_fabricstreaming): NVIDIA NeMo Relay streaming support for the NVIDIA NeMo Fabric Python SDK. +- [`nemo_fabric.openai_streaming`](./nemo_fabric.openai_streaming.md#module-nemo_fabricopenai_streaming): Adapter-native OpenAI streaming for the NVIDIA NeMo Fabric Python SDK. - [`nemo_fabric.models`](./nemo_fabric.models.md#module-nemo_fabricmodels): Pydantic SDK models for NVIDIA NeMo Fabric configuration and requests. - [`nemo_fabric.types`](./nemo_fabric.types.md#module-nemo_fabrictypes): Public data contracts for the NeMo Fabric Python SDK. - [`nemo_fabric.errors`](./nemo_fabric.errors.md#module-nemo_fabricerrors): Public exception hierarchy for the NeMo Fabric Python SDK. @@ -23,6 +24,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`runtime.Runtime`](./nemo_fabric.runtime.md#class-runtime): One logical, stateful harness execution. - [`runtime.RuntimeStatus`](./nemo_fabric.runtime.md#class-runtimestatus): Lifecycle state of a runtime. - [`streaming.InvokeStream`](./nemo_fabric.streaming.md#class-invokestream): Async iterator of raw ATOF records for one runtime invocation. +- [`openai_streaming.OpenAIInvokeStream`](./nemo_fabric.openai_streaming.md#class-openaiinvokestream): Async iterator of OpenAI chat-completion chunks for one invocation. - [`models.EnvironmentConfig`](./nemo_fabric.models.md#class-environmentconfig): Execution environment configuration supplied by the consumer. - [`models.FabricBaseModel`](./nemo_fabric.models.md#class-fabricbasemodel): Base class for SDK-facing Pydantic models. - [`models.FabricConfig`](./nemo_fabric.models.md#class-fabricconfig): SDK-facing typed NeMo Fabric agent configuration. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.openai_streaming.md b/docs/reference/api/python-library-reference/nemo_fabric.openai_streaming.md new file mode 100644 index 00000000..0086f2b1 --- /dev/null +++ b/docs/reference/api/python-library-reference/nemo_fabric.openai_streaming.md @@ -0,0 +1,54 @@ +--- +title: "OpenAI Streaming" +slug: "/reference/api/python-library-reference/openai-streaming" +description: "Consume adapter-native OpenAI Chat Completions chunks and terminal invocation results." +--- + + +# module `nemo_fabric.openai_streaming` + +Adapter-native OpenAI streaming for the NVIDIA NeMo Fabric Python SDK. + + + +--- + + +## class `OpenAIInvokeStream` + +Async iterator of OpenAI chat-completion chunks for one invocation. + +Await ``result()`` for the separate normalized terminal result. Call ``aclose()`` when iteration stops early; it drains the stream without cancelling the invocation. + + + + +--- + + +### method `aclose` + +```python +async def aclose() -> None +``` + +Discard unread chunks and wait without cancelling the invocation. + +--- + + +### method `result` + +```python +async def result() -> RunResult +``` + +Drain the stream and return its separate normalized terminal result. + + + + +--- + +_This file was automatically generated via [lazydocs](https://github.com/ml-tooling/lazydocs)._ diff --git a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md index 449ee093..a9bde67d 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md @@ -84,6 +84,12 @@ Return the current ``ACTIVE``, ``STOPPED``, or ``FAILED`` state. --- +### property supports_openai_streaming + +Return whether the selected adapter implements native OpenAI streaming. + +--- + ### property supports_streaming Return whether NVIDIA NeMo Relay ATOF streaming is enabled. @@ -127,6 +133,31 @@ Run one turn on this runtime. --- +### method `invoke_openai_stream` + +```python +def invoke_openai_stream( + *, + input: Any = None, + request: RunRequest | None = None, +) -> OpenAIInvokeStream +``` + +Start one turn and stream native OpenAI chat-completion chunks. + +The returned stream yields ``chat.completion.chunk`` mappings. Await ``stream.result()`` for the separate normalized terminal result. + + + +**Raises:** + + - `FabricCapabilityError`: If the selected adapter does not advertise native OpenAI streaming. + - `FabricConfigError`: If request fields conflict or are not JSON-compatible. + - `FabricStateError`: If another turn or stream is active. + +--- + + ### method `invoke_stream` ```python @@ -139,7 +170,7 @@ def invoke_stream( Start one turn and stream raw NeMo Relay ATOF records as they arrive. -``input`` and ``request`` are mutually exclusive. The returned :class:`InvokeStream` yields raw ATOF dictionaries. Await ``stream.result()`` for the terminal normalized :class:`RunResult`. +``input`` and ``request`` are mutually exclusive. The returned ``InvokeStream`` yields raw ATOF dictionaries. Await ``stream.result()`` for the terminal normalized ``RunResult``. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.streaming.md b/docs/reference/api/python-library-reference/nemo_fabric.streaming.md index 7ff2df8e..27485884 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.streaming.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.streaming.md @@ -1,5 +1,5 @@ --- -title: "Streaming" +title: "Relay Streaming" slug: "/reference/api/python-library-reference/streaming" description: "Consume raw NVIDIA NeMo Relay ATOF records and terminal invocation results." --- @@ -19,7 +19,7 @@ NeMo Relay streaming support for the NVIDIA NeMo Fabric Python SDK. Async iterator of raw ATOF records for one runtime invocation. -Consume the final normalized result separately with :meth:`result`. If iteration stops early, call :meth:`aclose` before starting another turn. +Consume the final normalized result separately with ``result()``. If iteration stops early, call ``aclose()`` before starting another turn. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index a7db8455..cb39f881 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 95 +position: 110 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index b02a4c35..35be00db 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 96 +position: 111 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 1ed654c0..19be8487 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 97 +position: 112 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx index c5ad5872..cdfb450c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx @@ -23,23 +23,23 @@ Upload ATIF artifacts to HTTP storage. #### Fields -### `endpoint: String` +##### `endpoint: String` HTTP storage endpoint. -### `headers: BTreeMap` +##### `headers: BTreeMap` Static HTTP headers. -### `header_env: BTreeMap` +##### `header_env: BTreeMap` Environment-variable-backed HTTP headers. -### `timeout_millis: u64` +##### `timeout_millis: u64` Request timeout in milliseconds. -### `extensions: BTreeMap` +##### `extensions: BTreeMap` Additive HTTP storage fields. @@ -51,39 +51,39 @@ Upload ATIF artifacts to S3-compatible storage. #### Fields -### `bucket: String` +##### `bucket: String` S3 bucket name. -### `key_prefix: Option` +##### `key_prefix: Option` Optional S3 object key prefix. -### `access_key_id: Option` +##### `access_key_id: Option` AWS access key id. -### `secret_access_key_var: Option` +##### `secret_access_key_var: Option` Environment variable containing the AWS secret access key. -### `session_token_var: Option` +##### `session_token_var: Option` Environment variable containing the AWS session token. -### `region: Option` +##### `region: Option` AWS region. -### `endpoint_url: Option` +##### `endpoint_url: Option` S3-compatible endpoint URL. -### `allow_http: Option` +##### `allow_http: Option` Allow HTTP endpoints for S3-compatible storage. -### `extensions: BTreeMap` +##### `extensions: BTreeMap` Additive S3 storage fields. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx index 04f7e794..ac19b076 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx @@ -23,19 +23,19 @@ Write ATOF records to a local file. #### Fields -### `output_directory: Option` +##### `output_directory: Option` Directory used for ATOF files. -### `filename: Option` +##### `filename: Option` ATOF file name. -### `mode: RelayAtofMode` +##### `mode: RelayAtofMode` File write mode. -### `extensions: BTreeMap` +##### `extensions: BTreeMap` Additive file sink fields. @@ -47,35 +47,35 @@ Send ATOF records to a remote stream. #### Fields -### `url: String` +##### `url: String` Stream URL. -### `transport: RelayAtofStreamTransport` +##### `transport: RelayAtofStreamTransport` Stream transport. -### `headers: BTreeMap` +##### `headers: BTreeMap` Static stream headers. -### `header_env: BTreeMap` +##### `header_env: BTreeMap` Environment-variable-backed stream headers. -### `timeout_millis: u64` +##### `timeout_millis: u64` Request timeout in milliseconds. -### `field_name_policy: RelayAtofStreamFieldNamePolicy` +##### `field_name_policy: RelayAtofStreamFieldNamePolicy` Field-name handling policy. -### `name: Option` +##### `name: Option` Optional stream sink name. -### `extensions: BTreeMap` +##### `extensions: BTreeMap` Additive stream sink fields. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index 49705395..8e127593 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 98 +position: 113 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index 5a4d351a..1188ea63 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 99 +position: 114 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 993c5e96..9a53df76 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: AdapterDescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: AdapterDescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: AdapterDescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: AdapterDescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: AdapterDescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: AdapterDescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: AdapterDescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: AdapterDescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -23,11 +23,11 @@ The base directory could not be resolved to an absolute path. #### Fields -### `path: PathBuf` +##### `path: PathBuf` Base directory supplied by the caller. -### `source: Error` +##### `source: Error` Underlying path-resolution error. @@ -45,11 +45,11 @@ A requested adapter id is not present in the agent config. #### Fields -### `adapter_id: String` +##### `adapter_id: String` Requested adapter id. -### `available: Vec` +##### `available: Vec` Available adapter ids. @@ -61,19 +61,19 @@ An adapter descriptor did not match the selected harness config. #### Fields -### `path: PathBuf` +##### `path: PathBuf` Adapter descriptor path. -### `field: &'static str` +##### `field: &'static str` Mismatched field. -### `expected: String` +##### `expected: String` Expected value. -### `actual: String` +##### `actual: String` Actual value. @@ -85,15 +85,15 @@ An adapter descriptor does not support a selected config value. #### Fields -### `adapter_id: String` +##### `adapter_id: String` Adapter id. -### `field: &'static str` +##### `field: &'static str` Unsupported field. -### `value: String` +##### `value: String` Unsupported value. @@ -105,11 +105,11 @@ An adapter descriptor is malformed. #### Fields -### `path: PathBuf` +##### `path: PathBuf` Adapter descriptor path. -### `message: String` +##### `message: String` Validation message. @@ -121,23 +121,23 @@ Adapter-owned harness settings do not satisfy the resolved descriptor schema. #### Fields -### `adapter_id: String` +##### `adapter_id: String` Selected adapter id. -### `descriptor_source: AdapterDescriptorSource` +##### `descriptor_source: AdapterDescriptorSource` Registry source of the selected descriptor. -### `descriptor_path: PathBuf` +##### `descriptor_path: PathBuf` Path to the selected descriptor. -### `settings_path: String` +##### `settings_path: String` Canonical path to the invalid setting. -### `reason: String` +##### `reason: String` Schema validation failure. @@ -149,23 +149,23 @@ Adapter-owned workflow configuration does not satisfy the resolved descriptor sc #### Fields -### `adapter_id: String` +##### `adapter_id: String` Selected adapter id. -### `descriptor_source: AdapterDescriptorSource` +##### `descriptor_source: AdapterDescriptorSource` Registry source of the selected descriptor. -### `descriptor_path: PathBuf` +##### `descriptor_path: PathBuf` Path to the selected descriptor. -### `workflow_path: String` +##### `workflow_path: String` Canonical path to the invalid workflow field. -### `reason: String` +##### `reason: String` Schema validation failure. @@ -177,23 +177,23 @@ A normalized tool definition does not satisfy the resolved descriptor schema. #### Fields -### `adapter_id: String` +##### `adapter_id: String` Selected adapter id. -### `descriptor_source: AdapterDescriptorSource` +##### `descriptor_source: AdapterDescriptorSource` Registry source of the selected descriptor. -### `descriptor_path: PathBuf` +##### `descriptor_path: PathBuf` Path to the selected descriptor. -### `definition_path: String` +##### `definition_path: String` Canonical path to the invalid definition field. -### `reason: String` +##### `reason: String` Schema validation failure. @@ -205,23 +205,23 @@ Adapter-owned extensions do not satisfy a descriptor extension schema. #### Fields -### `adapter_id: String` +##### `adapter_id: String` Selected adapter id. -### `descriptor_source: AdapterDescriptorSource` +##### `descriptor_source: AdapterDescriptorSource` Registry source of the selected descriptor. -### `descriptor_path: PathBuf` +##### `descriptor_path: PathBuf` Path to the selected descriptor. -### `extension_path: String` +##### `extension_path: String` Canonical path to the invalid extension field. -### `reason: String` +##### `reason: String` Schema validation failure. @@ -233,11 +233,11 @@ A normalized Fabric config field is invalid. #### Fields -### `field: String` +##### `field: String` Canonical configuration path. -### `reason: String` +##### `reason: String` Validation failure. @@ -249,15 +249,15 @@ A valid normalized field cannot be implemented by the selected adapter. #### Fields -### `adapter_id: String` +##### `adapter_id: String` Selected adapter id. -### `field: String` +##### `field: String` Canonical configuration path. -### `reason: String` +##### `reason: String` Compatibility failure. @@ -269,11 +269,11 @@ A requested schema is not known. #### Fields -### `schema: String` +##### `schema: String` Requested schema name. -### `available: Vec` +##### `available: Vec` Available schema names. @@ -285,14 +285,46 @@ Runtime invocation is not supported for the selected adapter. #### Fields -### `harness: String` +##### `harness: String` Harness type. -### `adapter_kind: AdapterKind` +##### `adapter_kind: AdapterKind` Adapter kind. +### `UnsupportedRuntimeCapability` + +
+ +A requested runtime capability is not implemented by the selected adapter. + +#### Fields + +##### `adapter_id: String` + +Selected adapter id or harness name. + +##### `capability: &'static str` + +Requested capability. + +### `InvalidOpenAiStreamTransport` + +
+ +The SDK-provided native streaming transport is invalid. + +#### Fields + +##### `field: &'static str` + +Invalid transport field. + +##### `reason: &'static str` + +Validation failure without credential material. + ### `AdapterLifecycleOperation`
@@ -301,23 +333,23 @@ A persistent local-host lifecycle operation failed. #### Fields -### `operation: &'static str` +##### `operation: &'static str` Lifecycle operation that failed. -### `runtime_id: String` +##### `runtime_id: String` Runtime whose host failed. -### `code: String` +##### `code: String` Stable failure code. -### `message: String` +##### `message: String` Human-readable failure message. -### `diagnostics: String` +##### `diagnostics: String` Bounded adapter-host diagnostics. @@ -329,19 +361,19 @@ A runtime handle was used with a different run plan than the one that created it #### Fields -### `field: &'static str` +##### `field: &'static str` Mismatched runtime handle field. -### `expected: String` +##### `expected: String` Expected value from the run plan. -### `actual: String` +##### `actual: String` Actual value from the runtime handle. -### `runtime_id: String` +##### `runtime_id: String` Runtime handle id. @@ -353,11 +385,11 @@ An environment provider is not runnable for the selected adapter in this POC. #### Fields -### `provider: String` +##### `provider: String` Environment provider. -### `adapter_kind: AdapterKind` +##### `adapter_kind: AdapterKind` Adapter kind. @@ -369,11 +401,11 @@ Process adapter settings were invalid. #### Fields -### `path: PathBuf` +##### `path: PathBuf` Config path. -### `source: Error` +##### `source: Error` Underlying JSON parse error. @@ -385,11 +417,11 @@ Python adapter settings were invalid. #### Fields -### `path: PathBuf` +##### `path: PathBuf` Config path. -### `source: Error` +##### `source: Error` Underlying JSON parse error. @@ -401,15 +433,15 @@ The resolved Python adapter interpreter could not be used. #### Fields -### `path: PathBuf` +##### `path: PathBuf` Resolved interpreter path. -### `origin: String` +##### `origin: String` Human-readable description of where the interpreter was resolved from. -### `reason: String` +##### `reason: String` Why the interpreter cannot be used. @@ -421,11 +453,11 @@ A process runner failed to start or complete. #### Fields -### `command: String` +##### `command: String` Command being run. -### `source: Error` +##### `source: Error` Underlying IO error. @@ -443,11 +475,11 @@ Filesystem read failed. #### Fields -### `path: PathBuf` +##### `path: PathBuf` File path. -### `source: Error` +##### `source: Error` Underlying IO error. @@ -459,11 +491,11 @@ Filesystem write failed. #### Fields -### `path: PathBuf` +##### `path: PathBuf` File path. -### `source: Error` +##### `source: Error` Underlying IO error. @@ -475,11 +507,11 @@ JSON parse failed. #### Fields -### `path: PathBuf` +##### `path: PathBuf` File path. -### `source: Error` +##### `source: Error` Underlying JSON error. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index 369e30a0..c6cc9d90 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 100 +position: 115 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 0605b0b1..d3473444 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 103 +position: 118 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 9a7468ac..f9c70831 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -90,6 +90,20 @@ Core config and runtime contract for NeMo Fabric. - `pub use runtime::ErrorStage;` - `pub use runtime::FabricEvent;` - `pub use runtime::InvocationHandle;` +- `pub use runtime::OPENAI_CHAT_COMPLETIONS_CHUNK_PROFILE;` +- `pub use runtime::OPENAI_STREAM_HOST;` +- `pub use runtime::OPENAI_STREAM_PROTOCOL_VERSION;` +- `pub use runtime::OpenAiChatCompletionChunk;` +- `pub use runtime::OpenAiChatCompletionChunkChoice;` +- `pub use runtime::OpenAiChatCompletionChunkDelta;` +- `pub use runtime::OpenAiChatCompletionChunkObject;` +- `pub use runtime::OpenAiStreamHost;` +- `pub use runtime::OpenAiStreamInvocation;` +- `pub use runtime::OpenAiStreamProfile;` +- `pub use runtime::OpenAiStreamProtocolVersion;` +- `pub use runtime::OpenAiStreamRecord;` +- `pub use runtime::OpenAiStreamSink;` +- `pub use runtime::OpenAiStreamTransport;` - `pub use runtime::RunRequest;` - `pub use runtime::RunResult;` - `pub use runtime::RunStatus;` @@ -97,6 +111,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use runtime::RuntimeHandle;` - `pub use runtime::RuntimeTelemetryContext;` - `pub use runtime::TelemetryRef;` +- `pub use runtime::invoke_openai_stream;` - `pub use runtime::invoke_runtime;` - `pub use runtime::prepare_environment;` - `pub use runtime::run_plan;` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/constant-openai-chat-completions-chunk-profile.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/constant-openai-chat-completions-chunk-profile.mdx new file mode 100644 index 00000000..4546cedb --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/constant-openai-chat-completions-chunk-profile.mdx @@ -0,0 +1,14 @@ +--- +title: "Constant OPENAI_CHAT_COMPLETIONS_CHUNK_PROFILE" +sidebar-title: "OPENAI_CHAT_COMPLETIONS_CHUNK_PROFILE" +description: "OpenAI event profile supported by the initial native streaming contract." +position: 27 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"openai.chat_completions.chunk/v1\";"}} />
+ +OpenAI event profile supported by the initial native streaming contract. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/constant-openai-stream-host.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/constant-openai-stream-host.mdx new file mode 100644 index 00000000..175850fb --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/constant-openai-stream-host.mdx @@ -0,0 +1,14 @@ +--- +title: "Constant OPENAI_STREAM_HOST" +sidebar-title: "OPENAI_STREAM_HOST" +description: "SDK-owned loopback host for adapter-native OpenAI streaming." +position: 28 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"127.0.0.1\";"}} />
+ +SDK-owned loopback host for adapter-native OpenAI streaming. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/constant-openai-stream-protocol-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/constant-openai-stream-protocol-version.mdx new file mode 100644 index 00000000..c4d5af9b --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/constant-openai-stream-protocol-version.mdx @@ -0,0 +1,14 @@ +--- +title: "Constant OPENAI_STREAM_PROTOCOL_VERSION" +sidebar-title: "OPENAI_STREAM_PROTOCOL_VERSION" +description: "Southbound protocol version for adapter-native OpenAI streaming." +position: 29 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"fabric.openai_stream/v1alpha1\";"}} />
+ +Southbound protocol version for adapter-native OpenAI streaming. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx index af8e4e58..d9960308 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx @@ -2,7 +2,7 @@ title: "Enum Error Stage" sidebar-title: "ErrorStage" description: "NeMo Fabric lifecycle stage associated with an error." -position: 14 +position: 20 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx new file mode 100644 index 00000000..5feb51ac --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx @@ -0,0 +1,108 @@ +--- +title: "Enum OpenAI Chat Completion Chunk Object" +sidebar-title: "OpenAiChatCompletionChunkObject" +description: "Exact OpenAI object discriminator accepted by the v1 chunk profile." +position: 21 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum OpenAiChatCompletionChunkObject { + ChatCompletionChunk, +} +``` + +Exact OpenAI object discriminator accepted by the v1 chunk profile. + +## Variants + +### `ChatCompletionChunk` + +
+ +OpenAI Chat Completions streaming chunk. + +## Trait Implementations + +### `impl Clone for OpenAiChatCompletionChunkObject` + +
Clone for OpenAiChatCompletionChunkObject"}} />
+ +#### `clone` + +
clone(&self) -> OpenAiChatCompletionChunkObject"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for OpenAiChatCompletionChunkObject` + +
Debug for OpenAiChatCompletionChunkObject"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for OpenAiChatCompletionChunkObject` + +
Deserialize<'de> for OpenAiChatCompletionChunkObject"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for OpenAiChatCompletionChunkObject` + +
OpenAiChatCompletionChunkObject"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for OpenAiChatCompletionChunkObject` + +
PartialEq for OpenAiChatCompletionChunkObject"}} />
+ +#### `eq` + +
eq(&self, other: &OpenAiChatCompletionChunkObject) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for OpenAiChatCompletionChunkObject` + +
Serialize for OpenAiChatCompletionChunkObject"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for OpenAiChatCompletionChunkObject` + +
Copy for OpenAiChatCompletionChunkObject"}} />
+ +### `impl Eq for OpenAiChatCompletionChunkObject` + +
Eq for OpenAiChatCompletionChunkObject"}} />
+ +### `impl StructuralPartialEq for OpenAiChatCompletionChunkObject` + +
StructuralPartialEq for OpenAiChatCompletionChunkObject"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx new file mode 100644 index 00000000..40cb0086 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx @@ -0,0 +1,108 @@ +--- +title: "Enum OpenAI Stream Host" +sidebar-title: "OpenAiStreamHost" +description: "Supported native-streaming listener host." +position: 22 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum OpenAiStreamHost { + Ipv4Loopback, +} +``` + +Supported native-streaming listener host. + +## Variants + +### `Ipv4Loopback` + +
+ +SDK-owned IPv4 loopback listener. + +## Trait Implementations + +### `impl Clone for OpenAiStreamHost` + +
Clone for OpenAiStreamHost"}} />
+ +#### `clone` + +
clone(&self) -> OpenAiStreamHost"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for OpenAiStreamHost` + +
Debug for OpenAiStreamHost"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for OpenAiStreamHost` + +
Deserialize<'de> for OpenAiStreamHost"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for OpenAiStreamHost` + +
OpenAiStreamHost"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for OpenAiStreamHost` + +
PartialEq for OpenAiStreamHost"}} />
+ +#### `eq` + +
eq(&self, other: &OpenAiStreamHost) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for OpenAiStreamHost` + +
Serialize for OpenAiStreamHost"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for OpenAiStreamHost` + +
Copy for OpenAiStreamHost"}} />
+ +### `impl Eq for OpenAiStreamHost` + +
Eq for OpenAiStreamHost"}} />
+ +### `impl StructuralPartialEq for OpenAiStreamHost` + +
StructuralPartialEq for OpenAiStreamHost"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx new file mode 100644 index 00000000..2dbdf95f --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx @@ -0,0 +1,108 @@ +--- +title: "Enum OpenAI Stream Profile" +sidebar-title: "OpenAiStreamProfile" +description: "Supported OpenAI-compatible chunk profile." +position: 23 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum OpenAiStreamProfile { + ChatCompletionsChunkV1, +} +``` + +Supported OpenAI-compatible chunk profile. + +## Variants + +### `ChatCompletionsChunkV1` + +
+ +OpenAI Chat Completions chunk objects. + +## Trait Implementations + +### `impl Clone for OpenAiStreamProfile` + +
Clone for OpenAiStreamProfile"}} />
+ +#### `clone` + +
clone(&self) -> OpenAiStreamProfile"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for OpenAiStreamProfile` + +
Debug for OpenAiStreamProfile"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for OpenAiStreamProfile` + +
Deserialize<'de> for OpenAiStreamProfile"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for OpenAiStreamProfile` + +
OpenAiStreamProfile"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for OpenAiStreamProfile` + +
PartialEq for OpenAiStreamProfile"}} />
+ +#### `eq` + +
eq(&self, other: &OpenAiStreamProfile) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for OpenAiStreamProfile` + +
Serialize for OpenAiStreamProfile"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for OpenAiStreamProfile` + +
Copy for OpenAiStreamProfile"}} />
+ +### `impl Eq for OpenAiStreamProfile` + +
Eq for OpenAiStreamProfile"}} />
+ +### `impl StructuralPartialEq for OpenAiStreamProfile` + +
StructuralPartialEq for OpenAiStreamProfile"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx new file mode 100644 index 00000000..fe01b847 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx @@ -0,0 +1,108 @@ +--- +title: "Enum OpenAI Stream Protocol Version" +sidebar-title: "OpenAiStreamProtocolVersion" +description: "Supported southbound native-streaming protocol version." +position: 24 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum OpenAiStreamProtocolVersion { + V1Alpha1, +} +``` + +Supported southbound native-streaming protocol version. + +## Variants + +### `V1Alpha1` + +
+ +Initial authenticated loopback HTTP and chunked-NDJSON protocol. + +## Trait Implementations + +### `impl Clone for OpenAiStreamProtocolVersion` + +
Clone for OpenAiStreamProtocolVersion"}} />
+ +#### `clone` + +
clone(&self) -> OpenAiStreamProtocolVersion"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for OpenAiStreamProtocolVersion` + +
Debug for OpenAiStreamProtocolVersion"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for OpenAiStreamProtocolVersion` + +
Deserialize<'de> for OpenAiStreamProtocolVersion"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for OpenAiStreamProtocolVersion` + +
OpenAiStreamProtocolVersion"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for OpenAiStreamProtocolVersion` + +
PartialEq for OpenAiStreamProtocolVersion"}} />
+ +#### `eq` + +
eq(&self, other: &OpenAiStreamProtocolVersion) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for OpenAiStreamProtocolVersion` + +
Serialize for OpenAiStreamProtocolVersion"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for OpenAiStreamProtocolVersion` + +
Copy for OpenAiStreamProtocolVersion"}} />
+ +### `impl Eq for OpenAiStreamProtocolVersion` + +
Eq for OpenAiStreamProtocolVersion"}} />
+ +### `impl StructuralPartialEq for OpenAiStreamProtocolVersion` + +
StructuralPartialEq for OpenAiStreamProtocolVersion"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx new file mode 100644 index 00000000..55cb1ec0 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx @@ -0,0 +1,142 @@ +--- +title: "Enum OpenAI Stream Record" +sidebar-title: "OpenAiStreamRecord" +description: "One correlated NDJSON record on the adapter-native stream channel." +position: 25 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
u64,\n        runtime_id: String,\n        invocation_id: String,\n        request_id: String,\n        chunk: OpenAiChatCompletionChunk,\n    },\n    End {\n        sequence: u64,\n        runtime_id: String,\n        invocation_id: String,\n        request_id: String,\n    },\n}"}} />
+ +One correlated NDJSON record on the adapter-native stream channel. + +## Variants + +### `Chunk` + +
+ +One OpenAI Chat Completions chunk. + +#### Fields + +##### `sequence: u64` + +Monotonic zero-based record sequence. + +##### `runtime_id: String` + +Runtime id for stream correlation. + +##### `invocation_id: String` + +Invocation id for stream correlation. + +##### `request_id: String` + +Request id for stream correlation. + +##### `chunk: OpenAiChatCompletionChunk` + +OpenAI-compatible chunk passed through to the consumer. + +### `End` + +
+ +Explicit successful end of the progressive event channel. + +#### Fields + +##### `sequence: u64` + +Monotonic zero-based record sequence. + +##### `runtime_id: String` + +Runtime id for stream correlation. + +##### `invocation_id: String` + +Invocation id for stream correlation. + +##### `request_id: String` + +Request id for stream correlation. + +## Trait Implementations + +### `impl Clone for OpenAiStreamRecord` + +
Clone for OpenAiStreamRecord"}} />
+ +#### `clone` + +
clone(&self) -> OpenAiStreamRecord"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for OpenAiStreamRecord` + +
Debug for OpenAiStreamRecord"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for OpenAiStreamRecord` + +
Deserialize<'de> for OpenAiStreamRecord"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for OpenAiStreamRecord` + +
OpenAiStreamRecord"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for OpenAiStreamRecord` + +
PartialEq for OpenAiStreamRecord"}} />
+ +#### `eq` + +
eq(&self, other: &OpenAiStreamRecord) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for OpenAiStreamRecord` + +
Serialize for OpenAiStreamRecord"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for OpenAiStreamRecord` + +
StructuralPartialEq for OpenAiStreamRecord"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx index a4cefeb0..51fa75b4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx @@ -2,7 +2,7 @@ title: "Enum RunStatus" sidebar-title: "RunStatus" description: "Runtime completion status." -position: 15 +position: 26 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx new file mode 100644 index 00000000..5bbc15ce --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx @@ -0,0 +1,14 @@ +--- +title: "Function invoke_openai_stream" +sidebar-title: "invoke_openai_stream" +description: "Invoke a started harness runtime and pass through native OpenAI chat-completion chunks." +position: 30 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
RunPlan,\n    runtime: &RuntimeHandle,\n    request: RunRequest,\n    transport: OpenAiStreamTransport,\n) -> Result<RunResult>"}} />
+ +Invoke a started harness runtime and pass through native OpenAI chat-completion chunks. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx index 1e5e3a73..f08f89e6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx @@ -2,7 +2,7 @@ title: "Function invoke_runtime" sidebar-title: "invoke_runtime" description: "Invoke a started harness runtime." -position: 16 +position: 31 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-environment.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-environment.mdx index 73c2275a..3ddb0319 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-environment.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-environment.mdx @@ -2,7 +2,7 @@ title: "Function prepare_environment" sidebar-title: "prepare_environment" description: "Resolve or attach to the execution environment context for a run plan." -position: 17 +position: 32 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx index 297cb504..4f56332b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx @@ -2,7 +2,7 @@ title: "Function run_plan" sidebar-title: "run_plan" description: "Invoke a NeMo Fabric run plan." -position: 18 +position: 33 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx index ae6c0acb..3c0c120d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime" sidebar-title: "start_runtime" description: "Start or connect to a harness runtime." -position: 19 +position: 34 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx index 3d5bae34..0a895471 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx @@ -2,7 +2,7 @@ title: "Function stop_runtime" sidebar-title: "stop_runtime" description: "Stop or detach from a harness runtime." -position: 20 +position: 35 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index fd6af663..4df3a9e7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 101 +position: 116 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -20,6 +20,12 @@ Runtime invocation helpers. - [ErrorInfo](struct-errorinfo.mdx): Normalized error metadata. - [FabricEvent](struct-fabricevent.mdx): NeMo Fabric lifecycle or progress event. - [InvocationHandle](struct-invocationhandle.mdx): One request sent to a runtime. +- [OpenAiChatCompletionChunk](struct-openaichatcompletionchunk.mdx): OpenAI Chat Completions chunk accepted by the native streaming profile. +- [OpenAiChatCompletionChunkChoice](struct-openaichatcompletionchunkchoice.mdx): One choice within an OpenAI Chat Completions streaming chunk. +- [OpenAiChatCompletionChunkDelta](struct-openaichatcompletionchunkdelta.mdx): Incremental assistant message fields carried by one OpenAI choice. +- [OpenAiStreamInvocation](struct-openaistreaminvocation.mdx): One adapter-native OpenAI streaming invocation. +- [OpenAiStreamSink](struct-openaistreamsink.mdx): Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric. +- [OpenAiStreamTransport](struct-openaistreamtransport.mdx): SDK-owned loopback transport for one native OpenAI streaming invocation. - [RunRequest](struct-runrequest.mdx): A request passed to a NeMo Fabric-managed harness runtime. - [RunResult](struct-runresult.mdx): Result from a NeMo Fabric-managed harness invocation. - [RuntimeContext](struct-runtimecontext.mdx): Context generated for one invocation of a started runtime. @@ -30,10 +36,22 @@ Runtime invocation helpers. ## Enums - [ErrorStage](enum-errorstage.mdx): NeMo Fabric lifecycle stage associated with an error. +- [OpenAiChatCompletionChunkObject](enum-openaichatcompletionchunkobject.mdx): Exact OpenAI object discriminator accepted by the v1 chunk profile. +- [OpenAiStreamHost](enum-openaistreamhost.mdx): Supported native-streaming listener host. +- [OpenAiStreamProfile](enum-openaistreamprofile.mdx): Supported OpenAI-compatible chunk profile. +- [OpenAiStreamProtocolVersion](enum-openaistreamprotocolversion.mdx): Supported southbound native-streaming protocol version. +- [OpenAiStreamRecord](enum-openaistreamrecord.mdx): One correlated NDJSON record on the adapter-native stream channel. - [RunStatus](enum-runstatus.mdx): Runtime completion status. +## Constants + +- [OPENAI_CHAT_COMPLETIONS_CHUNK_PROFILE](constant-openai-chat-completions-chunk-profile.mdx): OpenAI event profile supported by the initial native streaming contract. +- [OPENAI_STREAM_HOST](constant-openai-stream-host.mdx): SDK-owned loopback host for adapter-native OpenAI streaming. +- [OPENAI_STREAM_PROTOCOL_VERSION](constant-openai-stream-protocol-version.mdx): Southbound protocol version for adapter-native OpenAI streaming. + ## Functions +- [invoke_openai_stream](fn-invoke-openai-stream.mdx): Invoke a started harness runtime and pass through native OpenAI chat-completion chunks. - [invoke_runtime](fn-invoke-runtime.mdx): Invoke a started harness runtime. - [prepare_environment](fn-prepare-environment.mdx): Resolve or attach to the execution environment context for a run plan. - [run_plan](fn-run-plan.mdx): Invoke a NeMo Fabric run plan. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx new file mode 100644 index 00000000..026b2de7 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx @@ -0,0 +1,118 @@ +--- +title: "Struct OpenAI Chat Completion Chunk" +sidebar-title: "OpenAiChatCompletionChunk" +description: "OpenAI Chat Completions chunk accepted by the native streaming profile." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub object: OpenAiChatCompletionChunkObject,\n    pub created: u64,\n    pub model: String,\n    pub choices: Vec<OpenAiChatCompletionChunkChoice>,\n    pub usage: Option<BTreeMap<String, Value>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+ +OpenAI Chat Completions chunk accepted by the native streaming profile. + +## Fields + +### `id: String` + +Provider-generated response identifier. + +### `object: OpenAiChatCompletionChunkObject` + +Exact OpenAI streaming object discriminator. + +### `created: u64` + +Unix timestamp in seconds. + +### `model: String` + +Model identifier. + +### `choices: Vec` + +Incremental choices; an explicit usage-only chunk can contain none. + +### `usage: Option>` + +Optional token-usage data. + +### `extensions: BTreeMap` + +Additional OpenAI-compatible top-level fields preserved by pass-through. + +## Trait Implementations + +### `impl Clone for OpenAiChatCompletionChunk` + +
Clone for OpenAiChatCompletionChunk"}} />
+ +#### `clone` + +
clone(&self) -> OpenAiChatCompletionChunk"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for OpenAiChatCompletionChunk` + +
Debug for OpenAiChatCompletionChunk"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for OpenAiChatCompletionChunk` + +
Deserialize<'de> for OpenAiChatCompletionChunk"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for OpenAiChatCompletionChunk` + +
OpenAiChatCompletionChunk"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for OpenAiChatCompletionChunk` + +
PartialEq for OpenAiChatCompletionChunk"}} />
+ +#### `eq` + +
eq(&self, other: &OpenAiChatCompletionChunk) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for OpenAiChatCompletionChunk` + +
Serialize for OpenAiChatCompletionChunk"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for OpenAiChatCompletionChunk` + +
StructuralPartialEq for OpenAiChatCompletionChunk"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx new file mode 100644 index 00000000..94b2e70e --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx @@ -0,0 +1,110 @@ +--- +title: "Struct OpenAI Chat Completion Chunk Choice" +sidebar-title: "OpenAiChatCompletionChunkChoice" +description: "One choice within an OpenAI Chat Completions streaming chunk." +position: 9 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
u32,\n    pub delta: OpenAiChatCompletionChunkDelta,\n    pub finish_reason: Option<String>,\n    pub logprobs: Option<BTreeMap<String, Value>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+ +One choice within an OpenAI Chat Completions streaming chunk. + +## Fields + +### `index: u32` + +Choice index within the response. + +### `delta: OpenAiChatCompletionChunkDelta` + +Incremental assistant message fields. + +### `finish_reason: Option` + +Terminal reason when this choice finishes. + +### `logprobs: Option>` + +Incremental log-probability information. + +### `extensions: BTreeMap` + +Additional OpenAI-compatible choice fields preserved by pass-through. + +## Trait Implementations + +### `impl Clone for OpenAiChatCompletionChunkChoice` + +
Clone for OpenAiChatCompletionChunkChoice"}} />
+ +#### `clone` + +
clone(&self) -> OpenAiChatCompletionChunkChoice"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for OpenAiChatCompletionChunkChoice` + +
Debug for OpenAiChatCompletionChunkChoice"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for OpenAiChatCompletionChunkChoice` + +
Deserialize<'de> for OpenAiChatCompletionChunkChoice"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for OpenAiChatCompletionChunkChoice` + +
OpenAiChatCompletionChunkChoice"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for OpenAiChatCompletionChunkChoice` + +
PartialEq for OpenAiChatCompletionChunkChoice"}} />
+ +#### `eq` + +
eq(&self, other: &OpenAiChatCompletionChunkChoice) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for OpenAiChatCompletionChunkChoice` + +
Serialize for OpenAiChatCompletionChunkChoice"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for OpenAiChatCompletionChunkChoice` + +
StructuralPartialEq for OpenAiChatCompletionChunkChoice"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx new file mode 100644 index 00000000..845afed4 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx @@ -0,0 +1,114 @@ +--- +title: "Struct OpenAI Chat Completion Chunk Delta" +sidebar-title: "OpenAiChatCompletionChunkDelta" +description: "Incremental assistant message fields carried by one OpenAI choice." +position: 10 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<String>,\n    pub refusal: Option<String>,\n    pub role: Option<String>,\n    pub function_call: Option<BTreeMap<String, Value>>,\n    pub tool_calls: Option<Vec<BTreeMap<String, Value>>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+ +Incremental assistant message fields carried by one OpenAI choice. + +## Fields + +### `content: Option` + +Incremental text content. + +### `refusal: Option` + +Incremental refusal content. + +### `role: Option` + +Incremental message role. + +### `function_call: Option>` + +Legacy incremental function-call fields. + +### `tool_calls: Option>>` + +Incremental tool-call fields. + +### `extensions: BTreeMap` + +Additional OpenAI-compatible delta fields preserved by pass-through. + +## Trait Implementations + +### `impl Clone for OpenAiChatCompletionChunkDelta` + +
Clone for OpenAiChatCompletionChunkDelta"}} />
+ +#### `clone` + +
clone(&self) -> OpenAiChatCompletionChunkDelta"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for OpenAiChatCompletionChunkDelta` + +
Debug for OpenAiChatCompletionChunkDelta"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for OpenAiChatCompletionChunkDelta` + +
Deserialize<'de> for OpenAiChatCompletionChunkDelta"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for OpenAiChatCompletionChunkDelta` + +
OpenAiChatCompletionChunkDelta"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for OpenAiChatCompletionChunkDelta` + +
PartialEq for OpenAiChatCompletionChunkDelta"}} />
+ +#### `eq` + +
eq(&self, other: &OpenAiChatCompletionChunkDelta) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for OpenAiChatCompletionChunkDelta` + +
Serialize for OpenAiChatCompletionChunkDelta"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for OpenAiChatCompletionChunkDelta` + +
StructuralPartialEq for OpenAiChatCompletionChunkDelta"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx new file mode 100644 index 00000000..372c5908 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx @@ -0,0 +1,102 @@ +--- +title: "Struct OpenAI Stream Invocation" +sidebar-title: "OpenAiStreamInvocation" +description: "One adapter-native OpenAI streaming invocation." +position: 11 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
RuntimeContext,\n    pub request: RunRequest,\n    pub stream: OpenAiStreamSink,\n}"}} />
+ +One adapter-native OpenAI streaming invocation. + +## Fields + +### `runtime_context: RuntimeContext` + +Invocation context generated by NeMo Fabric. + +### `request: RunRequest` + +Typed caller request for this invocation. + +### `stream: OpenAiStreamSink` + +Authenticated progressive-output sink for this invocation. + +## Trait Implementations + +### `impl Clone for OpenAiStreamInvocation` + +
Clone for OpenAiStreamInvocation"}} />
+ +#### `clone` + +
clone(&self) -> OpenAiStreamInvocation"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for OpenAiStreamInvocation` + +
Debug for OpenAiStreamInvocation"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for OpenAiStreamInvocation` + +
Deserialize<'de> for OpenAiStreamInvocation"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for OpenAiStreamInvocation` + +
OpenAiStreamInvocation"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for OpenAiStreamInvocation` + +
PartialEq for OpenAiStreamInvocation"}} />
+ +#### `eq` + +
eq(&self, other: &OpenAiStreamInvocation) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for OpenAiStreamInvocation` + +
Serialize for OpenAiStreamInvocation"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for OpenAiStreamInvocation` + +
StructuralPartialEq for OpenAiStreamInvocation"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx new file mode 100644 index 00000000..8ad3827d --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx @@ -0,0 +1,122 @@ +--- +title: "Struct OpenAI Stream Sink" +sidebar-title: "OpenAiStreamSink" +description: "Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric." +position: 12 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
OpenAiStreamProtocolVersion,\n    pub profile: OpenAiStreamProfile,\n    pub host: OpenAiStreamHost,\n    pub port: u16,\n    pub token: String,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n}"}} />
+ +Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric. + +## Fields + +### `protocol_version: OpenAiStreamProtocolVersion` + +Southbound stream protocol version. + +### `profile: OpenAiStreamProfile` + +OpenAI event profile emitted on this stream. + +### `host: OpenAiStreamHost` + +Loopback host owned by the SDK listener. + +### `port: u16` + +Loopback TCP port owned by the SDK listener. + +### `token: String` + +Single-use bearer token. Adapters must not log or persist this value. + +### `runtime_id: String` + +Runtime id for stream correlation. + +### `invocation_id: String` + +Invocation id for stream correlation. + +### `request_id: String` + +Request id for stream correlation. + +## Trait Implementations + +### `impl Clone for OpenAiStreamSink` + +
Clone for OpenAiStreamSink"}} />
+ +#### `clone` + +
clone(&self) -> OpenAiStreamSink"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for OpenAiStreamSink` + +
Debug for OpenAiStreamSink"}} />
+ +#### `fmt` + +
fmt(&self, formatter: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for OpenAiStreamSink` + +
Deserialize<'de> for OpenAiStreamSink"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for OpenAiStreamSink` + +
OpenAiStreamSink"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for OpenAiStreamSink` + +
PartialEq for OpenAiStreamSink"}} />
+ +#### `eq` + +
eq(&self, other: &OpenAiStreamSink) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for OpenAiStreamSink` + +
Serialize for OpenAiStreamSink"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for OpenAiStreamSink` + +
StructuralPartialEq for OpenAiStreamSink"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx new file mode 100644 index 00000000..2ed1e5c6 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx @@ -0,0 +1,98 @@ +--- +title: "Struct OpenAI Stream Transport" +sidebar-title: "OpenAiStreamTransport" +description: "SDK-owned loopback transport for one native OpenAI streaming invocation." +position: 13 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
u16,\n    pub token: String,\n}"}} />
+ +SDK-owned loopback transport for one native OpenAI streaming invocation. + +## Fields + +### `port: u16` + +Loopback TCP port owned by the SDK listener. + +### `token: String` + +Single-use bearer token used to authenticate the adapter connection. + +## Trait Implementations + +### `impl Clone for OpenAiStreamTransport` + +
Clone for OpenAiStreamTransport"}} />
+ +#### `clone` + +
clone(&self) -> OpenAiStreamTransport"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for OpenAiStreamTransport` + +
Debug for OpenAiStreamTransport"}} />
+ +#### `fmt` + +
fmt(&self, formatter: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for OpenAiStreamTransport` + +
Deserialize<'de> for OpenAiStreamTransport"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for OpenAiStreamTransport` + +
OpenAiStreamTransport"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for OpenAiStreamTransport` + +
PartialEq for OpenAiStreamTransport"}} />
+ +#### `eq` + +
eq(&self, other: &OpenAiStreamTransport) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for OpenAiStreamTransport` + +
Serialize for OpenAiStreamTransport"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for OpenAiStreamTransport` + +
StructuralPartialEq for OpenAiStreamTransport"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx index 513777d2..dfaf781b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx @@ -2,7 +2,7 @@ title: "Struct RunRequest" sidebar-title: "RunRequest" description: "A request passed to a NeMo Fabric-managed harness runtime." -position: 8 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 79a0d674..b29a4b8a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -2,7 +2,7 @@ title: "Struct RunResult" sidebar-title: "RunResult" description: "Result from a NeMo Fabric-managed harness invocation." -position: 9 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx index 070c793a..c2746339 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Context" sidebar-title: "RuntimeContext" description: "Context generated for one invocation of a started runtime." -position: 10 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx index 49394e7d..d5f209be 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Handle" sidebar-title: "RuntimeHandle" description: "Active or resumable harness runtime." -position: 11 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx index ff03e8ee..72aa9124 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Telemetry Context" sidebar-title: "RuntimeTelemetryContext" description: "Runtime telemetry config passed to adapters." -position: 12 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx index 1ae9136e..348a4cd0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Ref" sidebar-title: "TelemetryRef" description: "Reference to telemetry emitted by Relay or another configured telemetry path." -position: 13 +position: 19 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index fea6cf94..8fe3f43c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -18,6 +18,8 @@ pub enum SchemaName { AdapterDescriptor, RunPlan, AdapterInvocation, + OpenAiStreamInvocation, + OpenAiStreamRecord, RuntimeContext, EnvironmentHandle, RuntimeHandle, @@ -76,6 +78,18 @@ Resolved run plan schema. Initialized-runtime invocation payload schema. +### `OpenAiStreamInvocation` + +
+ +Adapter-facing native OpenAI streaming invocation schema. + +### `OpenAiStreamRecord` + +
+ +Adapter-native OpenAI streaming NDJSON record schema. + ### `RuntimeContext`
@@ -138,7 +152,7 @@ NeMo Fabric lifecycle event schema. #### `ALL` -
16]"}} />
+
18]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 7a1bf0fb..fa7d5749 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 102 +position: 117 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 2fa8de2c..ab94fd26 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -365,8 +365,9 @@ The following table summarizes the SDK entry points and runtime methods: | `Fabric.plan(config, base_dir=...)` | No | You need to inspect the selected adapter, capability mapping, and runtime capabilities before running. | Does not start a runtime. | | `Fabric.doctor(config, *, base_dir=...)` | Yes | You need preflight diagnostics for adapter resolution, capability routing, declared requirements, and environment assumptions. | Checks can inspect local binaries, environment variables, and files without starting a runtime. | | `Fabric.run(config, base_dir=..., input=...)` | Yes | You need one complete start, invoke, result, stop lifecycle. | `base_dir` is optional. Pass a `RunRequest` instead when the invocation needs IDs, context, or overrides. | -| `Fabric.start_runtime(config, ..., streaming=False)` | Yes | You need state across multiple ordered invocations. | Returns a `Runtime`. Use `streaming=True` with NeMo Relay enabled to provision streaming. | +| `Fabric.start_runtime(config, ..., streaming=False)` | Yes | You need state across multiple ordered invocations. | Returns a `Runtime`. Use `streaming=True` with NeMo Relay enabled only to provision Relay ATOF streaming. | | `Runtime.invoke(...)` | Yes | You need one turn on an existing runtime. | A runtime permits one active invocation at a time. | +| `Runtime.invoke_openai_stream(...)` | No | You need adapter-native OpenAI Chat Completions chunks for one turn. | Requires descriptor `capabilities.streaming`. Returns an async `OpenAIInvokeStream`; await `stream.result()` for the separate terminal `RunResult`. | | `Runtime.invoke_stream(...)` | No | You need live ATOF records generated by NeMo Relay for one turn. | Returns an async `InvokeStream`; await `stream.result()` for the terminal `RunResult`. | | `Runtime.stop()` | Yes | You need to stop or detach from the runtime. | Called automatically when using `async with`. | @@ -430,15 +431,62 @@ on one live thread, Deep Agents invokes one compiled graph and checkpointer, Hermes Agent reuses one agent and session database, and Claude keeps one connected SDK client. Harness-native identifiers remain adapter-internal. +## Native OpenAI Streaming + +Use `Runtime.invoke_openai_stream()` when the selected adapter provides native +OpenAI Chat Completions chunks. `Runtime.supports_openai_streaming` reflects the +selected descriptor's existing `capabilities.streaming` claim. If the claim is +false, `invoke_openai_stream()` raises `FabricCapabilityError` with code +`openai_streaming_unavailable`. + +```python +from nemo_fabric import Fabric + +async with await Fabric().start_runtime(config) as runtime: + if not runtime.supports_openai_streaming: + raise RuntimeError("the selected adapter does not support OpenAI streaming") + + stream = runtime.invoke_openai_stream(input="Review the latest patch") + async for chunk in stream: + print(chunk) + + result = await stream.result() + print(result.status, result.output) +``` + +`invoke_openai_stream(...)` is synchronous and starts exactly one adapter +invocation in the background. Its `OpenAIInvokeStream` yields only JSON mappings +whose `object` field is `chat.completion.chunk`. An empty stream is valid. The +terminal normalized `RunResult` is not a final chunk; obtain it separately with +`await stream.result()` and treat it as authoritative. + +Only one turn can be active on a runtime. Fully consume the stream before +starting another turn. If iteration stops early, call `await stream.aclose()`; +it drains the invocation and discards unread chunks so the runtime can be reused. +It does not cancel the target invocation. Awaiting `stream.result()` also drains +and discards any unread chunks before it returns, so consume the iterator first +when the application needs every chunk. + +The SDK owns an authenticated listener on the loopback interface and supplies +the adapter host with a single-invocation HTTP sink that uses chunked NDJSON +framing. It validates chunk shape, framing, authentication, and invocation +correlation. This transport is an implementation detail; consumers provide no +listener address, token, or protocol configuration. + +Native OpenAI streaming does not require NeMo Relay or +`start_runtime(..., streaming=True)`. It is independent of Relay ATOF streaming: +`supports_openai_streaming` and `invoke_openai_stream()` describe adapter-native +OpenAI chunks, while `supports_streaming` and `invoke_stream()` describe Relay +ATOF. Enabling one does not enable or change the other. + ## NeMo Relay Streaming Enable NeMo Relay before starting a runtime to consume raw ATOF records while a turn runs. `Runtime.supports_streaming` reports whether this NeMo Relay path is available. It is separate from `RuntimeCapabilities.streaming`, which describes adapter-native progressive output. This separation is intentional: -`Runtime.invoke_stream()` exposes only ATOF records generated by NeMo Relay. A future -normalized NeMo Fabric streaming contract will address adapter-native progressive -output, such as Codex app-server message and item deltas. +`Runtime.invoke_stream()` exposes only ATOF records generated by NeMo Relay; it +does not call the adapter's native OpenAI stream operation. The following example streams one invocation and collects its terminal result: diff --git a/python/src/nemo_fabric/__init__.py b/python/src/nemo_fabric/__init__.py index 1183f401..6268e1c5 100644 --- a/python/src/nemo_fabric/__init__.py +++ b/python/src/nemo_fabric/__init__.py @@ -40,6 +40,7 @@ from nemo_fabric.models import ToolDefinitionConfig from nemo_fabric.models import WorkflowConfig from nemo_fabric.models import WorkflowEntrypointConfig +from nemo_fabric.openai_streaming import OpenAIInvokeStream from nemo_fabric.runtime import Runtime from nemo_fabric.runtime import RuntimeStatus from nemo_fabric.streaming import InvokeStream @@ -80,6 +81,7 @@ "McpServerConfig", "MetadataConfig", "ModelConfig", + "OpenAIInvokeStream", "RelayAtifConfig", "RelayAtofConfig", "RelayAtofFileSinkConfig", diff --git a/python/src/nemo_fabric/_native.pyi b/python/src/nemo_fabric/_native.pyi index 8df4b4bc..bc5254a9 100644 --- a/python/src/nemo_fabric/_native.pyi +++ b/python/src/nemo_fabric/_native.pyi @@ -24,4 +24,10 @@ def invoke_runtime( runtime_json: str, request_json: str, ) -> str: ... +def invoke_openai_stream( + plan_json: str, + runtime_json: str, + request_json: str, + transport_json: str, +) -> str: ... def stop_runtime(plan_json: str, runtime_json: str) -> str: ... diff --git a/python/src/nemo_fabric/openai_streaming.py b/python/src/nemo_fabric/openai_streaming.py new file mode 100644 index 00000000..ec6a3adb --- /dev/null +++ b/python/src/nemo_fabric/openai_streaming.py @@ -0,0 +1,695 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Adapter-native OpenAI streaming for the NVIDIA NeMo Fabric Python SDK.""" + +from __future__ import annotations + +import asyncio +import json +import secrets +from collections.abc import Awaitable, Callable +from contextlib import suppress +from typing import Any + +from nemo_fabric.errors import FabricRuntimeError +from nemo_fabric.types import RunResult + +_END = object() +_HOST = "127.0.0.1" +_OPENAI_STREAM_COMPLETION_TIMEOUT = 10.0 +_OPENAI_STREAM_HEADER_TIMEOUT = 10.0 +_MAX_PENDING_CONNECTIONS = 8 +_MAX_RECORD_BYTES = 1024 * 1024 +_QUEUE_MAX_BYTES = 16 * 1024 * 1024 +_QUEUE_MAXSIZE = 1024 +_READ_SIZE = 64 * 1024 +_STREAM_PATH = "/openai-stream" +_UINT32_MAX = (1 << 32) - 1 +_UINT64_MAX = (1 << 64) - 1 + + +class _ProtocolError(ValueError): + def __init__(self, message: str, status: int = 400) -> None: + super().__init__(message) + self.status = status + + +class _ChunkQueue: + def __init__(self, *, maxsize: int, max_bytes: int) -> None: + self._queue: asyncio.Queue[tuple[dict[str, Any] | object, int]] = ( + asyncio.Queue(maxsize=maxsize) + ) + self._max_bytes = max_bytes + self._queued_bytes = 0 + self._space_available = asyncio.Event() + self._space_available.set() + + def empty(self) -> bool: + return self._queue.empty() + + async def put(self, item: dict[str, Any] | object, size: int = 0) -> None: + if size > self._max_bytes: + raise _ProtocolError("OpenAI stream record exceeds the queue byte limit", 413) + while self._queued_bytes + size > self._max_bytes: + self._space_available.clear() + await self._space_available.wait() + self._queued_bytes += size + try: + await self._queue.put((item, size)) + except BaseException: + self._queued_bytes -= size + self._space_available.set() + raise + + async def get(self) -> dict[str, Any] | object: + item, size = await self._queue.get() + self._queued_bytes -= size + self._space_available.set() + return item + + def get_nowait(self) -> dict[str, Any] | object: + item, size = self._queue.get_nowait() + self._queued_bytes -= size + self._space_available.set() + return item + + +class _OpenAIStreamListener: + """Receive one authenticated chunked-NDJSON OpenAI event stream.""" + + def __init__( + self, + *, + runtime_id: str, + request_id: str, + port: int = 0, + maxsize: int = _QUEUE_MAXSIZE, + max_bytes: int = _QUEUE_MAX_BYTES, + max_record_bytes: int = _MAX_RECORD_BYTES, + ) -> None: + self._runtime_id = runtime_id + self._request_id = request_id + self._port = port + self._token = secrets.token_urlsafe(32) + self._queue = _ChunkQueue(maxsize=maxsize, max_bytes=max_bytes) + self._max_record_bytes = min(max_record_bytes, max_bytes) + self._server: asyncio.Server | None = None + self._bound_port: int | None = None + self._invocation_id: str | None = None + self._expected_sequence = 0 + self._connected = False + self._ended = False + self._discard = False + self._error: FabricRuntimeError | None = None + self._completion = asyncio.Event() + self._tasks: set[asyncio.Task[None]] = set() + self._writers: set[asyncio.StreamWriter] = set() + + @property + def transport(self) -> dict[str, Any]: + if self._bound_port is None: + raise RuntimeError("OpenAI stream listener is not started") + return {"port": self._bound_port, "token": self._token} + + @property + def invocation_id(self) -> str | None: + return self._invocation_id + + @property + def error(self) -> FabricRuntimeError | None: + return self._error + + @property + def records(self) -> _ChunkQueue: + return self._queue + + async def start(self) -> None: + self._server = await asyncio.start_server(self._accept, _HOST, self._port) + socket = self._server.sockets[0] + self._bound_port = int(socket.getsockname()[1]) + + def discard(self) -> None: + self._discard = True + + async def wait_completed(self) -> None: + await self._completion.wait() + + def fail(self, message: str) -> None: + self._set_error(message) + self._completion.set() + + def _accept( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + ) -> None: + if len(self._tasks) >= _MAX_PENDING_CONNECTIONS: + writer.close() + return + task = asyncio.create_task(self._handle_client(reader, writer)) + self._tasks.add(task) + task.add_done_callback(self._task_done) + + def _task_done(self, task: asyncio.Task[None]) -> None: + self._tasks.discard(task) + if not task.cancelled(): + task.exception() + + async def _handle_client( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + ) -> None: + self._writers.add(writer) + claimed = False + try: + request = await asyncio.wait_for( + reader.readuntil(b"\r\n\r\n"), + _OPENAI_STREAM_HEADER_TIMEOUT, + ) + request_line, *header_lines = request[:-4].split(b"\r\n") + method, target, _version = request_line.decode("ascii").split(" ", 2) + headers = _http_headers(header_lines) + if method != "POST" or target.split("?", 1)[0] != _STREAM_PATH: + raise _ProtocolError("Unknown OpenAI stream endpoint", 404) + authorization = headers.get("authorization", "") + if not secrets.compare_digest(authorization, f"Bearer {self._token}"): + raise _ProtocolError("Invalid OpenAI stream authorization", 401) + if headers.get("content-type", "").split(";", 1)[0].strip() != ( + "application/x-ndjson" + ): + raise _ProtocolError("OpenAI stream must use NDJSON", 415) + transfer_codings = [ + coding.strip().lower() + for coding in headers.get("transfer-encoding", "").split(",") + if coding.strip() + ] + if not transfer_codings or transfer_codings[-1] != "chunked": + raise _ProtocolError("OpenAI stream must use chunked transfer encoding", 411) + if headers.get("expect", "").lower() != "100-continue": + raise _ProtocolError("OpenAI stream must use Expect: 100-continue", 417) + # Authentication and request validation are connection-local. Claim the + # invocation only after they succeed, with no await between check/set. + if self._connected: + raise _ProtocolError("OpenAI stream listener accepts one connection", 409) + self._connected = True + claimed = True + writer.write(b"HTTP/1.1 100 Continue\r\n\r\n") + await writer.drain() + buffer = bytearray() + await self._read_chunked(reader, buffer) + if buffer.strip(): + await self._emit_line(bytes(buffer)) + if not self._ended: + raise _ProtocolError("OpenAI stream ended without an end record") + await _write_http_response(writer, 200, "OK") + except _ProtocolError as error: + if claimed: + self._set_error(str(error)) + with suppress(ConnectionError): + await _write_http_response(writer, error.status, "Rejected") + except ( + UnicodeDecodeError, + ValueError, + RecursionError, + asyncio.IncompleteReadError, + asyncio.LimitOverrunError, + ): + if claimed: + self._set_error("OpenAI stream contained malformed transport data") + with suppress(ConnectionError): + await _write_http_response(writer, 400, "Bad Request") + except ConnectionError: + if claimed: + self._set_error("OpenAI stream connection closed before completion") + except asyncio.CancelledError: + raise + except Exception: + if claimed: + self._set_error("OpenAI stream transport failed while parsing records") + with suppress(ConnectionError): + await _write_http_response(writer, 400, "Bad Request") + finally: + if claimed and not self._ended and self._error is not None: + await self._queue.put(_END) + if claimed: + self._completion.set() + self._writers.discard(writer) + writer.close() + with suppress(ConnectionError): + await writer.wait_closed() + + async def _read_chunked( + self, + reader: asyncio.StreamReader, + buffer: bytearray, + ) -> None: + while True: + size_line = await reader.readline() + size = int(size_line.split(b";", 1)[0].strip(), 16) + if size == 0: + while True: + trailer = await reader.readline() + if trailer in (b"\r\n", b"\n"): + return + if trailer == b"": + raise _ProtocolError("Incomplete OpenAI stream trailers") + remaining = size + while remaining: + chunk = await reader.readexactly(min(_READ_SIZE, remaining)) + remaining -= len(chunk) + await self._feed(buffer, chunk) + if await reader.readexactly(2) != b"\r\n": + raise _ProtocolError("Invalid OpenAI stream chunk terminator") + + async def _feed(self, buffer: bytearray, chunk: bytes) -> None: + buffer.extend(chunk) + while True: + newline = buffer.find(b"\n") + if newline < 0: + if len(buffer) > self._max_record_bytes: + raise _ProtocolError("OpenAI stream record exceeds 1 MiB", 413) + return + if newline > self._max_record_bytes: + raise _ProtocolError("OpenAI stream record exceeds 1 MiB", 413) + line = bytes(buffer[:newline]) + del buffer[: newline + 1] + await self._emit_line(line) + + async def _emit_line(self, line: bytes) -> None: + stripped = line.strip() + if not stripped: + return + if len(stripped) > self._max_record_bytes: + raise _ProtocolError("OpenAI stream record exceeds 1 MiB", 413) + try: + record = json.loads(stripped, parse_constant=_reject_json_constant) + except (json.JSONDecodeError, RecursionError, ValueError) as error: + raise _ProtocolError("OpenAI stream record is not valid JSON") from error + if not isinstance(record, dict): + raise _ProtocolError("OpenAI stream record must be a mapping") + self._validate_identity(record) + sequence = record.get("sequence") + if ( + isinstance(sequence, bool) + or not isinstance(sequence, int) + or sequence != self._expected_sequence + ): + raise _ProtocolError("OpenAI stream record sequence is not monotonic") + self._expected_sequence += 1 + record_type = record.get("type") + if record_type == "chunk": + if set(record) != { + "type", + "sequence", + "runtime_id", + "invocation_id", + "request_id", + "chunk", + }: + raise _ProtocolError("OpenAI stream chunk record shape is invalid") + if self._ended: + raise _ProtocolError("OpenAI stream emitted a chunk after end") + chunk = _validate_openai_chunk(record.get("chunk")) + if not self._discard: + await self._queue.put(chunk, len(stripped)) + return + if record_type == "end": + if self._ended or set(record) != { + "type", + "sequence", + "runtime_id", + "invocation_id", + "request_id", + }: + raise _ProtocolError("OpenAI stream end record is invalid") + self._ended = True + await self._queue.put(_END) + return + raise _ProtocolError("OpenAI stream record has an unknown type") + + def _validate_identity(self, record: dict[str, Any]) -> None: + if record.get("runtime_id") != self._runtime_id: + raise _ProtocolError("OpenAI stream runtime ID does not match") + if record.get("request_id") != self._request_id: + raise _ProtocolError("OpenAI stream request ID does not match") + invocation_id = record.get("invocation_id") + if not isinstance(invocation_id, str) or not invocation_id: + raise _ProtocolError("OpenAI stream invocation ID is missing") + if self._invocation_id is None: + self._invocation_id = invocation_id + elif invocation_id != self._invocation_id: + raise _ProtocolError("OpenAI stream invocation ID changed") + + def _set_error(self, message: str) -> None: + if self._error is None: + self._error = FabricRuntimeError( + message, + stage="invoke", + code="openai_stream_protocol_error", + ) + + async def close(self) -> None: + if self._server is not None: + self._server.close() + await self._server.wait_closed() + self._server = None + for writer in tuple(self._writers): + writer.close() + for task in tuple(self._tasks): + task.cancel() + if self._tasks: + await asyncio.gather(*self._tasks, return_exceptions=True) + self._writers.clear() + self._bound_port = None + + +def _validate_openai_chunk(value: Any) -> dict[str, Any]: + if not isinstance(value, dict) or value.get("object") != "chat.completion.chunk": + raise _ProtocolError( + "OpenAI stream event must be a chat.completion.chunk mapping" + ) + identifier = value.get("id") + model = value.get("model") + created = value.get("created") + choices = value.get("choices") + if not isinstance(identifier, str) or not identifier.strip(): + raise _ProtocolError( + "OpenAI stream event id must be a non-empty string containing " + "a non-whitespace character" + ) + if not isinstance(model, str) or not model.strip(): + raise _ProtocolError( + "OpenAI stream event model must be a non-empty string containing " + "a non-whitespace character" + ) + if ( + isinstance(created, bool) + or not isinstance(created, int) + or not 0 <= created <= _UINT64_MAX + ): + raise _ProtocolError( + "OpenAI stream event created must be an unsigned 64-bit integer" + ) + if not isinstance(choices, list): + raise _ProtocolError("OpenAI stream event choices must be a list") + for choice in choices: + if not isinstance(choice, dict): + raise _ProtocolError("OpenAI stream choices must be mappings") + index = choice.get("index") + delta = choice.get("delta") + if ( + isinstance(index, bool) + or not isinstance(index, int) + or not 0 <= index <= _UINT32_MAX + ): + raise _ProtocolError( + "OpenAI stream choice index must be an unsigned 32-bit integer" + ) + if not isinstance(delta, dict): + raise _ProtocolError("OpenAI stream choice delta must be a mapping") + for name in ("content", "refusal", "role"): + if name in delta and delta[name] is not None and not isinstance( + delta[name], str + ): + raise _ProtocolError( + f"OpenAI stream choice delta {name} must be a string or null" + ) + if "function_call" in delta and delta["function_call"] is not None: + if not isinstance(delta["function_call"], dict): + raise _ProtocolError( + "OpenAI stream choice delta function_call must be a mapping or null" + ) + if "tool_calls" in delta and delta["tool_calls"] is not None: + tool_calls = delta["tool_calls"] + if not isinstance(tool_calls, list) or not all( + isinstance(tool_call, dict) for tool_call in tool_calls + ): + raise _ProtocolError( + "OpenAI stream choice delta tool_calls must be a list of mappings or null" + ) + if "finish_reason" in choice and choice["finish_reason"] is not None: + if not isinstance(choice["finish_reason"], str): + raise _ProtocolError( + "OpenAI stream choice finish_reason must be a string or null" + ) + if "logprobs" in choice and choice["logprobs"] is not None: + if not isinstance(choice["logprobs"], dict): + raise _ProtocolError( + "OpenAI stream choice logprobs must be a mapping or null" + ) + if "usage" in value and value["usage"] is not None: + if not isinstance(value["usage"], dict): + raise _ProtocolError("OpenAI stream event usage must be a mapping or null") + return value + + +def _reject_json_constant(value: str) -> None: + raise ValueError(f"non-standard JSON constant {value}") + + +class OpenAIInvokeStream: + """Async iterator of OpenAI chat-completion chunks for one invocation. + + Await ``result()`` for the separate normalized terminal result. Call + ``aclose()`` when iteration stops early; it drains the stream without + cancelling the invocation. + """ + + def __init__( + self, + invoke: Callable[[dict[str, Any]], Awaitable[RunResult]], + *, + runtime_id: str, + request_id: str, + on_result: Callable[[RunResult], None] | None = None, + on_protocol_failure: Callable[[], None] | None = None, + ) -> None: + """lazydocs: ignore""" + + self._listener = _OpenAIStreamListener( + runtime_id=runtime_id, + request_id=request_id, + ) + self._closed = False + self._finalized = False + self._end_observed = False + self._pending_item: dict[str, Any] | object | None = None + self._accepted_result: RunResult | None = None + self._on_result = on_result + self._on_protocol_failure = on_protocol_failure + self._protocol_failure_reported = False + self._finalize_lock = asyncio.Lock() + run = self._run(invoke) + try: + self._task = asyncio.create_task(run) + except BaseException: + run.close() + raise + + async def _run( + self, + invoke: Callable[[dict[str, Any]], Awaitable[RunResult]], + ) -> RunResult: + await self._listener.start() + return await invoke(self._listener.transport) + + def __aiter__(self) -> OpenAIInvokeStream: + return self + + async def __anext__(self) -> dict[str, Any]: + if self._closed: + await self._finalize() + raise StopAsyncIteration + queue = self._listener.records + while True: + if self._end_observed: + await self._finalize() + self._raise_protocol_error() + raise StopAsyncIteration + if self._pending_item is not None: + item = self._pending_item + self._pending_item = None + elif not queue.empty(): + item = queue.get_nowait() + elif self._task.done(): + if self._task.cancelled() or self._task.exception() is not None: + await self._finalize() + raise StopAsyncIteration + getter = asyncio.create_task(queue.get()) + try: + await asyncio.wait( + {getter}, + timeout=_OPENAI_STREAM_COMPLETION_TIMEOUT, + ) + except asyncio.CancelledError: + if not getter.done(): + getter.cancel() + try: + self._pending_item = await getter + except asyncio.CancelledError: + pass + raise + if getter.done() and not getter.cancelled(): + item = getter.result() + else: + getter.cancel() + with suppress(asyncio.CancelledError): + await getter + self._listener.fail( + "OpenAI stream did not establish and complete its event channel" + ) + await self._finalize() + self._raise_protocol_error() + raise StopAsyncIteration from None + else: + getter = asyncio.create_task(queue.get()) + try: + await asyncio.wait( + {getter, self._task}, + return_when=asyncio.FIRST_COMPLETED, + ) + except asyncio.CancelledError: + if not getter.done(): + getter.cancel() + try: + self._pending_item = await getter + except asyncio.CancelledError: + pass + raise + if getter.done() and not getter.cancelled(): + item = getter.result() + else: + getter.cancel() + with suppress(asyncio.CancelledError): + await getter + continue + if item is _END: + self._end_observed = True + await self._finalize() + self._raise_protocol_error() + raise StopAsyncIteration + return item # type: ignore[return-value] + + async def result(self) -> RunResult: + """Drain the stream and return its separate normalized terminal result.""" + + await self._finalize() + try: + result = await asyncio.shield(self._task) + except asyncio.CancelledError: + if self._task.cancelled(): + self._raise_protocol_error() + raise + except Exception: + self._raise_protocol_error() + raise + self._raise_protocol_error() + return self._accepted_result if self._accepted_result is not None else result + + async def aclose(self) -> None: + """Discard unread chunks and wait without cancelling the invocation.""" + + self._closed = True + await self._finalize() + + async def _finalize(self) -> None: + async with self._finalize_lock: + if self._finalized: + return + self._listener.discard() + queue = self._listener.records + while not self._task.done(): + getter = asyncio.create_task(queue.get()) + try: + await asyncio.wait( + {getter, self._task}, + return_when=asyncio.FIRST_COMPLETED, + ) + finally: + if not getter.done(): + getter.cancel() + with suppress(asyncio.CancelledError): + await getter + while not queue.empty(): + queue.get_nowait() + + result: RunResult | None = None + try: + result = await asyncio.shield(self._task) + except asyncio.CancelledError: + if not self._task.cancelled(): + raise + except Exception: + pass + + if result is not None: + try: + await asyncio.wait_for( + self._listener.wait_completed(), + _OPENAI_STREAM_COMPLETION_TIMEOUT, + ) + except TimeoutError: + self._listener.fail( + "OpenAI stream did not establish and complete its event channel" + ) + while not queue.empty(): + queue.get_nowait() + self._validate_and_accept_result(result) + + self._pending_item = None + if self._listener.error is not None: + self._report_protocol_failure() + await self._listener.close() + self._finalized = True + + def _validate_and_accept_result(self, result: RunResult) -> None: + if self._accepted_result is not None: + return + if self._listener.error is None: + invocation_id = self._listener.invocation_id + if invocation_id is None or result.invocation_id != invocation_id: + self._listener._set_error( + "OpenAI stream invocation ID does not match its terminal result" + ) + if self._listener.error is not None: + self._report_protocol_failure() + return + self._accepted_result = result + if self._on_result is not None: + self._on_result(result) + + def _report_protocol_failure(self) -> None: + if self._protocol_failure_reported: + return + self._protocol_failure_reported = True + if self._on_protocol_failure is not None: + self._on_protocol_failure() + + def _raise_protocol_error(self) -> None: + error = self._listener.error + if error is not None: + self._report_protocol_failure() + raise error + + +def _http_headers(lines: list[bytes]) -> dict[str, str]: + headers: dict[str, str] = {} + for line in lines: + name, value = line.decode("iso-8859-1").split(":", 1) + headers[name.strip().lower()] = value.strip() + return headers + + +async def _write_http_response( + writer: asyncio.StreamWriter, + status: int, + reason: str, +) -> None: + writer.write( + f"HTTP/1.1 {status} {reason}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".encode( + "ascii" + ) + ) + await writer.drain() diff --git a/python/src/nemo_fabric/runtime.py b/python/src/nemo_fabric/runtime.py index 958c2bfb..0689a0b2 100644 --- a/python/src/nemo_fabric/runtime.py +++ b/python/src/nemo_fabric/runtime.py @@ -22,6 +22,7 @@ FabricStateError, ) from nemo_fabric.models import RunRequest +from nemo_fabric.openai_streaming import OpenAIInvokeStream from nemo_fabric.streaming import InvokeStream, _AtofStreamListener from nemo_fabric.types import RunPlan, RunResult, RuntimeHandle @@ -72,7 +73,7 @@ def __init__( self._invocations: list[dict[str, Any]] = [] self._status = RuntimeStatus.ACTIVE self._current_task: asyncio.Task[Any] | None = None - self._current_stream: InvokeStream | None = None + self._current_stream: InvokeStream | OpenAIInvokeStream | None = None self._stream_listener = stream_listener self._closing = False @@ -112,6 +113,25 @@ def supports_streaming(self) -> bool: return self._stream_listener is not None + @property + def supports_openai_streaming(self) -> bool: + """Return whether the selected adapter implements native OpenAI streaming.""" + + adapter_descriptor = self._plan.get("adapter_descriptor") + descriptor = ( + adapter_descriptor.get("descriptor") + if isinstance(adapter_descriptor, Mapping) + else None + ) + descriptor_capabilities = ( + descriptor.get("capabilities") if isinstance(descriptor, Mapping) else None + ) + return ( + self._plan.capabilities.streaming + and isinstance(descriptor_capabilities, Mapping) + and descriptor_capabilities.get("streaming") is True + ) + async def invoke( self, *, @@ -153,6 +173,9 @@ async def invoke( async def _invoke_payload( self, payload: dict[str, Any], + *, + openai_stream_transport: Mapping[str, Any] | None = None, + absorb_result: bool = True, ) -> RunResult: self._ensure_invocable() self._current_task = asyncio.current_task() @@ -168,19 +191,29 @@ async def _invoke_payload( def invoke() -> dict[str, Any]: nonlocal native_result - native_result = json.loads( - native.invoke_runtime( - json.dumps(self._plan.to_mapping()), - json.dumps(self._runtime.to_mapping()), - json.dumps(payload), + plan_json = json.dumps(self._plan.to_mapping()) + runtime_json = json.dumps(self._runtime.to_mapping()) + request_json = json.dumps(payload) + if openai_stream_transport is None: + encoded = native.invoke_runtime( + plan_json, + runtime_json, + request_json, ) - ) + else: + encoded = native.invoke_openai_stream( + plan_json, + runtime_json, + request_json, + json.dumps(dict(openai_stream_transport)), + ) + native_result = json.loads(encoded) return native_result result = await _call_blocking(invoke) typed_result = RunResult.from_mapping(result) except asyncio.CancelledError: - if native_result is not None: + if absorb_result and native_result is not None: try: self._absorb(RunResult.from_mapping(native_result)) except Exception: @@ -214,7 +247,8 @@ def stop_after_cancel() -> Any: except Exception as error: self._status = RuntimeStatus.FAILED raise FabricRuntimeError(str(error), stage="invoke") from error - self._absorb(typed_result) + if absorb_result: + self._absorb(typed_result) return typed_result except FabricError: raise @@ -232,8 +266,8 @@ def invoke_stream( """Start one turn and stream raw NeMo Relay ATOF records as they arrive. ``input`` and ``request`` are mutually exclusive. The returned - :class:`InvokeStream` yields raw ATOF dictionaries. Await - ``stream.result()`` for the terminal normalized :class:`RunResult`. + ``InvokeStream`` yields raw ATOF dictionaries. Await + ``stream.result()`` for the terminal normalized ``RunResult``. Raises: FabricCapabilityError: If the runtime was not started with NeMo Relay @@ -267,6 +301,56 @@ def invoke_stream( self._current_stream = stream return stream + def invoke_openai_stream( + self, + *, + input: Any = None, + request: RunRequest | None = None, + ) -> OpenAIInvokeStream: + """Start one turn and stream native OpenAI chat-completion chunks. + + The returned stream yields ``chat.completion.chunk`` mappings. Await + ``stream.result()`` for the separate normalized terminal result. + + Raises: + FabricCapabilityError: If the selected adapter does not advertise + native OpenAI streaming. + FabricConfigError: If request fields conflict or are not + JSON-compatible. + FabricStateError: If another turn or stream is active. + """ + + if not self.supports_openai_streaming: + raise FabricCapabilityError( + "the selected adapter does not support native OpenAI streaming", + stage="invoke", + code="openai_streaming_unavailable", + details={"capability": "streaming"}, + ) + if self._current_stream is not None and not self._current_stream._finalized: + raise FabricStateError( + "a streaming invocation is active; fully consume it or call " + "`await stream.aclose()` before starting another turn" + ) + self._ensure_invocable() + payload = _run_request_payload(input=input, request=request) + stream = OpenAIInvokeStream( + lambda transport: self._invoke_payload( + payload, + openai_stream_transport=transport, + absorb_result=False, + ), + runtime_id=self.runtime_id, + request_id=payload["request_id"], + on_result=self._absorb, + on_protocol_failure=self._mark_failed, + ) + self._current_stream = stream + return stream + + def _mark_failed(self) -> None: + self._status = RuntimeStatus.FAILED + def _ensure_invocable(self) -> None: if self._status is not RuntimeStatus.ACTIVE: raise FabricStateError(f"cannot invoke a {self._status.value} runtime") @@ -295,8 +379,8 @@ async def stop(self) -> None: if self._current_stream is not None and not self._current_stream._finalized: if not self._current_stream._task.done(): raise FabricStateError( - "cannot stop while a streaming invocation is active; await " - "`stream.result()` and then call `await stream.aclose()`" + "cannot stop while a streaming invocation is active; consume " + "the stream or await `stream.result()` or `stream.aclose()`" ) await self._current_stream.aclose() if self._current_task is not None: diff --git a/python/src/nemo_fabric/streaming.py b/python/src/nemo_fabric/streaming.py index 0999d18f..b20aa28d 100644 --- a/python/src/nemo_fabric/streaming.py +++ b/python/src/nemo_fabric/streaming.py @@ -104,8 +104,8 @@ def _release(self, size: int) -> None: class InvokeStream: """Async iterator of raw ATOF records for one runtime invocation. - Consume the final normalized result separately with :meth:`result`. If - iteration stops early, call :meth:`aclose` before starting another turn. + Consume the final normalized result separately with ``result()``. If + iteration stops early, call ``aclose()`` before starting another turn. """ def __init__( diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 5f954dcd..c3e2c37c 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -27,13 +27,15 @@ schemas/ │ ├── agent-run-result.schema.json │ ├── runtime-context.schema.json │ └── legacy/ -│ └── adapter-invocation.schema.json +│ ├── adapter-invocation.schema.json +│ ├── openai-stream-invocation.schema.json +│ └── openai-stream-record.schema.json └── *.schema.json # Northbound and Fabric-runtime contracts ``` An adapter author can treat `adapter-contract/` as the complete schema entry -point. The `legacy/` subdirectory contains only the transitional local-host -payload used while first-party adapters migrate to the typed execution types. +point. The `legacy/` subdirectory contains transitional local-host payloads +used while first-party adapters migrate to typed execution types. `FabricConfig` is the northbound source of consumer intent. Planning produces the `CapabilityPlan` as routed evidence and projects the fields accepted by the @@ -70,6 +72,14 @@ descriptor schema. an initialized persistent local adapter host. It contains `runtime_context` and the northbound `run-request`. It will be removed after adapters consume the typed southbound request directly. +- `adapter-contract/legacy/openai-stream-invocation`: current native OpenAI + stream payload sent to an initialized persistent local adapter host. It adds + a Fabric-owned authenticated loopback stream sink to the per-turn runtime + context and request. The common host validates and removes the sink before + calling `invoke_openai_stream(payload, emit)`. +- `adapter-contract/legacy/openai-stream-record`: correlated chunk and explicit + end records carried as chunked NDJSON. The chunk variant freezes the + `openai.chat_completions.chunk/v1` profile accepted by the SDK listener. ## Fabric Consumer and Runtime Contracts diff --git a/schemas/adapter-contract/legacy/openai-stream-invocation.schema.json b/schemas/adapter-contract/legacy/openai-stream-invocation.schema.json new file mode 100644 index 00000000..1728c553 --- /dev/null +++ b/schemas/adapter-contract/legacy/openai-stream-invocation.schema.json @@ -0,0 +1,368 @@ +{ + "$defs": { + "ArtifactManifest": { + "additionalProperties": false, + "description": "Manifest of run artifacts.", + "properties": { + "artifacts": { + "description": "Artifact entries.", + "items": { + "$ref": "#/$defs/ArtifactRef" + }, + "type": "array" + }, + "root": { + "description": "Artifact root directory.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, + "ArtifactRef": { + "additionalProperties": false, + "description": "Reference to one artifact.", + "properties": { + "kind": { + "description": "Artifact kind.", + "type": "string" + }, + "media_type": { + "description": "Optional media type.", + "type": [ + "string", + "null" + ] + }, + "metadata": { + "additionalProperties": true, + "description": "Artifact-specific metadata preserved across the Rust and Python SDK boundary.", + "type": "object" + }, + "name": { + "description": "Logical artifact name.", + "type": "string" + }, + "path": { + "description": "Artifact path.", + "type": "string" + } + }, + "required": [ + "name", + "kind", + "path" + ], + "type": "object" + }, + "ControlLocation": { + "description": "Where NeMo Fabric control code runs relative to the environment.", + "oneOf": [ + { + "const": "external_control", + "description": "NeMo Fabric runs on the host/control plane and starts or connects to the harness in the environment.", + "type": "string" + }, + { + "const": "in_env_control", + "description": "NeMo Fabric runs inside the prepared environment with the harness.", + "type": "string" + } + ] + }, + "EnvironmentHandle": { + "additionalProperties": false, + "description": "Resolved execution environment context.", + "properties": { + "artifacts": { + "description": "Artifact root visible to the harness runtime.", + "type": [ + "string", + "null" + ] + }, + "connection": { + "additionalProperties": true, + "description": "Provider connection metadata.", + "type": "object" + }, + "control_location": { + "$ref": "#/$defs/ControlLocation", + "description": "Where NeMo Fabric control code runs." + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "description": "Environment variables visible to the harness and its tools.", + "type": "object" + }, + "environment_id": { + "description": "Environment handle id.", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "description": "Provider-specific metadata.", + "type": "object" + }, + "ownership": { + "$ref": "#/$defs/EnvironmentOwnership", + "description": "Whether NeMo Fabric owns the environment resource." + }, + "provider": { + "description": "Environment provider.", + "type": "string" + }, + "workspace": { + "description": "Workspace visible to the harness runtime.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "environment_id", + "provider", + "control_location", + "ownership" + ], + "type": "object" + }, + "EnvironmentOwnership": { + "description": "Whether NeMo Fabric owns the underlying environment resource.", + "oneOf": [ + { + "const": "caller_owned", + "description": "The caller or a surrounding system owns the environment resource.", + "type": "string" + }, + { + "const": "fabric_owned", + "description": "NeMo Fabric created or leased the environment resource and may release it.", + "type": "string" + } + ] + }, + "OpenAiStreamHost": { + "description": "Supported native-streaming listener host.", + "oneOf": [ + { + "const": "127.0.0.1", + "description": "SDK-owned IPv4 loopback listener.", + "type": "string" + } + ] + }, + "OpenAiStreamProfile": { + "description": "Supported OpenAI-compatible chunk profile.", + "oneOf": [ + { + "const": "openai.chat_completions.chunk/v1", + "description": "OpenAI Chat Completions chunk objects.", + "type": "string" + } + ] + }, + "OpenAiStreamProtocolVersion": { + "description": "Supported southbound native-streaming protocol version.", + "oneOf": [ + { + "const": "fabric.openai_stream/v1alpha1", + "description": "Initial authenticated loopback HTTP and chunked-NDJSON protocol.", + "type": "string" + } + ] + }, + "OpenAiStreamSink": { + "additionalProperties": false, + "description": "Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric.", + "properties": { + "host": { + "$ref": "#/$defs/OpenAiStreamHost", + "description": "Loopback host owned by the SDK listener." + }, + "invocation_id": { + "description": "Invocation id for stream correlation.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "port": { + "description": "Loopback TCP port owned by the SDK listener.", + "format": "uint16", + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "profile": { + "$ref": "#/$defs/OpenAiStreamProfile", + "description": "OpenAI event profile emitted on this stream." + }, + "protocol_version": { + "$ref": "#/$defs/OpenAiStreamProtocolVersion", + "description": "Southbound stream protocol version." + }, + "request_id": { + "description": "Request id for stream correlation.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "runtime_id": { + "description": "Runtime id for stream correlation.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "token": { + "description": "Single-use bearer token. Adapters must not log or persist this value.", + "minLength": 1, + "pattern": "^[^\\r\\n]*\\S[^\\r\\n]*$", + "type": "string" + } + }, + "required": [ + "protocol_version", + "profile", + "host", + "port", + "token", + "runtime_id", + "invocation_id", + "request_id" + ], + "type": "object" + }, + "RunRequest": { + "description": "A request passed to a NeMo Fabric-managed harness runtime.", + "properties": { + "context": { + "additionalProperties": true, + "description": "Runtime context such as task, rollout, workflow, or caller metadata.", + "type": "object" + }, + "input": { + "default": null, + "description": "Request payload for the harness." + }, + "overrides": { + "description": "Per-invocation overrides allowed by the resolved config." + }, + "request_id": { + "description": "Request id.", + "type": "string" + } + }, + "required": [ + "request_id" + ], + "type": "object" + }, + "RuntimeContext": { + "additionalProperties": false, + "description": "Context generated for one invocation of a started runtime.", + "properties": { + "artifacts": { + "$ref": "#/$defs/ArtifactManifest", + "description": "Artifact manifest visible to the adapter at invocation start." + }, + "environment": { + "$ref": "#/$defs/EnvironmentHandle", + "description": "Prepared execution environment." + }, + "invocation_id": { + "description": "Invocation handle id.", + "type": "string" + }, + "request_id": { + "description": "Request id.", + "type": "string" + }, + "runtime_id": { + "description": "Runtime handle id.", + "type": "string" + }, + "telemetry": { + "anyOf": [ + { + "$ref": "#/$defs/RuntimeTelemetryContext" + }, + { + "type": "null" + } + ], + "description": "Runtime telemetry context generated for this invocation." + } + }, + "required": [ + "runtime_id", + "invocation_id", + "request_id", + "environment", + "artifacts" + ], + "type": "object" + }, + "RuntimeTelemetryContext": { + "additionalProperties": false, + "description": "Runtime telemetry config passed to adapters.", + "properties": { + "config_path": { + "description": "Generated Relay config path for this invocation.", + "type": [ + "string", + "null" + ] + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "description": "Environment variables NeMo Fabric applies while invoking the adapter.", + "type": "object" + }, + "metadata": { + "additionalProperties": true, + "description": "Additional telemetry metadata surfaced to consumers and adapters.", + "type": "object" + }, + "relay_enabled": { + "description": "Whether Relay is enabled for this invocation.", + "type": "boolean" + } + }, + "required": [ + "relay_enabled" + ], + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "One adapter-native OpenAI streaming invocation.", + "properties": { + "request": { + "$ref": "#/$defs/RunRequest", + "description": "Typed caller request for this invocation." + }, + "runtime_context": { + "$ref": "#/$defs/RuntimeContext", + "description": "Invocation context generated by NeMo Fabric." + }, + "stream": { + "$ref": "#/$defs/OpenAiStreamSink", + "description": "Authenticated progressive-output sink for this invocation." + } + }, + "required": [ + "runtime_context", + "request", + "stream" + ], + "title": "OpenAiStreamInvocation", + "type": "object" +} \ No newline at end of file diff --git a/schemas/adapter-contract/legacy/openai-stream-record.schema.json b/schemas/adapter-contract/legacy/openai-stream-record.schema.json new file mode 100644 index 00000000..c74eecc2 --- /dev/null +++ b/schemas/adapter-contract/legacy/openai-stream-record.schema.json @@ -0,0 +1,246 @@ +{ + "$defs": { + "OpenAiChatCompletionChunk": { + "additionalProperties": true, + "description": "OpenAI Chat Completions chunk accepted by the native streaming profile.", + "properties": { + "choices": { + "description": "Incremental choices; an explicit usage-only chunk can contain none.", + "items": { + "$ref": "#/$defs/OpenAiChatCompletionChunkChoice" + }, + "type": "array" + }, + "created": { + "description": "Unix timestamp in seconds.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, + "id": { + "description": "Provider-generated response identifier.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "model": { + "description": "Model identifier.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "object": { + "$ref": "#/$defs/OpenAiChatCompletionChunkObject", + "description": "Exact OpenAI streaming object discriminator." + }, + "usage": { + "additionalProperties": true, + "description": "Optional token-usage data.", + "type": [ + "object", + "null" + ] + } + }, + "required": [ + "id", + "object", + "created", + "model", + "choices" + ], + "type": "object" + }, + "OpenAiChatCompletionChunkChoice": { + "additionalProperties": true, + "description": "One choice within an OpenAI Chat Completions streaming chunk.", + "properties": { + "delta": { + "$ref": "#/$defs/OpenAiChatCompletionChunkDelta", + "description": "Incremental assistant message fields." + }, + "finish_reason": { + "description": "Terminal reason when this choice finishes.", + "type": [ + "string", + "null" + ] + }, + "index": { + "description": "Choice index within the response.", + "format": "uint32", + "maximum": 4294967295, + "minimum": 0, + "type": "integer" + }, + "logprobs": { + "additionalProperties": true, + "description": "Incremental log-probability information.", + "type": [ + "object", + "null" + ] + } + }, + "required": [ + "index", + "delta" + ], + "type": "object" + }, + "OpenAiChatCompletionChunkDelta": { + "additionalProperties": true, + "description": "Incremental assistant message fields carried by one OpenAI choice.", + "properties": { + "content": { + "description": "Incremental text content.", + "type": [ + "string", + "null" + ] + }, + "function_call": { + "additionalProperties": true, + "description": "Legacy incremental function-call fields.", + "type": [ + "object", + "null" + ] + }, + "refusal": { + "description": "Incremental refusal content.", + "type": [ + "string", + "null" + ] + }, + "role": { + "description": "Incremental message role.", + "type": [ + "string", + "null" + ] + }, + "tool_calls": { + "description": "Incremental tool-call fields.", + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" + }, + "OpenAiChatCompletionChunkObject": { + "description": "Exact OpenAI object discriminator accepted by the v1 chunk profile.", + "oneOf": [ + { + "const": "chat.completion.chunk", + "description": "OpenAI Chat Completions streaming chunk.", + "type": "string" + } + ] + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "One correlated NDJSON record on the adapter-native stream channel.", + "oneOf": [ + { + "additionalProperties": false, + "description": "One OpenAI Chat Completions chunk.", + "properties": { + "chunk": { + "$ref": "#/$defs/OpenAiChatCompletionChunk", + "description": "OpenAI-compatible chunk passed through to the consumer." + }, + "invocation_id": { + "description": "Invocation id for stream correlation.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "request_id": { + "description": "Request id for stream correlation.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "runtime_id": { + "description": "Runtime id for stream correlation.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "sequence": { + "description": "Monotonic zero-based record sequence.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, + "type": { + "const": "chunk", + "type": "string" + } + }, + "required": [ + "type", + "sequence", + "runtime_id", + "invocation_id", + "request_id", + "chunk" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Explicit successful end of the progressive event channel.", + "properties": { + "invocation_id": { + "description": "Invocation id for stream correlation.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "request_id": { + "description": "Request id for stream correlation.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "runtime_id": { + "description": "Runtime id for stream correlation.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "sequence": { + "description": "Monotonic zero-based record sequence.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, + "type": { + "const": "end", + "type": "string" + } + }, + "required": [ + "type", + "sequence", + "runtime_id", + "invocation_id", + "request_id" + ], + "type": "object" + } + ], + "title": "OpenAiStreamRecord" +} \ No newline at end of file diff --git a/scripts/docs/enhance_python_api_reference.py b/scripts/docs/enhance_python_api_reference.py index 87563660..daaf988b 100644 --- a/scripts/docs/enhance_python_api_reference.py +++ b/scripts/docs/enhance_python_api_reference.py @@ -19,6 +19,7 @@ "nemo_fabric.client", "nemo_fabric.runtime", "nemo_fabric.streaming", + "nemo_fabric.openai_streaming", "nemo_fabric.models", "nemo_fabric.types", "nemo_fabric.errors", diff --git a/scripts/docs/generate_rust_library_reference.py b/scripts/docs/generate_rust_library_reference.py index 15319925..75117f22 100644 --- a/scripts/docs/generate_rust_library_reference.py +++ b/scripts/docs/generate_rust_library_reference.py @@ -396,7 +396,9 @@ def _block_markdown(node: PageElement, page: Page, pages_by_html: dict[Path, Pag if node.name == "span" and "section-header" in classes: text = _plain_code(node) - return f"### {_inline_code(text)}\n\n" if text else "" + parent_classes = _tag_classes(node.parent) if isinstance(node.parent, Tag) else set() + level = "#####" if "sub-variant-field" in parent_classes else "###" + return f"{level} {_inline_code(text)}\n\n" if text else "" if node.name == "p": text = re.sub(r"\s+", " ", _inline_markdown(node, page, pages_by_html)).strip() @@ -509,6 +511,7 @@ def _page_title(soup: BeautifulSoup, page: Page) -> str: title, ) title = re.sub(r"\s+", " ", title) + title = re.sub(r"\bOpen Ai(?=[A-Z])", "OpenAI ", title) if title.startswith("Crate "): return page.crate_name return title diff --git a/scripts/generate_api_docs.sh b/scripts/generate_api_docs.sh index 305cd809..daf2e7ac 100755 --- a/scripts/generate_api_docs.sh +++ b/scripts/generate_api_docs.sh @@ -24,6 +24,7 @@ PYTHONPATH="python/src" lazydocs \ "nemo_fabric.client" \ "nemo_fabric.runtime" \ "nemo_fabric.streaming" \ + "nemo_fabric.openai_streaming" \ "nemo_fabric.models" \ "nemo_fabric.types" \ "nemo_fabric.errors" @@ -45,7 +46,8 @@ perl -0pi -e 's/(^#{1,2} (?:module|class)<\/kbd> `[^`]+`\n)(?!\n)/$1\n/gm' "$out"/*.md # lazydocs omits the async marker from generated method signatures. perl -0pi -e 's/(### method<\/kbd> `(aclose|result)`\n\n```python\n)\2\(/${1}async def ${2}(/g' \ - "$out/nemo_fabric.streaming.md" + "$out/nemo_fabric.streaming.md" \ + "$out/nemo_fabric.openai_streaming.md" # Restore signature semantics that lazydocs drops and add field-level contracts # for the SDK's Pydantic and immutable mapping models. @@ -85,9 +87,14 @@ add_frontmatter \ "/reference/api/python-library-reference/runtime" add_frontmatter \ "$out/nemo_fabric.streaming.md" \ - "Streaming" \ + "Relay Streaming" \ "Consume raw NVIDIA NeMo Relay ATOF records and terminal invocation results." \ "/reference/api/python-library-reference/streaming" +add_frontmatter \ + "$out/nemo_fabric.openai_streaming.md" \ + "OpenAI Streaming" \ + "Consume adapter-native OpenAI Chat Completions chunks and terminal invocation results." \ + "/reference/api/python-library-reference/openai-streaming" add_frontmatter \ "$out/nemo_fabric.models.md" \ "Models" \ diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 394401e5..9aa59e2c 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -55,8 +55,10 @@ translation: `tool_definition_schema`, and `extension_schemas` where applicable. - Declare runtime requirements and telemetry outputs without secret values. - Leave optional capability flags false unless the installed NeMo Fabric runtime - exposes and tests that adapter operation. Relay-backed ATOF streaming does - not require adapter-native streaming. + exposes and tests that adapter operation. Set `capabilities.streaming` only + when the adapter implements native OpenAI Chat Completions streaming through + `invoke_openai_stream`. Relay-backed ATOF streaming is independent and does + not require this capability. Validate descriptor schemas without importing adapter code. Keep all schema references local to the descriptor document; do not rely on HTTP or file @@ -107,10 +109,23 @@ one `stop` for each NeMo Fabric runtime. - Translate one request and one terminal outcome in `invoke`. - Make `stop` safe after partial startup and failed invocation. - Isolate mutable state between independent runtimes. +- If the descriptor declares `capabilities.streaming`, implement + `async invoke_openai_stream(payload, emit)`. Execute the target exactly once, + await `emit(chunk)` only for the `openai.chat_completions.chunk/v1` profile, + and return one JSON-compatible terminal outcome. Each chunk requires + non-empty `id` and `model`, a nonnegative integer `created`, the exact + `chat.completion.chunk` discriminator, and structurally valid `choices`. An + invocation that emits no chunks is valid. - Do not add an adapter streaming method for Relay-backed `Runtime.invoke_stream()`; execute ordinary `invoke` and use the provided telemetry context. +For native OpenAI streaming, the SDK owns the authenticated loopback HTTP +transport with chunked NDJSON framing. The common host validates the transport, +removes its credentials from the adapter payload, and supplies the `emit` +callback. Do not persist or log stream credentials, write chunks to stdout, add +SSE framing, or forward other target-native event profiles. + For a Python adapter that opts into the common host: ```python @@ -126,6 +141,11 @@ class TargetRuntime: async def invoke(self, payload): ... + async def invoke_openai_stream(self, payload, emit): + async for chunk in self.target.stream(payload["request"]): + await emit(chunk) + return self.target.terminal_result() + async def stop(self): ... @@ -164,8 +184,13 @@ Complete these checks before handing off an adapter: 4. Run `doctor(...)` with both missing and satisfied requirements. 5. Test start, success, target failure, malformed output, repeated invocation, stop, partial-start cleanup, EOF cleanup, and two-runtime isolation. -6. Test Relay correlation if telemetry support is claimed. -7. Report the adapter package version, contract version, required-profile +6. If native OpenAI streaming is claimed, test empty and multi-chunk streams, + malformed and oversized records, invalid chunks, sequence and identity + mismatches, a missing end record, early consumer close without cancellation, + a separate terminal result, one active turn, and exactly one target + invocation. +7. Test Relay correlation separately if telemetry support is claimed. +8. Report the adapter package version, contract version, required-profile result, and every optional capability as supported or unsupported. Do not claim automated NeMo Fabric conformance until the published conformance diff --git a/skills/nemo-fabric-integrate/SKILL.md b/skills/nemo-fabric-integrate/SKILL.md index d8123b75..3fa95403 100644 --- a/skills/nemo-fabric-integrate/SKILL.md +++ b/skills/nemo-fabric-integrate/SKILL.md @@ -172,6 +172,16 @@ Pick the smallest lifecycle the consumer needs: (`stop()` can raise `FabricRuntimeError`; see Consume Results And Handle Errors). A runtime accepts one active invocation at a time; overlapping calls raise `FabricStateError`. +- **Native OpenAI stream** — adapter-native OpenAI Chat Completions chunks plus + a separate terminal normalized result. Check + `runtime.supports_openai_streaming`, call + `runtime.invoke_openai_stream(...)`, iterate the returned + `OpenAIInvokeStream`, and then await `stream.result()`. The selected adapter + descriptor must declare `capabilities.streaming`. Each yielded mapping has + `object == "chat.completion.chunk"`; an empty stream is valid. If iteration + stops early, call `await stream.aclose()` to drain without cancelling the + target invocation. This path does not require NeMo Relay or + `streaming=True`. - **NVIDIA NeMo Relay stream** — live, raw ATOF records plus a terminal normalized result. Enable NeMo Relay, pass `streaming=True` to `start_runtime(...)`, call `runtime.invoke_stream(...)`, iterate the returned `InvokeStream`, and then @@ -180,9 +190,8 @@ Pick the smallest lifecycle the consumer needs: failures remain normalized `RunResult` values. If iteration stops early, call `await stream.aclose()` before starting another turn. `aclose()` waits for the turn to finish; it does not cancel the harness invocation. The SDK - intentionally exposes only ATOF records generated by NeMo Relay. A future - normalized NeMo Fabric contract will address adapter-native progressive - output. The listener + intentionally exposes only ATOF records generated by NeMo Relay. This path is + independent of native OpenAI streaming. The listener limits each record to 1 MiB and its queue to 1,024 records or 16 MiB of encoded data. It correlates records through the NeMo Fabric request ID for in-process harnesses. For gateway harnesses, it uses the NeMo Relay turn-scope @@ -210,9 +219,9 @@ local execution mechanism in `FabricConfig`. Do not replay an invocation after a runtime failure. Stop the failed runtime and explicitly start a new one according to the application's retry policy. -The lifecycle fragment below shows both forms. It assumes the caller has already -set `config = to_fabric_config(job)` and chosen `base`, as described in the -configuration example above: +The lifecycle fragment below shows the available forms. It assumes the caller +has already set `config = to_fabric_config(job)` and chosen `base`, as described +in the configuration example above: ```python import asyncio @@ -231,6 +240,14 @@ async def main() -> None: first = await runtime.invoke(input="Inspect the repository") second = await runtime.invoke(input="Now review the latest patch") + # Adapter-native OpenAI Chat Completions chunks + async with await fabric.start_runtime(config, base_dir=base) as runtime: + if runtime.supports_openai_streaming: + stream = runtime.invoke_openai_stream(input="Review the latest patch") + async for chunk in stream: + print(chunk) + openai_streamed_result = await stream.result() + # NeMo Relay streaming streaming_config = config.model_copy(deep=True).enable_relay() async with await fabric.start_runtime( @@ -255,6 +272,14 @@ only to carry one invocation's ATOF records. Treat `stream.result()` as authoritative, and reconstruct nested work from ATOF `uuid` and `parent_uuid` fields rather than stream order. +For native OpenAI streaming, the SDK owns the authenticated loopback HTTP +transport, chunked NDJSON framing, and correlation values. Consumer code +supplies no listener or credentials. The adapter executes exactly one +invocation, and the terminal `RunResult` remains separate from the chunk stream. +Fully consume the stream or call `await stream.aclose()` before starting another +turn. Awaiting `stream.result()` also drains and discards unread native OpenAI +chunks, so consume the iterator first when the application needs every chunk. + ## Validate Before Running Resolve and diagnose before spending work on a runtime, especially in a new @@ -337,7 +362,10 @@ result-field and error inventory, and - [ ] The consumer config object is translated directly into an in-memory `FabricConfig`. - [ ] Only public `nemo_fabric` symbols are imported; no `_native` or adapter internals. - [ ] The consumer config is built in memory and passed directly to NeMo Fabric. -- [ ] The right lifecycle is chosen: `run(...)` for a single invocation, `start_runtime(...)` with `async with` for multi-turn, or `invoke_stream(...)` for raw NeMo Relay ATOF. +- [ ] The right lifecycle is chosen: `run(...)` for a single invocation, + `start_runtime(...)` with `async with` for multi-turn, + `invoke_openai_stream(...)` for descriptor-gated OpenAI chunks, or + `invoke_stream(...)` for raw NeMo Relay ATOF. - [ ] `plan(...)` and `doctor(...)` validate adapter selection, capabilities, and environment before execution. - [ ] Installation, adapter dependencies, and credentials are owned by the environment, not consumer code. - [ ] `RunResult` status, error, and events are inspected before output; artifacts and telemetry are captured. @@ -356,7 +384,8 @@ Link to these canonical sources instead of duplicating them: stubs are authoritative for exact signatures, fields, and defaults): [client](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.client.md), [runtime](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.runtime.md), - [streaming](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.streaming.md), + [native OpenAI streaming](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.openai_streaming.md), + [Relay streaming](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.streaming.md), [models](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.models.md), [types](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.types.md), [errors](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.errors.md) diff --git a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md index 53aa6ef8..29861af5 100644 --- a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md +++ b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md @@ -8,9 +8,10 @@ SPDX-License-Identifier: Apache-2.0 `Fabric()` is the primary entrypoint. It is a plain, reusable object — not a lifecycle context manager — and can plan, diagnose, or start multiple independent runtimes. The generated -[client reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.client.md) -and [runtime reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.runtime.md) -and [streaming reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.streaming.md) +[client reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.client.md), +[runtime reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.runtime.md), +[native OpenAI streaming reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.openai_streaming.md), +and [Relay streaming reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/reference/api/python-library-reference/nemo_fabric.streaming.md) document the public methods with their `async` and keyword-only markers. The installed `nemo_fabric` package also ships type information (`py.typed`) for static analysis. @@ -24,7 +25,7 @@ The following table lists the `Fabric` methods and when to use each: | `plan(config, *, base_dir=...)` | No | You need the selected adapter, capability routing, and runtime capabilities before running. | `RunPlan` | | `doctor(config, *, base_dir=...)` | Yes | You need preflight diagnostics for adapter resolution, capability routing, declared requirements, and environment assumptions. | `DoctorReport` | | `run(config, *, base_dir=..., input=... \| request=...)` | Yes | You need one complete start, invoke, result, and stop cycle. | `RunResult` | -| `start_runtime(config, *, base_dir=..., overrides=..., streaming=False)` | Yes | You need state across multiple ordered invocations. Pass `streaming=True` with NVIDIA NeMo Relay enabled to provision `invoke_stream(...)`. | `Runtime` | +| `start_runtime(config, *, base_dir=..., overrides=..., streaming=False)` | Yes | You need state across multiple ordered invocations. Pass `streaming=True` with NVIDIA NeMo Relay enabled only to provision `invoke_stream(...)`. | `Runtime` | `input` and `request` on `run(...)` are mutually exclusive. Use `input=...` for the common case; use `request=RunRequest(...)` when the invocation needs a @@ -37,9 +38,11 @@ The following table lists the `Runtime` members for driving a stateful runtime. | Member | Async | Notes | | --- | --- | --- | | `invoke(*, input=... \| request=...)` | Yes | One turn on an active runtime. One active invocation at a time; overlap raises `FabricStateError`. | +| `invoke_openai_stream(*, input=... \| request=...)` | No | Start exactly one descriptor-gated native invocation and return an async `OpenAIInvokeStream` of `chat.completion.chunk` mappings. Await `stream.result()` for the separate terminal `RunResult`. | | `invoke_stream(*, input=... \| request=...)` | No | Start one NeMo Relay turn and return an async `InvokeStream` of raw ATOF records. Await `stream.result()` for the terminal `RunResult`. | | `stop()` | Yes | Stop the runtime. Called automatically by `async with`. | | `status` | No | `RuntimeStatus`: `ACTIVE`, `STOPPED`, or `FAILED`. | +| `supports_openai_streaming` | No | `True` when the selected descriptor declares `capabilities.streaming` for native OpenAI Chat Completions chunks. | | `supports_streaming` | No | `True` when NeMo Relay ATOF streaming was enabled at runtime startup. | | `runtime_id` | No | Opaque identifier for this runtime lifecycle. | | `messages` / `invocations` | No | Copied harness history and per-turn IDs. | @@ -53,6 +56,20 @@ async with await fabric.start_runtime(config, base_dir=base) as runtime: result = await runtime.invoke(input="…") ``` +## Stream Handles + +The two stream handles represent independent contracts: + +| Class | Async Iteration | Terminal Result | Early Close | +| --- | --- | --- | --- | +| `OpenAIInvokeStream` | Adapter-native `chat.completion.chunk` mappings; an empty stream is valid. | `await result()` drains and discards unread chunks, then returns the separate authoritative `RunResult`. | `await aclose()` drains and discards unread chunks without cancelling the invocation. | +| `InvokeStream` | Raw invocation-correlated ATOF records generated by NeMo Relay. | `await result()` returns the separate authoritative `RunResult`. | `await aclose()` drains and discards unread records without cancelling the invocation. | + +Fully consume either stream or call `await stream.aclose()` before starting +another turn. For native OpenAI streaming, `result()` also finalizes and +discards unread chunks. Relay `InvokeStream.result()` does not consume unread +ATOF records. + ## Execution Model NVIDIA NeMo Fabric separates configuration, planning, runtime lifecycle, and @@ -60,6 +77,7 @@ individual invocations: ```text FabricConfig -> plan() -> RunPlan -> start_runtime() -> Runtime -> invoke() -> RunResult + \-> invoke_openai_stream() -> OpenAIInvokeStream \-> invoke_stream() -> InvokeStream ``` diff --git a/tests/adapters/test_adapters_common_lifecycle.py b/tests/adapters/test_adapters_common_lifecycle.py index 43b3b232..7bec8cc1 100644 --- a/tests/adapters/test_adapters_common_lifecycle.py +++ b/tests/adapters/test_adapters_common_lifecycle.py @@ -12,6 +12,7 @@ import pytest from nemo_fabric_adapters.common import lifecycle from nemo_fabric_adapter_contract.models import AgentConfig +from nemo_fabric.openai_streaming import _END, _OpenAIStreamListener def _request(operation: str, payload: dict[str, Any]) -> dict[str, Any]: @@ -26,6 +27,428 @@ def _streams(requests: list[dict[str, Any]]) -> tuple[io.StringIO, io.StringIO]: return input_stream, io.StringIO() +class _BackpressuredStreamWriter: + def __init__(self) -> None: + self.parts: list[bytes] = [] + self.drain_calls = 0 + self.first_drain_started = asyncio.Event() + self.release_first_drain = asyncio.Event() + self.closed = False + + def write(self, data: bytes) -> None: + self.parts.append(data) + + async def drain(self) -> None: + self.drain_calls += 1 + if self.drain_calls == 1: + self.first_drain_started.set() + await self.release_first_drain.wait() + + def close(self) -> None: + self.closed = True + + async def wait_closed(self) -> None: + pass + + +def _openai_stream_payload( + listener: _OpenAIStreamListener, + *, + runtime_id: str = "runtime-1", + invocation_id: str = "invocation-1", + request_id: str = "request-1", +) -> dict[str, Any]: + return { + "runtime_context": { + "runtime_id": runtime_id, + "invocation_id": invocation_id, + "request_id": request_id, + }, + "request": {"request_id": request_id, "input": "hello"}, + "stream": { + "protocol_version": "fabric.openai_stream/v1alpha1", + "profile": "openai.chat_completions.chunk/v1", + "host": "127.0.0.1", + **listener.transport, + "runtime_id": runtime_id, + "invocation_id": invocation_id, + "request_id": request_id, + }, + } + + +async def test_lifecycle_host_streams_openai_chunks_out_of_band(): + listener = _OpenAIStreamListener(runtime_id="runtime-1", request_id="request-1") + await listener.start() + stream_payload = _openai_stream_payload(listener) + input_stream, output_stream = _streams( + [ + _request("start", {"runtime_context": {"runtime_id": "runtime-1"}}), + _request("invoke_openai_stream", stream_payload), + _request("stop", {"runtime_id": "runtime-1"}), + ] + ) + received_payloads = [] + + class Runtime: + async def start(self, _payload): + pass + + async def invoke(self, _payload): + raise AssertionError("ordinary invoke is not expected") + + async def invoke_openai_stream(self, payload, emit): + received_payloads.append(payload) + await emit( + { + "id": "chunk-1", + "object": "chat.completion.chunk", + "created": (1 << 64) - 1, + "model": "test-model", + "choices": [{"index": 0, "delta": {"content": "hel"}}], + } + ) + await emit( + { + "id": "chunk-2", + "object": "chat.completion.chunk", + "created": 0, + "model": "test-model", + "choices": [{"index": 0, "delta": {"content": "lo"}}], + } + ) + return {"response": "hello"} + + async def stop(self): + pass + + try: + await lifecycle._serve( + Runtime, + config_loader=None, + input_stream=input_stream, + output_stream=output_stream, + ) + records = [ + await asyncio.wait_for(listener.records.get(), timeout=1), + await asyncio.wait_for(listener.records.get(), timeout=1), + await asyncio.wait_for(listener.records.get(), timeout=1), + ] + finally: + await listener.close() + + responses = [json.loads(line) for line in output_stream.getvalue().splitlines()] + assert [response["operation"] for response in responses] == [ + "start", + "invoke_openai_stream", + "stop", + ] + assert responses[1]["outcome"] == { + "status": "succeeded", + "output": {"response": "hello"}, + } + assert [record["id"] for record in records[:2]] == ["chunk-1", "chunk-2"] + assert records[2] is _END + assert received_payloads == [ + { + "runtime_context": stream_payload["runtime_context"], + "request": stream_payload["request"], + } + ] + + +async def test_openai_stream_writer_serializes_concurrent_emits_under_backpressure(): + transport = _BackpressuredStreamWriter() + writer = lifecycle._OpenAIStreamWriter( + asyncio.StreamReader(), + transport, + { + "runtime_id": "runtime-1", + "invocation_id": "invocation-1", + "request_id": "request-1", + }, + ) + + def chunk(identifier: str) -> dict[str, Any]: + return { + "id": identifier, + "object": "chat.completion.chunk", + "created": 0, + "model": "test-model", + "choices": [], + } + + first = asyncio.create_task(writer.emit(chunk("chunk-1"))) + await asyncio.wait_for(transport.first_drain_started.wait(), timeout=1) + second = asyncio.create_task(writer.emit(chunk("chunk-2"))) + await asyncio.sleep(0) + transport.release_first_drain.set() + await asyncio.gather(first, second) + + records = [json.loads(part) for part in transport.parts if part.startswith(b"{")] + assert [record["sequence"] for record in records] == [0, 1] + assert [record["chunk"]["id"] for record in records] == ["chunk-1", "chunk-2"] + + +async def test_openai_stream_writer_serializes_finish_after_an_inflight_emit(): + reader = asyncio.StreamReader() + reader.feed_data(b"HTTP/1.1 200 OK\r\n\r\n") + reader.feed_eof() + transport = _BackpressuredStreamWriter() + writer = lifecycle._OpenAIStreamWriter( + reader, + transport, + { + "runtime_id": "runtime-1", + "invocation_id": "invocation-1", + "request_id": "request-1", + }, + ) + chunk = { + "id": "chunk-1", + "object": "chat.completion.chunk", + "created": 0, + "model": "test-model", + "choices": [], + } + + emit = asyncio.create_task(writer.emit(chunk)) + await asyncio.wait_for(transport.first_drain_started.wait(), timeout=1) + finish = asyncio.create_task(writer.finish()) + await asyncio.sleep(0) + transport.release_first_drain.set() + await asyncio.gather(emit, finish) + + records = [json.loads(part) for part in transport.parts if part.startswith(b"{")] + assert [(record["type"], record["sequence"]) for record in records] == [ + ("chunk", 0), + ("end", 1), + ] + assert transport.parts[-1] == b"0\r\n\r\n" + assert transport.closed + + +async def test_openai_stream_connect_preserves_cancellation_during_cleanup( + monkeypatch: pytest.MonkeyPatch, +): + listener = _OpenAIStreamListener(runtime_id="runtime-1", request_id="request-1") + await listener.start() + + class CancellingWriter: + def __init__(self) -> None: + self.closed = False + + def write(self, _data: bytes) -> None: + pass + + async def drain(self) -> None: + raise asyncio.CancelledError + + def close(self) -> None: + self.closed = True + + async def wait_closed(self) -> None: + raise OSError("secondary cleanup failure") + + writer = CancellingWriter() + + async def open_connection(*_args, **_kwargs): + return asyncio.StreamReader(), writer + + monkeypatch.setattr(asyncio, "open_connection", open_connection) + try: + with pytest.raises(asyncio.CancelledError): + await lifecycle._OpenAIStreamWriter.connect( + _openai_stream_payload(listener) + ) + finally: + await listener.close() + + assert writer.closed + + +@pytest.mark.parametrize( + "primary_error", + [ + lifecycle.LifecycleError("adapter_failure", "adapter failed"), + asyncio.CancelledError(), + ], +) +async def test_openai_stream_preserves_adapter_failure_over_finish_failure( + monkeypatch: pytest.MonkeyPatch, + primary_error: BaseException, +): + finish_error = lifecycle.LifecycleError("finish_failure", "finish failed") + + class FailingWriter: + async def emit(self, _chunk) -> None: + pass + + async def finish(self) -> None: + raise finish_error + + async def connect(_payload): + return FailingWriter(), { + "runtime_context": {"runtime_id": "runtime-1"}, + "request": {"input": "fail"}, + } + + monkeypatch.setattr(lifecycle._OpenAIStreamWriter, "connect", connect) + + class Runtime: + async def invoke_openai_stream(self, _payload, _emit): + raise primary_error + + with pytest.raises(type(primary_error)) as caught: + await lifecycle._handle_invoke_openai_stream( + lifecycle._HostState(), + Runtime(), + {}, + ) + + if isinstance(primary_error, lifecycle.LifecycleError): + assert caught.value.code == primary_error.code + assert caught.value.__cause__ is finish_error + + +def test_lifecycle_host_rejects_unimplemented_openai_stream_without_poisoning_runtime(): + runtime_id = "runtime-1" + payload = { + "runtime_context": { + "runtime_id": runtime_id, + "invocation_id": "invocation-1", + "request_id": "request-1", + }, + "request": {"request_id": "request-1", "input": "hello"}, + "stream": {}, + } + input_stream, output_stream = _streams( + [ + _request("start", {"runtime_context": {"runtime_id": runtime_id}}), + _request("invoke_openai_stream", payload), + _request( + "invoke", + { + "runtime_context": {"runtime_id": runtime_id}, + "request": {"input": "still works"}, + }, + ), + _request("stop", {"runtime_id": runtime_id}), + ] + ) + + class Runtime: + async def start(self, _payload): + pass + + async def invoke(self, payload): + return {"input": payload["request"]["input"]} + + async def stop(self): + pass + + lifecycle.serve(Runtime, input_stream=input_stream, output_stream=output_stream) + + responses = [json.loads(line) for line in output_stream.getvalue().splitlines()] + assert responses[1]["outcome"]["error"] == { + "stage": "invoke", + "code": "lifecycle_openai_stream_unsupported", + "message": "Adapter runtime does not implement OpenAI streaming", + "retryable": False, + } + assert responses[2]["outcome"] == { + "status": "succeeded", + "output": {"input": "still works"}, + } + + +@pytest.mark.parametrize( + "chunk", + [ + { + "id": "missing-model", + "object": "chat.completion.chunk", + "created": 0, + "choices": [], + }, + { + "id": "invalid-choice", + "object": "chat.completion.chunk", + "created": 0, + "model": "test-model", + "choices": [{"index": False, "delta": {}}], + }, + { + "id": " ", + "object": "chat.completion.chunk", + "created": 0, + "model": "test-model", + "choices": [], + }, + { + "id": "blank-model", + "object": "chat.completion.chunk", + "created": 0, + "model": "\t", + "choices": [], + }, + { + "id": "created-overflow", + "object": "chat.completion.chunk", + "created": 1 << 64, + "model": "test-model", + "choices": [], + }, + { + "id": "index-overflow", + "object": "chat.completion.chunk", + "created": 0, + "model": "test-model", + "choices": [{"index": 1 << 32, "delta": {}}], + }, + ], +) +def test_common_host_rejects_chunks_outside_the_declared_openai_profile(chunk): + with pytest.raises(lifecycle.LifecycleError) as caught: + lifecycle._validated_openai_chunk(chunk) + + assert caught.value.code == "lifecycle_invalid_openai_stream_event" + + +def test_malformed_openai_stream_request_uses_the_invoke_error_stage(): + runtime_id = "runtime-1" + input_stream, output_stream = _streams( + [ + _request("start", {"runtime_context": {"runtime_id": runtime_id}}), + _request( + "invoke_openai_stream", + {"runtime_context": ["not", "a", "mapping"]}, + ), + _request("stop", {"runtime_id": runtime_id}), + ] + ) + + class Runtime: + async def start(self, _payload): + pass + + async def invoke(self, _payload): + pass + + async def stop(self): + pass + + lifecycle.serve(Runtime, input_stream=input_stream, output_stream=output_stream) + + responses = [json.loads(line) for line in output_stream.getvalue().splitlines()] + assert responses[1]["outcome"]["error"] == { + "stage": "invoke", + "code": "lifecycle_invalid_request", + "message": "Invalid lifecycle request", + "retryable": False, + } + + def test_lifecycle_host_reuses_one_runtime_and_one_event_loop(): runtime_id = "runtime-1" input_stream, output_stream = _streams( diff --git a/tests/docs/test_python_api_docs.py b/tests/docs/test_python_api_docs.py index 3f85fbb1..d5943638 100644 --- a/tests/docs/test_python_api_docs.py +++ b/tests/docs/test_python_api_docs.py @@ -39,6 +39,9 @@ "nemo_fabric.client": "/reference/api/python-library-reference/client", "nemo_fabric.runtime": "/reference/api/python-library-reference/runtime", "nemo_fabric.streaming": "/reference/api/python-library-reference/streaming", + "nemo_fabric.openai_streaming": ( + "/reference/api/python-library-reference/openai-streaming" + ), "nemo_fabric.models": "/reference/api/python-library-reference/models", "nemo_fabric.types": "/reference/api/python-library-reference/types", "nemo_fabric.errors": "/reference/api/python-library-reference/errors", diff --git a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/fabric-adapter.json b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/fabric-adapter.json index eefc2c3f..9ed8ca7d 100644 --- a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/fabric-adapter.json +++ b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/fabric-adapter.json @@ -3,6 +3,9 @@ "adapter_id": "test.fabric.hermes_shim", "harness": "hermes", "adapter_kind": "python", + "capabilities": { + "streaming": true + }, "runner": { "module": "nemo_fabric_test_adapters.hermes_shim.adapter", "cwd": ".", diff --git a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py index 642d98ce..95536c3b 100644 --- a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py +++ b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py @@ -6,6 +6,9 @@ from __future__ import annotations +from collections.abc import Awaitable +from collections.abc import Callable +from collections.abc import Mapping from pathlib import Path from typing import Any @@ -19,6 +22,7 @@ def main() -> None: class ShimRuntime: def __init__(self) -> None: self._start_payload: dict[str, Any] | None = None + self._openai_stream_invocations = 0 async def start(self, payload: dict[str, Any]) -> None: self._start_payload = payload @@ -36,8 +40,48 @@ async def invoke(self, invocation: dict[str, Any]) -> dict[str, Any]: } return run_selected_mode(payload) + async def invoke_openai_stream( + self, + invocation: dict[str, Any], + emit: Callable[[Mapping[str, Any]], Awaitable[None]], + ) -> dict[str, Any]: + if self._start_payload is None: + raise lifecycle.LifecycleError( + "hermes_runtime_not_started", + "shim runtime is not started", + ) + self._openai_stream_invocations += 1 + request = invocation.get("request") or {} + context = request.get("context") or {} + if context.get("openai_stream_mode") != "empty": + for index, content in enumerate(("hel", "lo")): + await emit( + { + "id": f"shim-chunk-{index}", + "object": "chat.completion.chunk", + "created": 0, + "model": "test-model", + "choices": [ + { + "index": 0, + "delta": {"content": content}, + "finish_reason": None, + } + ], + } + ) + payload = { + **self._start_payload, + "runtime_context": invocation.get("runtime_context"), + "request": request, + } + output = run_selected_mode(payload) + output["openai_stream_invocation_count"] = self._openai_stream_invocations + return output + async def stop(self) -> None: self._start_payload = None + self._openai_stream_invocations = 0 def fabric_config(payload: dict[str, Any]) -> dict[str, Any]: diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index 8b433991..c599949e 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -18,6 +18,7 @@ from nemo_fabric import Fabric from nemo_fabric import FabricConfig from nemo_fabric import FabricConfigError +from nemo_fabric import RunRequest async def test_native_sdk(hermes_shim_agent_dir: Path): @@ -175,6 +176,17 @@ async def smoke(client: Fabric, fixture_agent: Path) -> None: ) as runtime: first = await runtime.invoke(input="hello runtime one") second = await runtime.invoke(input="hello runtime two") + openai_stream = runtime.invoke_openai_stream(input="hello streaming") + openai_chunks = [chunk async for chunk in openai_stream] + openai_result = await openai_stream.result() + empty_stream = runtime.invoke_openai_stream( + request=RunRequest( + input="hello empty streaming", + context={"openai_stream_mode": "empty"}, + ) + ) + empty_chunks = [chunk async for chunk in empty_stream] + empty_result = await empty_stream.result() assert result["status"] == "succeeded" assert result.harness == "hermes" @@ -188,3 +200,15 @@ async def smoke(client: Fabric, fixture_agent: Path) -> None: assert first.harness == "hermes" assert first["runtime_id"] == second["runtime_id"] assert runtime.handle["runtime_id"] == first["runtime_id"] + assert runtime.supports_openai_streaming is True + assert [ + choice["delta"]["content"] + for chunk in openai_chunks + for choice in chunk["choices"] + ] == ["hel", "lo"] + assert openai_result.status == "succeeded" + assert openai_result.output["received"] == "hello streaming" + assert openai_result.output["openai_stream_invocation_count"] == 1 + assert empty_chunks == [] + assert empty_result.status == "succeeded" + assert empty_result.output["openai_stream_invocation_count"] == 2 diff --git a/tests/python/test_openai_streaming.py b/tests/python/test_openai_streaming.py new file mode 100644 index 00000000..32561018 --- /dev/null +++ b/tests/python/test_openai_streaming.py @@ -0,0 +1,941 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Behavior tests for adapter-native OpenAI streaming.""" + +from __future__ import annotations + +import asyncio +import inspect +import json +import socket +import threading +import time +from typing import Any +from unittest.mock import MagicMock + +import pytest + +import nemo_fabric.openai_streaming as openai_streaming +from nemo_fabric import ( + Fabric, + FabricCapabilityError, + FabricRuntimeError, + FabricStateError, + RunRequest, + Runtime, + RuntimeStatus, +) + + +def _plan( + *, + supports_openai_streaming: bool = True, + adapter_supports_openai_streaming: bool | None = None, +) -> dict[str, Any]: + if adapter_supports_openai_streaming is None: + adapter_supports_openai_streaming = supports_openai_streaming + return { + "agent_name": "demo", + "base_dir": ".", + "config": { + "metadata": {"name": "demo"}, + "harness": {"adapter_id": "test.fabric.shim"}, + "runtime": {}, + }, + "adapter_descriptor": { + "descriptor": { + "adapter_id": "test.fabric.shim", + "harness": "shim", + "adapter_kind": "python", + "capabilities": { + "streaming": adapter_supports_openai_streaming, + }, + } + }, + "capabilities": { + "service": False, + "streaming": supports_openai_streaming, + "updates": False, + "cancellation": False, + }, + } + + +def _runtime() -> dict[str, Any]: + return { + "runtime_id": "runtime-1", + "runtime_binding": "fabric-runtime-binding-test", + "agent_name": "demo", + "harness": "shim", + "adapter_kind": "python", + "adapter_id": "test.fabric.shim", + "environment": { + "environment_id": "environment-1", + "provider": "local", + "control_location": "external_control", + "ownership": "caller_owned", + }, + } + + +def _result( + request: dict[str, Any], + runtime: dict[str, Any], + *, + invocation_id: str, + failed: bool = False, +) -> dict[str, Any]: + result = { + "agent_name": "demo", + "harness": "shim", + "adapter_kind": "python", + "adapter_id": "test.fabric.shim", + "runtime_id": runtime["runtime_id"], + "invocation_id": invocation_id, + "request_id": request["request_id"], + "status": "failed" if failed else "succeeded", + "output": {"response": None if failed else "hello"}, + "artifacts": {"artifacts": []}, + "events": [], + } + if failed: + result["error"] = { + "stage": "invoke", + "code": "shim_failed", + "message": "shim reported failure", + "retryable": False, + } + return result + + +def _chunk(identifier: str, content: str) -> dict[str, Any]: + return { + "id": identifier, + "object": "chat.completion.chunk", + "created": (1 << 64) - 1, + "model": "test-model", + "choices": [{"index": 0, "delta": {"content": content}}], + } + + +def _record( + *, + record_type: str = "chunk", + sequence: Any = 0, + runtime_id: str = "runtime-1", + invocation_id: str = "invocation-1", + request_id: str = "request-1", + chunk: dict[str, Any] | None = None, +) -> dict[str, Any]: + record = { + "type": record_type, + "sequence": sequence, + "runtime_id": runtime_id, + "invocation_id": invocation_id, + "request_id": request_id, + } + if record_type == "chunk": + record["chunk"] = chunk or _chunk("chunk-1", "hello") + return record + + +def _read_http_status(stream: Any) -> int: + status_line = stream.readline().decode("ascii") + status = int(status_line.split(" ", 2)[1]) + while stream.readline() not in (b"\r\n", b"\n", b""): + pass + return status + + +async def _read_async_http_status(reader: asyncio.StreamReader) -> int: + status_line = await reader.readline() + status = int(status_line.decode("ascii").split(" ", 2)[1]) + while await reader.readline() not in (b"\r\n", b"\n", b""): + pass + return status + + +def _stream_request(transport: dict[str, Any], *, token: str | None = None) -> bytes: + return ( + "POST /openai-stream HTTP/1.1\r\n" + f"Host: 127.0.0.1:{transport['port']}\r\n" + f"Authorization: Bearer {token or transport['token']}\r\n" + "Content-Type: application/x-ndjson\r\n" + "Transfer-Encoding: chunked\r\n" + "Expect: 100-continue\r\n" + "Connection: close\r\n\r\n" + ).encode("ascii") + + +def _send_stream( + transport: dict[str, Any], + *, + runtime_id: str, + invocation_id: str, + request_id: str, + chunks: list[dict[str, Any]], + token: str | None = None, +) -> None: + with socket.create_connection(("127.0.0.1", transport["port"]), timeout=2) as sock: + sock.sendall(_stream_request(transport, token=token)) + response = sock.makefile("rb") + status = _read_http_status(response) + if status != 100: + raise RuntimeError(f"listener rejected stream with HTTP {status}") + records = [ + { + "type": "chunk", + "sequence": sequence, + "runtime_id": runtime_id, + "invocation_id": invocation_id, + "request_id": request_id, + "chunk": chunk, + } + for sequence, chunk in enumerate(chunks) + ] + records.append( + { + "type": "end", + "sequence": len(chunks), + "runtime_id": runtime_id, + "invocation_id": invocation_id, + "request_id": request_id, + } + ) + for record in records: + encoded = json.dumps(record, separators=(",", ":")).encode() + b"\n" + sock.sendall(f"{len(encoded):X}\r\n".encode() + encoded + b"\r\n") + sock.sendall(b"0\r\n\r\n") + assert _read_http_status(response) == 200 + + +def _send_probe(transport: dict[str, Any], *, token: str) -> int: + with socket.create_connection(("127.0.0.1", transport["port"]), timeout=2) as sock: + sock.sendall(_stream_request(transport, token=token)) + return _read_http_status(sock.makefile("rb")) + + +def _send_records(transport: dict[str, Any], records: list[dict[str, Any]]) -> int: + with socket.create_connection(("127.0.0.1", transport["port"]), timeout=2) as sock: + sock.sendall(_stream_request(transport)) + response = sock.makefile("rb") + assert _read_http_status(response) == 100 + for record in records: + encoded = json.dumps(record, separators=(",", ":")).encode() + b"\n" + sock.sendall(f"{len(encoded):X}\r\n".encode() + encoded + b"\r\n") + sock.sendall(b"0\r\n\r\n") + return _read_http_status(response) + + +@pytest.fixture(name="mock_native") +def mock_native_fixture() -> MagicMock: + mock_native = MagicMock() + mock_native.requests = [] + + def invoke_openai_stream( + _plan_json: str, + runtime_json: str, + request_json: str, + transport_json: str, + ) -> str: + runtime = json.loads(runtime_json) + request = json.loads(request_json) + transport = json.loads(transport_json) + mock_native.requests.append(request) + invocation_id = f"invocation-{len(mock_native.requests)}" + _send_stream( + transport, + runtime_id=runtime["runtime_id"], + invocation_id=invocation_id, + request_id=request["request_id"], + chunks=[_chunk("chunk-1", "hel"), _chunk("chunk-2", "lo")], + ) + return json.dumps(_result(request, runtime, invocation_id=invocation_id)) + + def invoke( + _plan_json: str, + runtime_json: str, + request_json: str, + ) -> str: + runtime = json.loads(runtime_json) + request = json.loads(request_json) + mock_native.requests.append(request) + return json.dumps( + _result( + request, + runtime, + invocation_id=f"invocation-{len(mock_native.requests)}", + ) + ) + + mock_native.invoke_openai_stream.side_effect = invoke_openai_stream + mock_native.invoke_runtime.side_effect = invoke + mock_native.stop_runtime.return_value = "[]" + return mock_native + + +def _runtime_wrapper( + mock_native: MagicMock, + *, + supports_openai_streaming: bool = True, + adapter_supports_openai_streaming: bool | None = None, + relay_streaming: bool = False, +) -> Runtime: + client = Fabric() + client._native_module = lambda: mock_native # type: ignore[method-assign] + return Runtime( + client=client, + plan=_plan( + supports_openai_streaming=supports_openai_streaming, + adapter_supports_openai_streaming=adapter_supports_openai_streaming, + ), + runtime=_runtime(), + stream_listener=MagicMock() if relay_streaming else None, + ) + + +async def test_invoke_openai_stream_yields_chunks_and_separate_result(mock_native): + runtime = _runtime_wrapper(mock_native) + + stream = runtime.invoke_openai_stream(input="hello") + chunks = [chunk async for chunk in stream] + result = await stream.result() + + assert [chunk["id"] for chunk in chunks] == ["chunk-1", "chunk-2"] + assert result.output["response"] == "hello" + assert runtime.invocations == [ + { + "request_id": result.request_id, + "runtime_id": "runtime-1", + "invocation_id": "invocation-1", + } + ] + assert mock_native.invoke_openai_stream.call_count == 1 + + +async def test_invoke_openai_stream_accepts_an_empty_stream(mock_native): + def invoke_empty(_plan_json, runtime_json, request_json, transport_json): + runtime = json.loads(runtime_json) + request = json.loads(request_json) + _send_stream( + json.loads(transport_json), + runtime_id=runtime["runtime_id"], + invocation_id="invocation-empty", + request_id=request["request_id"], + chunks=[], + ) + return json.dumps( + _result(request, runtime, invocation_id="invocation-empty") + ) + + mock_native.invoke_openai_stream.side_effect = invoke_empty + runtime = _runtime_wrapper(mock_native) + + stream = runtime.invoke_openai_stream(input="empty") + + assert [chunk async for chunk in stream] == [] + assert (await stream.result()).status == "succeeded" + + +async def test_openai_stream_aclose_drains_without_cancelling(mock_native): + runtime = _runtime_wrapper(mock_native) + stream = runtime.invoke_openai_stream(input="hello") + + first = await anext(stream) + await stream.aclose() + result = await stream.result() + + assert first["id"] == "chunk-1" + assert result.output["response"] == "hello" + assert mock_native.invoke_openai_stream.call_count == 1 + assert runtime.status is RuntimeStatus.ACTIVE + + +async def test_result_drains_a_stream_larger_than_the_bounded_queue(mock_native): + def invoke_many(_plan_json, runtime_json, request_json, transport_json): + runtime = json.loads(runtime_json) + request = json.loads(request_json) + _send_stream( + json.loads(transport_json), + runtime_id=runtime["runtime_id"], + invocation_id="invocation-many", + request_id=request["request_id"], + chunks=[_chunk(f"chunk-{index}", "x") for index in range(1025)], + ) + return json.dumps( + _result(request, runtime, invocation_id="invocation-many") + ) + + mock_native.invoke_openai_stream.side_effect = invoke_many + runtime = _runtime_wrapper(mock_native) + + result = await runtime.invoke_openai_stream(input="many").result() + + assert result.status == "succeeded" + assert runtime.invocations[0]["invocation_id"] == "invocation-many" + + +async def test_successful_terminal_result_requires_an_explicit_stream_end( + mock_native, + monkeypatch, +): + def invoke_without_stream(_plan_json, runtime_json, request_json, _transport_json): + runtime = json.loads(runtime_json) + request = json.loads(request_json) + return json.dumps( + _result(request, runtime, invocation_id="invocation-without-stream") + ) + + mock_native.invoke_openai_stream.side_effect = invoke_without_stream + monkeypatch.setattr( + openai_streaming, + "_OPENAI_STREAM_COMPLETION_TIMEOUT", + 0.01, + ) + runtime = _runtime_wrapper(mock_native) + + with pytest.raises(FabricRuntimeError, match="did not establish and complete"): + await runtime.invoke_openai_stream(input="missing stream").result() + + assert runtime.status is RuntimeStatus.FAILED + assert runtime.invocations == [] + + +async def test_unauthenticated_probe_does_not_poison_the_adapter_stream(mock_native): + def invoke_after_probe(_plan_json, runtime_json, request_json, transport_json): + runtime = json.loads(runtime_json) + request = json.loads(request_json) + transport = json.loads(transport_json) + assert _send_probe(transport, token="wrong-token") == 401 + _send_stream( + transport, + runtime_id=runtime["runtime_id"], + invocation_id="invocation-after-probe", + request_id=request["request_id"], + chunks=[_chunk("chunk-after-probe", "safe")], + ) + return json.dumps( + _result(request, runtime, invocation_id="invocation-after-probe") + ) + + mock_native.invoke_openai_stream.side_effect = invoke_after_probe + runtime = _runtime_wrapper(mock_native) + stream = runtime.invoke_openai_stream(input="probe") + + assert [chunk["id"] async for chunk in stream] == ["chunk-after-probe"] + assert (await stream.result()).status == "succeeded" + assert runtime.status is RuntimeStatus.ACTIVE + + +async def test_natural_iteration_waits_for_explicit_end_after_terminal_result( + mock_native, +): + senders: list[threading.Thread] = [] + + def invoke_before_stream(_plan_json, runtime_json, request_json, transport_json): + runtime = json.loads(runtime_json) + request = json.loads(request_json) + transport = json.loads(transport_json) + + def send_later() -> None: + time.sleep(0.05) + _send_stream( + transport, + runtime_id=runtime["runtime_id"], + invocation_id="invocation-late-stream", + request_id=request["request_id"], + chunks=[_chunk("late-1", "late"), _chunk("late-2", " stream")], + ) + + sender = threading.Thread(target=send_later) + sender.start() + senders.append(sender) + return json.dumps( + _result(request, runtime, invocation_id="invocation-late-stream") + ) + + mock_native.invoke_openai_stream.side_effect = invoke_before_stream + runtime = _runtime_wrapper(mock_native) + stream = runtime.invoke_openai_stream(input="late") + + chunks = [chunk async for chunk in stream] + result = await stream.result() + for sender in senders: + sender.join(timeout=2) + + assert [chunk["id"] for chunk in chunks] == ["late-1", "late-2"] + assert result.invocation_id == "invocation-late-stream" + assert runtime.status is RuntimeStatus.ACTIVE + + +async def test_cancelling_iteration_after_end_preserves_stream_completion(mock_native): + def invoke_after_end(_plan_json, runtime_json, request_json, transport_json): + runtime = json.loads(runtime_json) + request = json.loads(request_json) + _send_stream( + json.loads(transport_json), + runtime_id=runtime["runtime_id"], + invocation_id="invocation-cancel-after-end", + request_id=request["request_id"], + chunks=[], + ) + time.sleep(0.1) + return json.dumps( + _result( + request, + runtime, + invocation_id="invocation-cancel-after-end", + ) + ) + + mock_native.invoke_openai_stream.side_effect = invoke_after_end + runtime = _runtime_wrapper(mock_native) + stream = runtime.invoke_openai_stream(input="cancel after end") + iterator = asyncio.create_task(anext(stream)) + + async def wait_for_end() -> None: + while not stream._end_observed: + await asyncio.sleep(0) + + try: + await asyncio.wait_for(wait_for_end(), timeout=1) + iterator.cancel() + with pytest.raises(asyncio.CancelledError): + await iterator + finally: + if not iterator.done(): + iterator.cancel() + await asyncio.gather(iterator, return_exceptions=True) + + with pytest.raises(StopAsyncIteration): + await anext(stream) + assert (await stream.result()).status == "succeeded" + assert runtime.status is RuntimeStatus.ACTIVE + + +async def test_cancelling_result_during_cleanup_absorbs_terminal_result_once( + mock_native, +): + runtime = _runtime_wrapper(mock_native) + stream = runtime.invoke_openai_stream(input="cancel cleanup") + close_started = asyncio.Event() + allow_close = asyncio.Event() + original_close = stream._listener.close + + async def delayed_close() -> None: + close_started.set() + await allow_close.wait() + await original_close() + + stream._listener.close = delayed_close + first_result = asyncio.create_task(stream.result()) + await close_started.wait() + first_result.cancel() + with pytest.raises(asyncio.CancelledError): + await first_result + assert len(runtime.invocations) == 1 + + allow_close.set() + result = await stream.result() + + assert result.status == "succeeded" + assert len(runtime.invocations) == 1 + + +async def test_reported_failure_after_a_chunk_keeps_runtime_usable(mock_native): + def invoke_failed(_plan_json, runtime_json, request_json, transport_json): + runtime = json.loads(runtime_json) + request = json.loads(request_json) + _send_stream( + json.loads(transport_json), + runtime_id=runtime["runtime_id"], + invocation_id="invocation-failed", + request_id=request["request_id"], + chunks=[_chunk("chunk-before-failure", "partial")], + ) + return json.dumps( + _result( + request, + runtime, + invocation_id="invocation-failed", + failed=True, + ) + ) + + mock_native.invoke_openai_stream.side_effect = invoke_failed + runtime = _runtime_wrapper(mock_native) + stream = runtime.invoke_openai_stream(input="fail") + + assert [chunk["id"] async for chunk in stream] == ["chunk-before-failure"] + assert (await stream.result()).status == "failed" + assert (await runtime.invoke(input="next")).status == "succeeded" + assert runtime.status is RuntimeStatus.ACTIVE + + +async def test_mismatched_terminal_identity_is_not_absorbed(mock_native): + def invoke_mismatch(_plan_json, runtime_json, request_json, transport_json): + runtime = json.loads(runtime_json) + request = json.loads(request_json) + _send_stream( + json.loads(transport_json), + runtime_id=runtime["runtime_id"], + invocation_id="invocation-stream", + request_id=request["request_id"], + chunks=[], + ) + return json.dumps( + _result(request, runtime, invocation_id="invocation-terminal") + ) + + mock_native.invoke_openai_stream.side_effect = invoke_mismatch + runtime = _runtime_wrapper(mock_native) + + with pytest.raises(FabricRuntimeError, match="does not match its terminal result"): + await runtime.invoke_openai_stream(input="mismatch").result() + + assert runtime.status is RuntimeStatus.FAILED + assert runtime.invocations == [] + + +@pytest.mark.parametrize( + ("record", "message"), + [ + (_record(sequence=False), "sequence is not monotonic"), + (_record(sequence=1), "sequence is not monotonic"), + (_record(runtime_id="runtime-other"), "runtime ID does not match"), + (_record(request_id="request-other"), "request ID does not match"), + ( + _record( + chunk={ + "id": "missing-model", + "object": "chat.completion.chunk", + "created": 0, + "choices": [], + } + ), + "model must be a non-empty string", + ), + ( + _record( + chunk={ + "id": " ", + "object": "chat.completion.chunk", + "created": 0, + "model": "test-model", + "choices": [], + } + ), + "id must be a non-empty string", + ), + ( + _record( + chunk={ + "id": "blank-model", + "object": "chat.completion.chunk", + "created": 0, + "model": "\t", + "choices": [], + } + ), + "model must be a non-empty string", + ), + ( + _record( + chunk={ + "id": "created-overflow", + "object": "chat.completion.chunk", + "created": 1 << 64, + "model": "test-model", + "choices": [], + } + ), + "created must be an unsigned 64-bit integer", + ), + ( + _record( + chunk={ + "id": "index-overflow", + "object": "chat.completion.chunk", + "created": 0, + "model": "test-model", + "choices": [{"index": 1 << 32, "delta": {}}], + } + ), + "index must be an unsigned 32-bit integer", + ), + ], +) +async def test_listener_rejects_invalid_record_invariants(record, message): + listener = openai_streaming._OpenAIStreamListener( + runtime_id="runtime-1", + request_id="request-1", + ) + + with pytest.raises(openai_streaming._ProtocolError, match=message): + await listener._emit_line(json.dumps(record).encode()) + + +async def test_listener_rejects_malformed_oversized_and_changed_identity(): + listener = openai_streaming._OpenAIStreamListener( + runtime_id="runtime-1", + request_id="request-1", + ) + + with pytest.raises(openai_streaming._ProtocolError, match="not valid JSON"): + await listener._emit_line(b"{") + nonfinite = _record() + nonfinite["chunk"]["extension"] = float("nan") + with pytest.raises(openai_streaming._ProtocolError, match="not valid JSON"): + await listener._emit_line(json.dumps(nonfinite).encode()) + + listener = openai_streaming._OpenAIStreamListener( + runtime_id="runtime-1", + request_id="request-1", + max_record_bytes=64, + ) + with pytest.raises(openai_streaming._ProtocolError, match="exceeds 1 MiB"): + await listener._emit_line(b"x" * 65) + + listener = openai_streaming._OpenAIStreamListener( + runtime_id="runtime-1", + request_id="request-1", + ) + await listener._emit_line(json.dumps(_record()).encode()) + with pytest.raises(openai_streaming._ProtocolError, match="invocation ID changed"): + await listener._emit_line( + json.dumps( + _record( + sequence=1, + invocation_id="invocation-other", + chunk=_chunk("chunk-2", "other"), + ) + ).encode() + ) + + +async def test_missing_end_record_fails_the_invocation(mock_native): + def invoke_without_end(_plan_json, runtime_json, request_json, transport_json): + runtime = json.loads(runtime_json) + request = json.loads(request_json) + status = _send_records( + json.loads(transport_json), + [ + _record( + runtime_id=runtime["runtime_id"], + invocation_id="invocation-missing-end", + request_id=request["request_id"], + ) + ], + ) + raise RuntimeError(f"stream listener rejected missing end with HTTP {status}") + + mock_native.invoke_openai_stream.side_effect = invoke_without_end + runtime = _runtime_wrapper(mock_native) + + with pytest.raises(FabricRuntimeError, match="without an end record"): + await runtime.invoke_openai_stream(input="missing end").result() + + assert runtime.status is RuntimeStatus.FAILED + + +async def test_claimed_invalid_chunk_reports_the_stable_protocol_error(mock_native): + def invoke_invalid_chunk(_plan_json, runtime_json, request_json, transport_json): + runtime = json.loads(runtime_json) + request = json.loads(request_json) + invalid_chunk = _chunk("invalid", "bad") + invalid_chunk.pop("model") + status = _send_records( + json.loads(transport_json), + [ + _record( + runtime_id=runtime["runtime_id"], + invocation_id="invocation-invalid-chunk", + request_id=request["request_id"], + chunk=invalid_chunk, + ) + ], + ) + raise RuntimeError(f"native writer failed with HTTP {status}") + + mock_native.invoke_openai_stream.side_effect = invoke_invalid_chunk + runtime = _runtime_wrapper(mock_native) + + with pytest.raises(FabricRuntimeError) as caught: + await runtime.invoke_openai_stream(input="invalid chunk").result() + + assert caught.value.code == "openai_stream_protocol_error" + assert "model must be a non-empty string" in str(caught.value) + assert runtime.status is RuntimeStatus.FAILED + + +async def test_listener_accepts_only_one_simultaneous_authenticated_connection(): + listener = openai_streaming._OpenAIStreamListener( + runtime_id="runtime-1", + request_id="request-1", + ) + await listener.start() + + connections: list[tuple[asyncio.StreamReader, asyncio.StreamWriter]] = [] + + async def candidate(): + reader, writer = await asyncio.open_connection( + "127.0.0.1", + listener.transport["port"], + ) + connections.append((reader, writer)) + writer.write(_stream_request(listener.transport)) + await writer.drain() + return await _read_async_http_status(reader), reader, writer + + candidates = [asyncio.create_task(candidate()) for _ in range(2)] + try: + results = await asyncio.gather(*candidates) + assert sorted(status for status, _reader, _writer in results) == [100, 409] + status, reader, writer = next(item for item in results if item[0] == 100) + assert status == 100 + encoded = json.dumps( + _record(record_type="end"), separators=(",", ":") + ).encode() + b"\n" + writer.write(f"{len(encoded):X}\r\n".encode() + encoded + b"\r\n0\r\n\r\n") + await writer.drain() + assert await _read_async_http_status(reader) == 200 + finally: + for candidate_task in candidates: + if not candidate_task.done(): + candidate_task.cancel() + await asyncio.gather(*candidates, return_exceptions=True) + for _reader, writer in connections: + writer.close() + await writer.wait_closed() + await listener.close() + + +async def test_listener_requires_an_exact_chunked_transfer_coding(): + listener = openai_streaming._OpenAIStreamListener( + runtime_id="runtime-1", + request_id="request-1", + ) + await listener.start() + reader, writer = await asyncio.open_connection( + "127.0.0.1", + listener.transport["port"], + ) + request = _stream_request(listener.transport).replace( + b"Transfer-Encoding: chunked", + b"Transfer-Encoding: unchunked", + ) + try: + writer.write(request) + await writer.drain() + + assert await _read_async_http_status(reader) == 411 + assert listener.error is None + finally: + writer.close() + await writer.wait_closed() + await listener.close() + + +@pytest.mark.parametrize( + ("supports_openai_streaming", "supports_relay_streaming"), + [(False, False), (False, True), (True, False), (True, True)], +) +def test_native_and_relay_streaming_capabilities_are_independent( + mock_native, + supports_openai_streaming: bool, + supports_relay_streaming: bool, +): + runtime = _runtime_wrapper( + mock_native, + supports_openai_streaming=supports_openai_streaming, + relay_streaming=supports_relay_streaming, + ) + + assert runtime.supports_openai_streaming is supports_openai_streaming + assert runtime.supports_streaming is supports_relay_streaming + + +def test_invoke_openai_stream_rejects_an_unsupported_adapter(mock_native): + runtime = _runtime_wrapper(mock_native, supports_openai_streaming=False) + + with pytest.raises(FabricCapabilityError) as caught: + runtime.invoke_openai_stream(input="hello") + + assert caught.value.code == "openai_streaming_unavailable" + mock_native.invoke_openai_stream.assert_not_called() + + +def test_openai_streaming_requires_the_resolved_descriptor_claim(mock_native): + runtime = _runtime_wrapper( + mock_native, + supports_openai_streaming=True, + adapter_supports_openai_streaming=False, + ) + + assert runtime.supports_openai_streaming is False + with pytest.raises(FabricCapabilityError) as caught: + runtime.invoke_openai_stream(input="hello") + + assert caught.value.code == "openai_streaming_unavailable" + mock_native.invoke_openai_stream.assert_not_called() + + +def test_openai_stream_constructor_closes_run_when_task_creation_fails(monkeypatch): + captured = [] + + def fail_create_task(coroutine): + captured.append(coroutine) + raise RuntimeError("no running event loop") + + async def invoke(_transport): + raise AssertionError("invoke must not run") + + monkeypatch.setattr(asyncio, "create_task", fail_create_task) + + with pytest.raises(RuntimeError, match="no running event loop"): + openai_streaming.OpenAIInvokeStream( + invoke, + runtime_id="runtime-1", + request_id="request-1", + ) + + assert len(captured) == 1 + assert inspect.getcoroutinestate(captured[0]) == inspect.CORO_CLOSED + + +async def test_invoke_openai_stream_rejects_concurrent_turns(mock_native): + runtime = _runtime_wrapper(mock_native) + stream = runtime.invoke_openai_stream(input="first") + + with pytest.raises(FabricStateError, match="streaming invocation is active"): + runtime.invoke_openai_stream(input="second") + with pytest.raises(FabricStateError, match="streaming invocation is active"): + await runtime.invoke(input="second") + + await stream.aclose() + + +async def test_openai_stream_protocol_failure_marks_runtime_failed(mock_native): + def invoke_with_bad_token(_plan_json, runtime_json, request_json, transport_json): + runtime = json.loads(runtime_json) + request = json.loads(request_json) + _send_stream( + json.loads(transport_json), + runtime_id=runtime["runtime_id"], + invocation_id="invocation-bad", + request_id=request["request_id"], + chunks=[], + token="wrong-token", + ) + raise AssertionError("listener must reject the token") + + mock_native.invoke_openai_stream.side_effect = invoke_with_bad_token + runtime = _runtime_wrapper(mock_native) + stream = runtime.invoke_openai_stream(request=RunRequest(input="hello")) + + with pytest.raises(FabricRuntimeError): + await stream.result() + + assert runtime.status is RuntimeStatus.FAILED + await stream.aclose()