From 337b556d568ec8ffddbf496f3facfaa4fbd41ba6 Mon Sep 17 00:00:00 2001 From: svonava Date: Sun, 19 Jul 2026 02:06:03 -0700 Subject: [PATCH 1/3] fix(sdk): serialize audio and video inputs --- packages/sie_gateway/openapi.json | 72 ++++++++++++ packages/sie_gateway/src/openapi.rs | 37 ++++++ .../sie_sdk/src/sie_sdk/client/_shared.py | 21 ++-- packages/sie_sdk/src/sie_sdk/client/async_.py | 25 ++-- packages/sie_sdk/src/sie_sdk/client/sync.py | 25 ++-- packages/sie_sdk/src/sie_sdk/media.py | 63 ++++++++++ packages/sie_sdk/tests/client/test_async.py | 34 +++++- packages/sie_sdk/tests/client/test_sync.py | 37 +++++- packages/sie_sdk/tests/test_media.py | 63 ++++++++++ packages/sie_server/openapi.json | 111 ++++++++++++++++-- .../sie_server/src/sie_server/api/openapi.py | 20 +++- .../src/sie_server/types/openapi.py | 17 +++ .../sie_server/tests/test_openapi_export.py | 10 ++ 13 files changed, 490 insertions(+), 45 deletions(-) create mode 100644 packages/sie_sdk/src/sie_sdk/media.py create mode 100644 packages/sie_sdk/tests/test_media.py diff --git a/packages/sie_gateway/openapi.json b/packages/sie_gateway/openapi.json index c77721930..df2810e83 100644 --- a/packages/sie_gateway/openapi.json +++ b/packages/sie_gateway/openapi.json @@ -41,6 +41,36 @@ ], "type": "object" }, + "AudioInput": { + "properties": { + "data": { + "items": { + "format": "int32", + "minimum": 0, + "type": "integer" + }, + "type": "array" + }, + "format": { + "type": [ + "string", + "null" + ] + }, + "sample_rate": { + "format": "int32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "data" + ], + "type": "object" + }, "BundleConfigDocument": { "properties": { "adapters": { @@ -1702,6 +1732,16 @@ }, "ItemInput": { "properties": { + "audio": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/AudioInput" + } + ] + }, "document": { "oneOf": [ { @@ -1733,6 +1773,16 @@ "string", "null" ] + }, + "video": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/VideoInput" + } + ] } }, "type": "object" @@ -2930,6 +2980,28 @@ ], "type": "object" }, + "VideoInput": { + "properties": { + "data": { + "items": { + "format": "int32", + "minimum": 0, + "type": "integer" + }, + "type": "array" + }, + "format": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "data" + ], + "type": "object" + }, "WorkerInfo": { "properties": { "bundle": { diff --git a/packages/sie_gateway/src/openapi.rs b/packages/sie_gateway/src/openapi.rs index 92f60f6dd..b6ad3f2cf 100644 --- a/packages/sie_gateway/src/openapi.rs +++ b/packages/sie_gateway/src/openapi.rs @@ -79,6 +79,7 @@ static OPENAPI_JSON: LazyLock = LazyLock::new(|| { InferenceInternalServerErrorResponse, InferenceServiceUnavailableResponse, AllItemsFailedResponse, + AudioInput, BundleRoutingConflictDetail, BundleConflictResponse, DocumentInput, @@ -144,6 +145,7 @@ static OPENAPI_JSON: LazyLock = LazyLock::new(|| { ScoreResponse, SparseVector, TimingInfo, + VideoInput, crate::types::pool::AssignedWorker, crate::types::worker::WorkerInfo )), @@ -2173,6 +2175,22 @@ pub struct ImageInput { pub format: Option, } +#[derive(Debug, Serialize, Deserialize, ToSchema)] +pub struct AudioInput { + pub data: Vec, + #[serde(default)] + pub format: Option, + #[serde(default)] + pub sample_rate: Option, +} + +#[derive(Debug, Serialize, Deserialize, ToSchema)] +pub struct VideoInput { + pub data: Vec, + #[serde(default)] + pub format: Option, +} + #[derive(Debug, Serialize, Deserialize, ToSchema)] pub struct DocumentInput { pub data: Vec, @@ -2189,6 +2207,10 @@ pub struct ItemInput { #[serde(default)] pub images: Option>, #[serde(default)] + pub audio: Option, + #[serde(default)] + pub video: Option, + #[serde(default)] pub document: Option, #[serde(default)] pub metadata: Option, @@ -2417,6 +2439,21 @@ mod tests { ); } + #[test] + fn openapi_json_documents_native_audio_and_video_inputs() { + let spec: serde_json::Value = serde_json::from_str(&OPENAPI_JSON).unwrap(); + let properties = &spec["components"]["schemas"]["ItemInput"]["properties"]; + + assert_eq!( + properties["audio"]["oneOf"][1]["$ref"], + "#/components/schemas/AudioInput" + ); + assert_eq!( + properties["video"]["oneOf"][1]["$ref"], + "#/components/schemas/VideoInput" + ); + } + #[tokio::test] async fn docs_ui_serves_self_contained_redoc_html() { let resp = docs_ui().await.into_response(); diff --git a/packages/sie_sdk/src/sie_sdk/client/_shared.py b/packages/sie_sdk/src/sie_sdk/client/_shared.py index 9973c96d5..5b518ea6e 100644 --- a/packages/sie_sdk/src/sie_sdk/client/_shared.py +++ b/packages/sie_sdk/src/sie_sdk/client/_shared.py @@ -18,6 +18,7 @@ import numpy as np from sie_sdk.images import convert_item_images +from sie_sdk.media import convert_item_media _logger = logging.getLogger(__name__) @@ -121,13 +122,19 @@ def json(self) -> Any: ... _retry_rng = random.Random() # noqa: S311 — non-cryptographic jitter only -def convert_score_images_for_wire(query: Any, items: Sequence[Any]) -> tuple[Any, list[Any]]: - """Convert image-bearing score query/items to the SDK image wire shape.""" - query_for_wire = convert_item_images({**query}) if "images" in query else query - items_for_wire = [ - convert_item_images({**item}) if "images" in item else item # ty: ignore[invalid-argument-type] - for item in items - ] +def convert_score_media_for_wire(query: Any, items: Sequence[Any]) -> tuple[Any, list[Any]]: + """Convert media-bearing score query/items to SDK wire shapes.""" + + def convert(item: Any) -> Any: + if not any(field in item for field in ("images", "audio", "video")): + return item + item_for_wire = {**item} + if "images" in item_for_wire: + item_for_wire = convert_item_images(item_for_wire) + return convert_item_media(item_for_wire) + + query_for_wire = convert(query) + items_for_wire = [convert(item) for item in items] return query_for_wire, items_for_wire diff --git a/packages/sie_sdk/src/sie_sdk/client/async_.py b/packages/sie_sdk/src/sie_sdk/client/async_.py index 6a0f342eb..bb93c8043 100644 --- a/packages/sie_sdk/src/sie_sdk/client/async_.py +++ b/packages/sie_sdk/src/sie_sdk/client/async_.py @@ -47,6 +47,7 @@ from sie_sdk.files import resolve_upload from sie_sdk.images import convert_item_images from sie_sdk.jobs import TERMINAL_JOB_STATES, build_job_body, decode_chunk_bytes, job_chunks +from sie_sdk.media import convert_item_media from sie_sdk.types import ( Batch, CapacityInfo, @@ -97,7 +98,7 @@ check_version_skew, compute_oom_backoff, compute_retry_delay, - convert_score_images_for_wire, + convert_score_media_for_wire, get_error_code, get_retry_after, get_sdk_version, @@ -1131,12 +1132,17 @@ async def encode( single_item = not isinstance(items, list) items_list = [items] if single_item else items - # Convert images to JPEG bytes for transport. - # Only copy items that have images — text-only items are passed through directly - items_for_wire = [ - convert_item_images({**item}) if "images" in item else item # ty: ignore[invalid-argument-type] - for item in items_list - ] + # Convert media to bytes for transport. + # Only copy media-bearing items; text-only items pass through directly. + items_for_wire = [] + for item in items_list: + if not any(field in item for field in ("images", "audio", "video")): + items_for_wire.append(item) + continue + wire_item: dict[str, Any] = {**item} # ty: ignore[invalid-argument-type] + if "images" in wire_item: + wire_item = convert_item_images(wire_item) + items_for_wire.append(convert_item_media(wire_item)) # Build request body request_body: dict[str, Any] = {"items": items_for_wire} @@ -1600,7 +1606,7 @@ async def score( # Resolve defaults and pool pool_name, resolved_gpu = await self._resolve_pool_and_gpu(gpu) resolved_options = self._resolve_options(options) - query_for_wire, items_for_wire = convert_score_images_for_wire(query, items) + query_for_wire, items_for_wire = convert_score_media_for_wire(query, items) # Build request body request_body: dict[str, Any] = { @@ -2337,7 +2343,7 @@ async def extract( single_item = not isinstance(items, list) items_list = [items] if single_item else items - # Convert images and documents to wire format (bytes + format hint) + # Convert media and documents to wire format (bytes + format hint) items_for_wire = [] for item in items_list: wire_item: dict[str, Any] = {**item} # ty: ignore[invalid-argument-type] @@ -2345,6 +2351,7 @@ async def extract( wire_item = convert_item_images(wire_item) if "document" in wire_item: wire_item = convert_item_document(wire_item) + wire_item = convert_item_media(wire_item) items_for_wire.append(wire_item) # Build request body diff --git a/packages/sie_sdk/src/sie_sdk/client/sync.py b/packages/sie_sdk/src/sie_sdk/client/sync.py index 9055dc9bd..167a3e85c 100644 --- a/packages/sie_sdk/src/sie_sdk/client/sync.py +++ b/packages/sie_sdk/src/sie_sdk/client/sync.py @@ -54,6 +54,7 @@ from sie_sdk.files import resolve_upload from sie_sdk.images import convert_item_images from sie_sdk.jobs import TERMINAL_JOB_STATES, build_job_body, decode_chunk_bytes, job_chunks +from sie_sdk.media import convert_item_media from sie_sdk.types import ( Batch, CapacityInfo, @@ -104,7 +105,7 @@ check_version_skew, compute_oom_backoff, compute_retry_delay, - convert_score_images_for_wire, + convert_score_media_for_wire, get_error_code, get_retry_after, get_sdk_version, @@ -1081,12 +1082,17 @@ def encode( single_item = not isinstance(items, list) items_list = [items] if single_item else items - # Convert images to JPEG bytes for transport. - # Only copy items that have images — text-only items are passed through directly - items_for_wire = [ - convert_item_images({**item}) if "images" in item else item # ty: ignore[invalid-argument-type] - for item in items_list - ] + # Convert media to bytes for transport. + # Only copy media-bearing items; text-only items pass through directly. + items_for_wire = [] + for item in items_list: + if not any(field in item for field in ("images", "audio", "video")): + items_for_wire.append(item) + continue + wire_item: dict[str, Any] = {**item} # ty: ignore[invalid-argument-type] + if "images" in wire_item: + wire_item = convert_item_images(wire_item) + items_for_wire.append(convert_item_media(wire_item)) # Build request body request_body: dict[str, Any] = {"items": items_for_wire} @@ -1729,7 +1735,7 @@ def score( # Resolve defaults and pool pool_name, resolved_gpu = self._resolve_pool_and_gpu(gpu) resolved_options = self._resolve_options(options) - query_for_wire, items_for_wire = convert_score_images_for_wire(query, items) + query_for_wire, items_for_wire = convert_score_media_for_wire(query, items) # Build request body request_body: dict[str, Any] = { @@ -2569,7 +2575,7 @@ def extract( single_item = not isinstance(items, list) items_list = [items] if single_item else items - # Convert images and documents to wire format (bytes + format hint) + # Convert media and documents to wire format (bytes + format hint) items_for_wire = [] for item in items_list: wire_item: dict[str, Any] = {**item} # ty: ignore[invalid-argument-type] @@ -2577,6 +2583,7 @@ def extract( wire_item = convert_item_images(wire_item) if "document" in wire_item: wire_item = convert_item_document(wire_item) + wire_item = convert_item_media(wire_item) items_for_wire.append(wire_item) # Build request body diff --git a/packages/sie_sdk/src/sie_sdk/media.py b/packages/sie_sdk/src/sie_sdk/media.py new file mode 100644 index 000000000..8e2014d63 --- /dev/null +++ b/packages/sie_sdk/src/sie_sdk/media.py @@ -0,0 +1,63 @@ +"""Audio and video conversion utilities for SIE SDK. + +Wire format: raw media bytes in msgpack with an optional format hint. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +MediaLike = bytes | str | Path + + +def infer_media_format(source: str | Path) -> str | None: + """Infer a media format hint from a path suffix.""" + suffix = Path(source).suffix.lower() + return suffix.removeprefix(".") or None + + +def to_media_bytes(media: MediaLike, *, kind: str) -> tuple[bytes, str | None]: + """Resolve an audio or video input to bytes and an optional format hint.""" + if isinstance(media, bytes): + return media, None + + if isinstance(media, (str, Path)): + path = Path(media) + if not path.exists(): + msg = f"{kind.capitalize()} file not found: {path}" + raise FileNotFoundError(msg) + return path.read_bytes(), infer_media_format(path) + + msg = f"Unsupported {kind} type: {type(media)}. Expected bytes, str, or Path." + raise TypeError(msg) + + +def _convert_media_field(item: dict[str, Any], field: str) -> None: + media = item.get(field) + if media is None: + return + + if isinstance(media, dict): + if "data" not in media: + msg = f"{field.capitalize()} input must contain a 'data' field." + raise ValueError(msg) + data, inferred = to_media_bytes(media["data"], kind=field) + converted: dict[str, Any] = { + "data": data, + "format": media.get("format", inferred), + } + if field == "audio" and "sample_rate" in media: + converted["sample_rate"] = media["sample_rate"] + item[field] = converted + return + + data, inferred = to_media_bytes(media, kind=field) + item[field] = {"data": data, "format": inferred} + + +def convert_item_media(item: dict[str, Any]) -> dict[str, Any]: + """Convert an item's audio and video fields to their wire shapes in-place.""" + _convert_media_field(item, "audio") + _convert_media_field(item, "video") + return item diff --git a/packages/sie_sdk/tests/client/test_async.py b/packages/sie_sdk/tests/client/test_async.py index cd52db739..b74f5f7c8 100644 --- a/packages/sie_sdk/tests/client/test_async.py +++ b/packages/sie_sdk/tests/client/test_async.py @@ -308,6 +308,25 @@ async def test_encode_single_item_returns_single_result(self) -> None: assert result["dense"].shape == (4,) await client.close() + @pytest.mark.asyncio + async def test_encode_converts_video_to_wire_format(self) -> None: + """Async encode converts video shorthand before msgpack serialization.""" + resp = _make_msgpack_response( + { + "model": "video-encoder", + "items": [{"dense": {"dims": 2, "dtype": "float32", "values": np.array([1.0, 2.0], dtype=np.float32)}}], + } + ) + + client = SIEAsyncClient("http://localhost:8080") + client._post = AsyncMock(return_value=resp) # type: ignore + await client.encode("video-encoder", {"video": b"video"}) + + body = client._post.call_args.kwargs["data"] + request_body = msgpack.unpackb(body, raw=False) + assert request_body["items"][0]["video"] == {"data": b"video", "format": None} + await client.close() + @pytest.mark.asyncio async def test_encode_list_returns_list(self) -> None: resp = _make_msgpack_response( @@ -421,7 +440,7 @@ async def test_score_returns_score_result(self) -> None: @pytest.mark.asyncio async def test_score_converts_image_query_and_items_to_wire_format(self) -> None: - """Async score converts image query/items before msgpack serialization.""" + """Async score converts media query/items before msgpack serialization.""" resp = _make_msgpack_response( { "model": "qwen3-vl-reranker", @@ -431,8 +450,8 @@ async def test_score_converts_image_query_and_items_to_wire_format(self) -> None query_image = b"\xff\xd8\xff\xe0query" item_image = b"\xff\xd8\xff\xe0item" - query = {"text": "rocket nozzle", "images": [query_image]} - item = {"id": "page-1", "images": [item_image]} + query = {"text": "rocket nozzle", "images": [query_image], "audio": b"query-audio"} + item = {"id": "page-1", "images": [item_image], "video": b"item-video"} client = SIEAsyncClient("http://localhost:8080") client._post = AsyncMock(return_value=resp) # type: ignore @@ -444,8 +463,12 @@ async def test_score_converts_image_query_and_items_to_wire_format(self) -> None item_wire = request_body["items"][0]["images"][0] assert query_wire == {"data": query_image, "format": "jpeg"} assert item_wire == {"data": item_image, "format": "jpeg"} + assert request_body["query"]["audio"] == {"data": b"query-audio", "format": None} + assert request_body["items"][0]["video"] == {"data": b"item-video", "format": None} assert query["images"][0] == query_image + assert query["audio"] == b"query-audio" assert item["images"][0] == item_image + assert item["video"] == b"item-video" await client.close() @@ -507,7 +530,7 @@ async def test_extract_list_returns_list(self) -> None: @pytest.mark.asyncio async def test_extract_converts_document_to_wire_format(self) -> None: - """Async extract converts document inputs and returns parsed `data`.""" + """Async extract converts document and video inputs.""" resp = _make_msgpack_response( { "model": "docling", @@ -519,7 +542,7 @@ async def test_extract_converts_document_to_wire_format(self) -> None: client._post = AsyncMock(return_value=resp) # type: ignore result = await client.extract( "docling", - {"document": b"%PDF-1.4 fake content"}, + {"document": b"%PDF-1.4 fake content", "video": b"video"}, ) body = client._post.call_args.kwargs["data"] @@ -527,6 +550,7 @@ async def test_extract_converts_document_to_wire_format(self) -> None: wire_doc = request_body["items"][0]["document"] assert wire_doc["data"] == b"%PDF-1.4 fake content" assert wire_doc["format"] is None + assert request_body["items"][0]["video"] == {"data": b"video", "format": None} assert result["data"] == {"document": {"pages": []}} await client.close() diff --git a/packages/sie_sdk/tests/client/test_sync.py b/packages/sie_sdk/tests/client/test_sync.py index 4e9bf1cd3..d50bd5ed0 100644 --- a/packages/sie_sdk/tests/client/test_sync.py +++ b/packages/sie_sdk/tests/client/test_sync.py @@ -227,6 +227,28 @@ def test_encode_single_item_returns_single_result(self) -> None: assert result["dense"].shape == (4,) client.close() + def test_encode_converts_audio_to_wire_format(self) -> None: + """encode() converts audio shorthand before msgpack serialization.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.content = msgpack.packb( + { + "model": "audio-encoder", + "items": [{"dense": {"dims": 2, "dtype": "float32", "values": np.array([1.0, 2.0], dtype=np.float32)}}], + }, + use_bin_type=True, + ) + + with patch("sie_sdk.client.sync.httpx.Client") as mock_client: + mock_client.return_value.post.return_value = mock_response + client = SIEClient("http://localhost:8080") + client.encode("audio-encoder", {"audio": b"audio"}) + + call_args = mock_client.return_value.post.call_args + request_body = msgpack.unpackb(call_args.kwargs["content"], raw=False) + assert request_body["items"][0]["audio"] == {"data": b"audio", "format": None} + client.close() + def test_encode_list_returns_list(self) -> None: """List of items input returns list of results.""" mock_response = MagicMock() @@ -776,7 +798,7 @@ def test_score_with_instruction(self) -> None: client.close() def test_score_converts_image_query_and_items_to_wire_format(self) -> None: - """score() converts image query/items before msgpack serialization.""" + """score() converts media query/items before msgpack serialization.""" mock_response = MagicMock() mock_response.status_code = 200 mock_response.content = msgpack.packb( @@ -789,8 +811,8 @@ def test_score_converts_image_query_and_items_to_wire_format(self) -> None: query_image = b"\xff\xd8\xff\xe0query" item_image = b"\xff\xd8\xff\xe0item" - query = {"text": "rocket nozzle", "images": [query_image]} - item = {"id": "page-1", "images": [item_image]} + query = {"text": "rocket nozzle", "images": [query_image], "audio": b"query-audio"} + item = {"id": "page-1", "images": [item_image], "video": b"item-video"} with patch("sie_sdk.client.sync.httpx.Client") as mock_client: mock_client.return_value.post.return_value = mock_response @@ -803,8 +825,12 @@ def test_score_converts_image_query_and_items_to_wire_format(self) -> None: item_wire = request_body["items"][0]["images"][0] assert query_wire == {"data": query_image, "format": "jpeg"} assert item_wire == {"data": item_image, "format": "jpeg"} + assert request_body["query"]["audio"] == {"data": b"query-audio", "format": None} + assert request_body["items"][0]["video"] == {"data": b"item-video", "format": None} assert query["images"][0] == query_image + assert query["audio"] == b"query-audio" assert item["images"][0] == item_image + assert item["video"] == b"item-video" client.close() def test_score_with_query_id(self) -> None: @@ -944,7 +970,7 @@ def test_extract_preserves_item_id(self) -> None: client.close() def test_extract_converts_document_to_wire_format(self) -> None: - """extract() converts document inputs (bytes/path) to {data, format} on the wire.""" + """extract() converts document and audio inputs to their wire formats.""" mock_response = MagicMock() mock_response.status_code = 200 mock_response.content = msgpack.packb( @@ -960,7 +986,7 @@ def test_extract_converts_document_to_wire_format(self) -> None: client = SIEClient("http://localhost:8080") result = client.extract( "docling", - {"document": b"%PDF-1.4 fake content"}, + {"document": b"%PDF-1.4 fake content", "audio": b"audio"}, ) call_args = mock_client.return_value.post.call_args @@ -968,6 +994,7 @@ def test_extract_converts_document_to_wire_format(self) -> None: wire_doc = request_body["items"][0]["document"] assert wire_doc["data"] == b"%PDF-1.4 fake content" assert wire_doc["format"] is None # bytes have no inferable format + assert request_body["items"][0]["audio"] == {"data": b"audio", "format": None} assert result["data"] == {"document": {"pages": []}} client.close() diff --git a/packages/sie_sdk/tests/test_media.py b/packages/sie_sdk/tests/test_media.py new file mode 100644 index 000000000..942e1eec2 --- /dev/null +++ b/packages/sie_sdk/tests/test_media.py @@ -0,0 +1,63 @@ +from pathlib import Path + +import pytest +from sie_sdk.media import convert_item_media, infer_media_format, to_media_bytes + + +def test_infer_media_format_from_suffix() -> None: + assert infer_media_format("recording.WAV") == "wav" + assert infer_media_format(Path("clip.webm")) == "webm" + assert infer_media_format("recording") is None + + +def test_to_media_bytes_passes_bytes_through() -> None: + assert to_media_bytes(b"audio", kind="audio") == (b"audio", None) + + +def test_to_media_bytes_reads_path_and_infers_format(tmp_path: Path) -> None: + recording = tmp_path / "recording.flac" + recording.write_bytes(b"audio") + + assert to_media_bytes(recording, kind="audio") == (b"audio", "flac") + + +def test_to_media_bytes_rejects_missing_path(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError, match="Audio file not found"): + to_media_bytes(tmp_path / "missing.wav", kind="audio") + + +def test_convert_item_media_converts_direct_inputs(tmp_path: Path) -> None: + clip = tmp_path / "clip.mp4" + clip.write_bytes(b"video") + item = {"audio": b"audio", "video": clip} + + result = convert_item_media(item) + + assert result is item + assert result["audio"] == {"data": b"audio", "format": None} + assert result["video"] == {"data": b"video", "format": "mp4"} + + +def test_convert_item_media_preserves_explicit_metadata(tmp_path: Path) -> None: + recording = tmp_path / "recording.wav" + recording.write_bytes(b"audio") + item = { + "audio": { + "data": recording, + "format": "pcm", + "sample_rate": 16_000, + } + } + + result = convert_item_media(item) + + assert result["audio"] == { + "data": b"audio", + "format": "pcm", + "sample_rate": 16_000, + } + + +def test_convert_item_media_requires_data_in_mapping() -> None: + with pytest.raises(ValueError, match="Audio input must contain a 'data' field"): + convert_item_media({"audio": {"format": "wav"}}) diff --git a/packages/sie_server/openapi.json b/packages/sie_server/openapi.json index 3fafe54a6..a6000feac 100644 --- a/packages/sie_server/openapi.json +++ b/packages/sie_server/openapi.json @@ -1505,6 +1505,48 @@ ], "title": "ValidationError" }, + "AudioInputModel": { + "description": "Audio input for audio models.", + "properties": { + "data": { + "description": "Audio data as bytes", + "format": "binary", + "title": "Data", + "type": "string" + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Audio format hint: 'wav', 'mp3', etc.", + "title": "Format" + }, + "sample_rate": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Audio sample rate in Hz", + "title": "Sample Rate" + } + }, + "required": [ + "data" + ], + "title": "AudioInputModel", + "type": "object" + }, "DocumentInputModel": { "description": "Document input for composite-document extractors (PDF, DOCX, HTML, ...).", "properties": { @@ -1674,7 +1716,7 @@ "anyOf": [ { "items": { - "$ref": "#/$defs/ImageInputModel" + "$ref": "#/components/schemas/ImageInputModel" }, "type": "array" }, @@ -1686,10 +1728,34 @@ "description": "Images for multimodal models", "title": "Images" }, + "audio": { + "anyOf": [ + { + "$ref": "#/components/schemas/AudioInputModel" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Audio for audio-capable models" + }, + "video": { + "anyOf": [ + { + "$ref": "#/components/schemas/VideoInputModel" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Video for video-capable models" + }, "document": { "anyOf": [ { - "$ref": "#/$defs/DocumentInputModel" + "$ref": "#/components/schemas/DocumentInputModel" }, { "type": "null" @@ -1716,6 +1782,35 @@ "title": "ItemModel", "type": "object" }, + "VideoInputModel": { + "description": "Video input for video models.", + "properties": { + "data": { + "description": "Video data as bytes", + "format": "binary", + "title": "Data", + "type": "string" + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Video format hint: 'mp4', 'webm', etc.", + "title": "Format" + } + }, + "required": [ + "data" + ], + "title": "VideoInputModel", + "type": "object" + }, "EncodeRequestModel": { "description": "Request body for encode endpoint.", "examples": [ @@ -1734,7 +1829,7 @@ "items": { "description": "Items to encode", "items": { - "$ref": "#/$defs/ItemModel" + "$ref": "#/components/schemas/ItemModel" }, "minItems": 1, "title": "Items", @@ -1743,7 +1838,7 @@ "params": { "anyOf": [ { - "$ref": "#/$defs/EncodeParamsModel" + "$ref": "#/components/schemas/EncodeParamsModel" }, { "type": "null" @@ -1862,7 +1957,7 @@ "items": { "description": "Items to extract from", "items": { - "$ref": "#/$defs/ItemModel" + "$ref": "#/components/schemas/ItemModel" }, "minItems": 1, "title": "Items", @@ -1871,7 +1966,7 @@ "params": { "anyOf": [ { - "$ref": "#/$defs/ExtractParamsModel" + "$ref": "#/components/schemas/ExtractParamsModel" }, { "type": "null" @@ -1906,13 +2001,13 @@ ], "properties": { "query": { - "$ref": "#/$defs/ItemModel", + "$ref": "#/components/schemas/ItemModel", "description": "Query item to score against" }, "items": { "description": "Items to score", "items": { - "$ref": "#/$defs/ItemModel" + "$ref": "#/components/schemas/ItemModel" }, "minItems": 1, "title": "Items", diff --git a/packages/sie_server/src/sie_server/api/openapi.py b/packages/sie_server/src/sie_server/api/openapi.py index 464481b59..0dae06536 100644 --- a/packages/sie_server/src/sie_server/api/openapi.py +++ b/packages/sie_server/src/sie_server/api/openapi.py @@ -63,9 +63,25 @@ def _add_request_body_schemas(openapi_schema: dict[str, Any]) -> None: if "$defs" in full_schema: for def_name, def_schema in full_schema["$defs"].items(): if def_name not in schemas: - schemas[def_name] = def_schema + schemas[def_name] = _rewrite_definition_refs(def_schema) del full_schema["$defs"] - schemas[model_name] = full_schema + schemas[model_name] = _rewrite_definition_refs(full_schema) + + +def _rewrite_definition_refs(value: Any) -> Any: + """Point Pydantic ``$defs`` references at OpenAPI components.""" + if isinstance(value, dict): + return { + key: ( + item.replace("#/$defs/", "#/components/schemas/") + if key == "$ref" and isinstance(item, str) + else _rewrite_definition_refs(item) + ) + for key, item in value.items() + } + if isinstance(value, list): + return [_rewrite_definition_refs(item) for item in value] + return value def _set_model_examples(openapi_schema: dict[str, Any], model_name: str) -> None: diff --git a/packages/sie_server/src/sie_server/types/openapi.py b/packages/sie_server/src/sie_server/types/openapi.py index a265586ca..af51fd5ca 100644 --- a/packages/sie_server/src/sie_server/types/openapi.py +++ b/packages/sie_server/src/sie_server/types/openapi.py @@ -20,6 +20,21 @@ class ImageInputModel(BaseModel): format: str | None = Field(default=None, description="Image format hint: 'jpeg', 'png', etc.") +class AudioInputModel(BaseModel): + """Audio input for audio models.""" + + data: bytes = Field(..., description="Audio data as bytes") + format: str | None = Field(default=None, description="Audio format hint: 'wav', 'mp3', etc.") + sample_rate: int | None = Field(default=None, description="Audio sample rate in Hz") + + +class VideoInputModel(BaseModel): + """Video input for video models.""" + + data: bytes = Field(..., description="Video data as bytes") + format: str | None = Field(default=None, description="Video format hint: 'mp4', 'webm', etc.") + + class DocumentInputModel(BaseModel): """Document input for composite-document extractors (PDF, DOCX, HTML, ...).""" @@ -33,6 +48,8 @@ class ItemModel(BaseModel): id: str | None = Field(default=None, description="Optional identifier for this item. Returned in response.") text: str | None = Field(default=None, description="Text content to encode", examples=["Hello, world!"]) images: list[ImageInputModel] | None = Field(default=None, description="Images for multimodal models") + audio: AudioInputModel | None = Field(default=None, description="Audio for audio-capable models") + video: VideoInputModel | None = Field(default=None, description="Video for video-capable models") document: DocumentInputModel | None = Field( default=None, description="Document for composite-document extractors (PDF, DOCX, HTML, ...)" ) diff --git a/packages/sie_server/tests/test_openapi_export.py b/packages/sie_server/tests/test_openapi_export.py index 53d4bc032..666d9ad9f 100644 --- a/packages/sie_server/tests/test_openapi_export.py +++ b/packages/sie_server/tests/test_openapi_export.py @@ -34,6 +34,16 @@ def test_openapi_has_request_body_schemas() -> None: assert name in schemas, f"Missing schema: {name}" +def test_openapi_item_documents_audio_and_video() -> None: + """The published Item schema includes every native media field.""" + result = runner.invoke(app, ["openapi"]) + spec = json.loads(result.output) + item_properties = spec["components"]["schemas"]["ItemModel"]["properties"] + + assert item_properties["audio"]["anyOf"][0]["$ref"] == "#/components/schemas/AudioInputModel" + assert item_properties["video"]["anyOf"][0]["$ref"] == "#/components/schemas/VideoInputModel" + + def test_openapi_output_file(tmp_path: Path) -> None: """CLI writes spec to a file when --output is given.""" out = tmp_path / "spec.json" From c869c119b4b18d28bc9e7562d0270e98cf934d2c Mon Sep 17 00:00:00 2001 From: svonava Date: Sun, 19 Jul 2026 03:03:44 -0700 Subject: [PATCH 2/3] fix(sdk): add TypeScript media input parity --- packages/sie_ts_sdk/src/client.ts | 45 +++++--- packages/sie_ts_sdk/src/index.ts | 11 ++ packages/sie_ts_sdk/src/internal/parsing.ts | 7 +- packages/sie_ts_sdk/src/media.ts | 109 ++++++++++++++++++++ packages/sie_ts_sdk/src/types.ts | 17 +++ packages/sie_ts_sdk/tests/client.test.ts | 97 +++++++++++++++++ packages/sie_ts_sdk/tests/media.test.ts | 50 +++++++++ packages/sie_ts_sdk/tests/types.test.ts | 17 +++ 8 files changed, 336 insertions(+), 17 deletions(-) create mode 100644 packages/sie_ts_sdk/src/media.ts create mode 100644 packages/sie_ts_sdk/tests/media.test.ts diff --git a/packages/sie_ts_sdk/src/client.ts b/packages/sie_ts_sdk/src/client.ts index f6881b8d1..38f8c71ce 100644 --- a/packages/sie_ts_sdk/src/client.ts +++ b/packages/sie_ts_sdk/src/client.ts @@ -89,6 +89,8 @@ import { decodeChunkBytes, jobChunks, } from "./jobs.js"; +import { toAudioWireFormat, toVideoWireFormat } from "./media.js"; +import type { AudioWireFormat, VideoWireFormat } from "./media.js"; import { packMessage, unpackMessage } from "./msgpack.js"; import { parseSseStream } from "./sse.js"; import type { @@ -232,8 +234,11 @@ function abortableSleep(ms: number, signal: AbortSignal): Promise { const _LEASE_RENEWAL_MAX_RETRIES = 5; -type ItemWithWireImages = Omit & { images?: ImageWireFormat[] }; -type ItemForWire = Item | ItemWithWireImages; +type ItemForWire = Omit & { + images?: ImageWireFormat[]; + audio?: AudioWireFormat; + video?: VideoWireFormat; +}; function isImageWireFormat(image: ImageInput | ImageWireFormat): image is ImageWireFormat { return typeof image === "object" && image !== null && "data" in image; @@ -246,15 +251,23 @@ async function imageForWire(image: ImageInput | ImageWireFormat): Promise { - if (!item.images || item.images.length === 0) { - return item; +async function itemForWire(item: Item): Promise { + const { images, audio, video, ...rest } = item; + const result: ItemForWire = rest; + if (images && images.length > 0) { + result.images = await Promise.all(images.map(imageForWire)); } - return { ...item, images: await Promise.all(item.images.map(imageForWire)) }; + if (audio !== undefined) { + result.audio = await toAudioWireFormat(audio); + } + if (video !== undefined) { + result.video = await toVideoWireFormat(video); + } + return result; } -async function itemsImagesForWire(items: Item[]): Promise { - return Promise.all(items.map(itemImagesForWire)); +async function itemsForWire(items: Item[]): Promise { + return Promise.all(items.map(itemForWire)); } /** @@ -435,12 +448,12 @@ export class SIEClient { ): Promise { const isSingleItem = !Array.isArray(items); const itemsArray = isSingleItem ? [items] : items; - const itemsForWire = await itemsImagesForWire(itemsArray); + const serializedItems = await itemsForWire(itemsArray); // Build request body - model is in URL path, not body // Wire format uses snake_case const body: Record = { - items: itemsForWire, + items: serializedItems, }; // Add params if any are specified @@ -1216,13 +1229,13 @@ export class SIEClient { items: Item[], options: ScoreOptions = {}, ): Promise { - const queryForWire = await itemImagesForWire(query); - const itemsForWire = await itemsImagesForWire(items); + const serializedQuery = await itemForWire(query); + const serializedItems = await itemsForWire(items); // Build request body const body: Record = { - query: queryForWire, - items: itemsForWire, + query: serializedQuery, + items: serializedItems, }; const waitForCapacity = options.waitForCapacity ?? this.defaultWaitForCapacity; @@ -1289,11 +1302,11 @@ export class SIEClient { ): Promise { const isSingleItem = !Array.isArray(items); const itemsArray = isSingleItem ? [items] : items; - const itemsForWire = await itemsImagesForWire(itemsArray); + const serializedItems = await itemsForWire(itemsArray); // Build request body const body: Record = { - items: itemsForWire, + items: serializedItems, }; // Add params diff --git a/packages/sie_ts_sdk/src/index.ts b/packages/sie_ts_sdk/src/index.ts index dde3081e1..dc3801a2c 100644 --- a/packages/sie_ts_sdk/src/index.ts +++ b/packages/sie_ts_sdk/src/index.ts @@ -52,6 +52,7 @@ export { SDK_VERSION } from "./version.js"; export type { // Core types Item, + DocumentInput, SparseResult, TimingInfo, EncodeResult, @@ -167,3 +168,13 @@ export { type ImageInput, type ImageWireFormat, } from "./images.js"; +export { + toAudioWireFormat, + toMediaBytes, + toVideoWireFormat, + type AudioInput, + type AudioWireFormat, + type MediaInput, + type VideoInput, + type VideoWireFormat, +} from "./media.js"; diff --git a/packages/sie_ts_sdk/src/internal/parsing.ts b/packages/sie_ts_sdk/src/internal/parsing.ts index 62e4479dd..0cf4f9ad5 100644 --- a/packages/sie_ts_sdk/src/internal/parsing.ts +++ b/packages/sie_ts_sdk/src/internal/parsing.ts @@ -308,6 +308,7 @@ interface WireExtractResult { relations?: WireRelation[]; classifications?: WireClassification[]; objects?: WireDetectedObject[]; + data?: Record; } /** @@ -407,7 +408,7 @@ function parseEntity(data: WireEntity): Entity { * Parse wire format to ExtractResult */ export function parseExtractResult(data: WireExtractResult): ExtractResult { - return { + const result: ExtractResult = { id: data.id, entities: data.entities.map(parseEntity), relations: (data.relations ?? []).map( @@ -432,6 +433,10 @@ export function parseExtractResult(data: WireExtractResult): ExtractResult { }), ), }; + if (data.data !== undefined) { + result.data = data.data; + } + return result; } /** diff --git a/packages/sie_ts_sdk/src/media.ts b/packages/sie_ts_sdk/src/media.ts new file mode 100644 index 000000000..e420faccc --- /dev/null +++ b/packages/sie_ts_sdk/src/media.ts @@ -0,0 +1,109 @@ +/** + * Audio and video conversion utilities for the SIE TypeScript SDK. + * + * Media is transported as raw bytes inside msgpack with optional format + * metadata. Inputs work in both Node.js (`Uint8Array` / `Buffer`) and browsers + * (`ArrayBuffer` / `Blob` / `File`). + */ + +/** Binary inputs accepted directly by audio and video Item fields. */ +export type MediaInput = Uint8Array | ArrayBuffer | Blob | string; + +/** User-facing audio input with optional wire metadata. */ +export interface AudioInput { + data: MediaInput; + format?: string; + sampleRate?: number; +} + +/** User-facing video input with an optional format hint. */ +export interface VideoInput { + data: MediaInput; + format?: string; +} + +/** Audio shape serialized onto the generic Item wire contract. */ +export interface AudioWireFormat { + data: Uint8Array; + format?: string; + sample_rate?: number; +} + +/** Video shape serialized onto the generic Item wire contract. */ +export interface VideoWireFormat { + data: Uint8Array; + format?: string; +} + +/** Convert bytes, browser binary objects, or base64 strings to bytes. */ +export async function toMediaBytes(input: MediaInput): Promise { + if (input instanceof Uint8Array) { + return input; + } + + if (input instanceof ArrayBuffer) { + return new Uint8Array(input); + } + + if (typeof Blob !== "undefined" && input instanceof Blob) { + return new Uint8Array(await input.arrayBuffer()); + } + + if (typeof input === "string") { + const dataUrlMatch = input.match(/^data:[^;]+;base64,(.+)$/); + return base64ToBytes(dataUrlMatch?.[1] ?? input); + } + + throw new Error(`Unsupported media input type: ${typeof input}`); +} + +/** Convert a direct or metadata-wrapped audio input to its msgpack wire shape. */ +export async function toAudioWireFormat( + input: MediaInput | AudioInput | AudioWireFormat, +): Promise { + const wrapped = isWrappedMedia(input) ? input : { data: input }; + const result: AudioWireFormat = { data: await toMediaBytes(wrapped.data) }; + if (wrapped.format !== undefined) { + result.format = wrapped.format; + } + const sampleRate = + "sampleRate" in wrapped + ? wrapped.sampleRate + : "sample_rate" in wrapped + ? wrapped.sample_rate + : undefined; + if (sampleRate !== undefined) { + result.sample_rate = sampleRate; + } + return result; +} + +/** Convert a direct or metadata-wrapped video input to its msgpack wire shape. */ +export async function toVideoWireFormat( + input: MediaInput | VideoInput | VideoWireFormat, +): Promise { + const wrapped = isWrappedMedia(input) ? input : { data: input }; + const result: VideoWireFormat = { data: await toMediaBytes(wrapped.data) }; + if (wrapped.format !== undefined) { + result.format = wrapped.format; + } + return result; +} + +function isWrappedMedia( + input: MediaInput | AudioInput | AudioWireFormat | VideoInput | VideoWireFormat, +): input is AudioInput | AudioWireFormat | VideoInput | VideoWireFormat { + return typeof input === "object" && input !== null && "data" in input; +} + +function base64ToBytes(base64: string): Uint8Array { + if (typeof atob === "function") { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; + } + return new Uint8Array(Buffer.from(base64, "base64")); +} diff --git a/packages/sie_ts_sdk/src/types.ts b/packages/sie_ts_sdk/src/types.ts index ebcd507b8..94ea01116 100644 --- a/packages/sie_ts_sdk/src/types.ts +++ b/packages/sie_ts_sdk/src/types.ts @@ -6,6 +6,13 @@ */ import type { ImageInput, ImageWireFormat } from "./images.js"; +import type { + AudioInput, + AudioWireFormat, + MediaInput, + VideoInput, + VideoWireFormat, +} from "./media.js"; /** * Output dtype options for quantized embeddings. @@ -50,6 +57,10 @@ export interface DocumentInput { * // With a document for composite-document extractors (Docling, ...) * { document: { data: pdfBytes, format: "pdf" } } * + * // With audio or video for media-capable models + * { audio: { data: wavBytes, format: "wav", sampleRate: 16000 } } + * { video: { data: mp4Bytes, format: "mp4" } } + * * // Pre-encoded multivector (for use with maxsim utility) * { multivector: [tokenEmbedding1, tokenEmbedding2, ...] } */ @@ -60,6 +71,10 @@ export interface Item { text?: string; /** Images for multimodal models; converted to wire format by the client */ images?: (ImageInput | ImageWireFormat)[]; + /** Audio for audio-capable models; converted to wire format by the client */ + audio?: MediaInput | AudioInput | AudioWireFormat; + /** Video for video-capable models; converted to wire format by the client */ + video?: MediaInput | VideoInput | VideoWireFormat; /** Document for composite-document extractors (PDF, DOCX, HTML, ...) */ document?: DocumentInput; /** Pre-encoded multivector (for use with maxsim utility) */ @@ -261,6 +276,8 @@ export interface ExtractResult { classifications: Classification[]; /** List of detected objects */ objects: DetectedObject[]; + /** Additional structured extractor output (for example Docling document data) */ + data?: Record; } /** diff --git a/packages/sie_ts_sdk/tests/client.test.ts b/packages/sie_ts_sdk/tests/client.test.ts index 99a2788d2..fd418f0c6 100644 --- a/packages/sie_ts_sdk/tests/client.test.ts +++ b/packages/sie_ts_sdk/tests/client.test.ts @@ -1156,6 +1156,41 @@ describe("Real-world usage patterns", () => { await client.close(); }); + + it("should convert audio and video to wire format before encode serialization", async () => { + const client = new SIEClient("http://localhost:8080"); + const audioBytes = new Uint8Array([1, 2, 3]); + const videoBytes = new Uint8Array([4, 5, 6]); + + mockFetch.mockResolvedValueOnce( + createMsgpackResponse({ + items: [{ dense: { values: new Float32Array([0.1, 0.2]) } }], + }), + ); + + await client.encode("media-encoder", { + audio: { data: audioBytes, format: "pcm", sampleRate: 16_000 }, + video: { data: videoBytes, format: "mp4" }, + }); + + const fetchCall = mockFetch.mock.calls[0]; + const body = fetchCall?.[1]?.body as Uint8Array; + const parsed = unpackMessage<{ + items?: { + audio?: { data: Uint8Array; format?: string; sample_rate?: number }; + video?: { data: Uint8Array; format?: string }; + }[]; + }>(body); + + expect(parsed.items?.[0]?.audio).toEqual({ + data: audioBytes, + format: "pcm", + sample_rate: 16_000, + }); + expect(parsed.items?.[0]?.video).toEqual({ data: videoBytes, format: "mp4" }); + + await client.close(); + }); }); describe("SIEClient.score() - reranking", () => { @@ -1271,6 +1306,32 @@ describe("SIEClient.score() - reranking", () => { expect(parsed.items?.[0]?.images?.[0]?.data).toEqual(itemImage); expect(parsed.items?.[0]?.images?.[0]?.format).toBe("jpeg"); }); + + it("should convert media query and items before score serialization", async () => { + const queryAudio = new Uint8Array([1, 2]); + const itemVideo = new Uint8Array([3, 4]); + + mockFetch.mockResolvedValueOnce( + createMsgpackResponse({ + model: "media-reranker", + scores: [{ item_id: "clip-1", score: 0.9, rank: 0 }], + }), + ); + + await client.score("media-reranker", { audio: { data: queryAudio, format: "wav" } }, [ + { id: "clip-1", video: { data: itemVideo, format: "webm" } }, + ]); + + const fetchCall = mockFetch.mock.calls[0]; + const body = fetchCall?.[1]?.body as Uint8Array; + const parsed = unpackMessage<{ + query?: { audio?: { data: Uint8Array; format?: string } }; + items?: { video?: { data: Uint8Array; format?: string } }[]; + }>(body); + + expect(parsed.query?.audio).toEqual({ data: queryAudio, format: "wav" }); + expect(parsed.items?.[0]?.video).toEqual({ data: itemVideo, format: "webm" }); + }); }); describe("SIEClient.extract() - NER", () => { @@ -1402,6 +1463,42 @@ describe("SIEClient.extract() - NER", () => { expect(parsed.items?.[0]?.images?.[0]?.format).toBe("jpeg"); }); + it("should convert audio to wire format and preserve structured extract data", async () => { + const audioBytes = new Uint8Array([1, 2, 3]); + + mockFetch.mockResolvedValueOnce( + createMsgpackResponse({ + items: [ + { + entities: [], + data: { text: "Transcribed speech", language: "en" }, + }, + ], + }), + ); + + const result = await client.extract( + "openai/whisper-base", + { audio: { data: audioBytes, format: "wav", sampleRate: 16_000 } }, + { labels: [] }, + ); + + const fetchCall = mockFetch.mock.calls[0]; + const body = fetchCall?.[1]?.body as Uint8Array; + const parsed = unpackMessage<{ + items?: { + audio?: { data: Uint8Array; format?: string; sample_rate?: number }; + }[]; + }>(body); + + expect(parsed.items?.[0]?.audio).toEqual({ + data: audioBytes, + format: "wav", + sample_rate: 16_000, + }); + expect(result.data).toEqual({ text: "Transcribed speech", language: "en" }); + }); + it("should pass threshold option in params", async () => { mockFetch.mockResolvedValueOnce( createMsgpackResponse({ diff --git a/packages/sie_ts_sdk/tests/media.test.ts b/packages/sie_ts_sdk/tests/media.test.ts new file mode 100644 index 000000000..2c6a10dca --- /dev/null +++ b/packages/sie_ts_sdk/tests/media.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { toAudioWireFormat, toMediaBytes, toVideoWireFormat } from "../src/index.js"; + +describe("media conversion", () => { + it("passes Uint8Array inputs through", async () => { + const bytes = new Uint8Array([1, 2, 3]); + + expect(await toMediaBytes(bytes)).toBe(bytes); + }); + + it("reads browser Blob inputs", async () => { + const blob = new Blob([new Uint8Array([4, 5, 6])], { type: "audio/wav" }); + + expect(await toMediaBytes(blob)).toEqual(new Uint8Array([4, 5, 6])); + }); + + it("decodes base64 data URLs", async () => { + expect(await toMediaBytes("data:audio/wav;base64,AQID")).toEqual(new Uint8Array([1, 2, 3])); + }); + + it("preserves audio format and converts sampleRate to the wire key", async () => { + const result = await toAudioWireFormat({ + data: new Uint8Array([1, 2]), + format: "pcm", + sampleRate: 16_000, + }); + + expect(result).toEqual({ + data: new Uint8Array([1, 2]), + format: "pcm", + sample_rate: 16_000, + }); + }); + + it("accepts an already wire-shaped audio input", async () => { + const result = await toAudioWireFormat({ + data: new Uint8Array([1, 2]), + format: "wav", + sample_rate: 48_000, + }); + + expect(result.sample_rate).toBe(48_000); + }); + + it("wraps direct video bytes without inventing a format", async () => { + const result = await toVideoWireFormat(new Uint8Array([9, 8, 7])); + + expect(result).toEqual({ data: new Uint8Array([9, 8, 7]) }); + }); +}); diff --git a/packages/sie_ts_sdk/tests/types.test.ts b/packages/sie_ts_sdk/tests/types.test.ts index ee1ca9c16..c26485e87 100644 --- a/packages/sie_ts_sdk/tests/types.test.ts +++ b/packages/sie_ts_sdk/tests/types.test.ts @@ -10,6 +10,7 @@ */ import { describe, expect, it } from "vitest"; +import type { AudioInput, VideoInput } from "../src/media.js"; import type { DocumentInput, EncodeResult, @@ -103,6 +104,22 @@ describe("Item creation - common user patterns", () => { expect(item.document?.format).toBeUndefined(); }); + + it("creates audio and video items with generic binary payloads", () => { + const audio: AudioInput = { + data: new Uint8Array([1, 2, 3]), + format: "wav", + sampleRate: 16_000, + }; + const video: VideoInput = { + data: new Uint8Array([4, 5, 6]), + format: "mp4", + }; + const item: Item = { audio, video }; + + expect("sampleRate" in (item.audio ?? {})).toBe(true); + expect("format" in (item.video ?? {})).toBe(true); + }); }); describe("EncodeResult - working with embeddings", () => { From 39277c589303fea721fb0e838b04338012fcf38c Mon Sep 17 00:00:00 2001 From: svonava Date: Sun, 19 Jul 2026 03:08:45 -0700 Subject: [PATCH 3/3] fix(sdk): align TypeScript extract options --- packages/sie_ts_sdk/src/client.ts | 23 ++++++---- packages/sie_ts_sdk/src/types.ts | 6 ++- packages/sie_ts_sdk/tests/client.test.ts | 53 ++++++++++++++++++++++++ packages/sie_ts_sdk/tests/types.test.ts | 15 +++++++ 4 files changed, 89 insertions(+), 8 deletions(-) diff --git a/packages/sie_ts_sdk/src/client.ts b/packages/sie_ts_sdk/src/client.ts index 38f8c71ce..1080725a7 100644 --- a/packages/sie_ts_sdk/src/client.ts +++ b/packages/sie_ts_sdk/src/client.ts @@ -1264,7 +1264,7 @@ export class SIEClient { * @param options - Extract options with labels * @returns Extract result with entities */ - async extract(model: string, item: Item, options: ExtractOptions): Promise; + async extract(model: string, item: Item, options?: ExtractOptions): Promise; /** * Extract entities from multiple items. @@ -1274,7 +1274,7 @@ export class SIEClient { * @param options - Extract options with labels * @returns Array of extract results in same order as input */ - async extract(model: string, items: Item[], options: ExtractOptions): Promise; + async extract(model: string, items: Item[], options?: ExtractOptions): Promise; /** * Extract entities from one or more items. @@ -1298,7 +1298,7 @@ export class SIEClient { async extract( model: string, items: Item | Item[], - options: ExtractOptions, + options: ExtractOptions = {}, ): Promise { const isSingleItem = !Array.isArray(items); const itemsArray = isSingleItem ? [items] : items; @@ -1310,16 +1310,25 @@ export class SIEClient { }; // Add params - const params: Record = { - labels: options.labels, - }; + const params: Record = {}; + if (options.labels !== undefined) { + params.labels = options.labels; + } + if (options.outputSchema !== undefined) { + params.output_schema = options.outputSchema; + } + if (options.instruction !== undefined) { + params.instruction = options.instruction; + } if (options.threshold !== undefined) { params.threshold = options.threshold; } if (options.adapterOptions !== undefined) { params.options = options.adapterOptions; } - body.params = params; + if (Object.keys(params).length > 0) { + body.params = params; + } const waitForCapacity = options.waitForCapacity ?? this.defaultWaitForCapacity; const { pool, gpu } = this.parseGpuParam(options.gpu); diff --git a/packages/sie_ts_sdk/src/types.ts b/packages/sie_ts_sdk/src/types.ts index 94ea01116..eaaa78836 100644 --- a/packages/sie_ts_sdk/src/types.ts +++ b/packages/sie_ts_sdk/src/types.ts @@ -1009,7 +1009,11 @@ export interface GenerateChunk { */ export interface ExtractOptions { /** Entity labels to extract (e.g., ["person", "organization"]) */ - labels: string[]; + labels?: string[]; + /** JSON schema for structured extraction output */ + outputSchema?: Record; + /** Optional extraction instruction for instruction-aware adapters */ + instruction?: string; /** Minimum confidence threshold (0-1) */ threshold?: number; /** GPU type for this request */ diff --git a/packages/sie_ts_sdk/tests/client.test.ts b/packages/sie_ts_sdk/tests/client.test.ts index fd418f0c6..742781e16 100644 --- a/packages/sie_ts_sdk/tests/client.test.ts +++ b/packages/sie_ts_sdk/tests/client.test.ts @@ -1442,6 +1442,59 @@ describe("SIEClient.extract() - NER", () => { expect(parsed.params?.labels).toEqual(["person", "organization", "location"]); }); + it("should allow extraction without labels or params", async () => { + mockFetch.mockResolvedValueOnce( + createMsgpackResponse({ + items: [{ entities: [], data: { markdown: "# Invoice" } }], + }), + ); + + const result = await client.extract("docling", { + document: { data: new Uint8Array([1, 2]), format: "pdf" }, + }); + + const fetchCall = mockFetch.mock.calls[0]; + const body = fetchCall?.[1]?.body as Uint8Array; + const parsed = unpackMessage<{ params?: Record }>(body); + + expect(parsed.params).toBeUndefined(); + expect(result.data).toEqual({ markdown: "# Invoice" }); + }); + + it("should serialize outputSchema and instruction using wire parameter names", async () => { + const outputSchema = { + type: "object", + properties: { total: { type: "string" } }, + required: ["total"], + }; + mockFetch.mockResolvedValueOnce( + createMsgpackResponse({ + items: [{ entities: [], data: { total: "$355.00" } }], + }), + ); + + await client.extract( + "document-fields", + { text: "Total due: $355.00" }, + { + outputSchema, + instruction: "Extract the requested invoice fields.", + }, + ); + + const fetchCall = mockFetch.mock.calls[0]; + const body = fetchCall?.[1]?.body as Uint8Array; + const parsed = unpackMessage<{ + params?: { + output_schema?: Record; + instruction?: string; + }; + }>(body); + + expect(parsed.params?.output_schema).toEqual(outputSchema); + expect(parsed.params?.instruction).toBe("Extract the requested invoice fields."); + }); + it("should convert images to wire format before extract serialization", async () => { const imageBytes = new Uint8Array([0xff, 0xd8, 0xff, 0xe0]); diff --git a/packages/sie_ts_sdk/tests/types.test.ts b/packages/sie_ts_sdk/tests/types.test.ts index c26485e87..5eefa38ea 100644 --- a/packages/sie_ts_sdk/tests/types.test.ts +++ b/packages/sie_ts_sdk/tests/types.test.ts @@ -15,6 +15,7 @@ import type { DocumentInput, EncodeResult, Entity, + ExtractOptions, ExtractResult, Item, ModelCapabilities, @@ -299,6 +300,20 @@ describe("ScoreResult - reranking results", () => { }); describe("ExtractResult - NER results", () => { + it("supports optional labels plus structured output and instruction options", () => { + const options: ExtractOptions = { + instruction: "Extract invoice fields.", + outputSchema: { + type: "object", + properties: { total: { type: "string" } }, + }, + }; + + expect(options.labels).toBeUndefined(); + expect(options.instruction).toBe("Extract invoice fields."); + expect(options.outputSchema?.type).toBe("object"); + }); + it("provides extracted entities with positions", () => { // User scenario: "I want to highlight entities in my UI" const result: ExtractResult = {