diff --git a/.agents/skills/celeste-python/SKILL.md b/.agents/skills/celeste-python/SKILL.md new file mode 100644 index 00000000..85e0fa39 --- /dev/null +++ b/.agents/skills/celeste-python/SKILL.md @@ -0,0 +1,100 @@ +--- +name: celeste-python +description: Use whenever writing, modifying, reviewing, or debugging code involving Celeste, celeste-ai, celeste-python, import celeste, src/celeste, or withceleste app integrations. This includes providers, modalities, models, artifacts, MIME types, parameters, tools, multimodal messages, streaming, structured outputs, protocol/base URL support, OpenSpec changes, and tests. Always use this skill before inventing Celeste types, registries, model catalogs, provider abstractions, request/response shapes, or syntax. +compatibility: Local celeste-python repository skill; no network required. +--- + +# Celeste Python Coding + +Use this skill to stay aligned with the actual Celeste SDK in this repository. +Celeste is evolving, so current source and tests are more reliable than model memory. + +## First Step + +Read `references/repo-map.md` before making Celeste-related code changes or review claims. +It gives the current public API, internal layering, extension seams, templates, and canonical tests. + +Then read only the references relevant to the task: + +- App-side integration: `references/public-api.md` +- SDK-internal provider, modality, model, parameter, protocol, streaming, or tool work: `references/sdk-architecture.md` +- Review/debugging or suspicious Celeste code: `references/anti-patterns.md` +- Test selection or final checks: `references/verification.md` + +## Source-Of-Truth Order + +When sources disagree, follow this order: + +1. Current source code in `src/celeste/` +2. Current tests and templates for existing behavior +3. README examples and public exports +4. Active OpenSpec artifacts for intended new behavior and acceptance criteria +5. Notes such as `common_agent_mistakes.md` +6. Model memory + +Treat `common_agent_mistakes.md` as advisory. Verify every warning against current code and the task context. + +## Route The Task + +Classify the task before coding: + +- App integration: code outside the SDK consuming Celeste. Prefer public namespaces and public exports. Keep the boundary thin. +- SDK internals: changes inside `src/celeste`, `templates`, or tests. Follow existing modality, provider, protocol, model, mapper, and streaming patterns. +- Review/debugging: identify whether issues are app-side duplication, SDK pattern drift, unsupported model/parameter assumptions, or test gaps. + +## App Integration Rules + +Use the public API first: + +- `celeste.text.*` +- `celeste.images.*` +- `celeste.audio.*` +- `celeste.videos.*` +- `celeste.documents.*` + +Use `create_client(...)` when explicit client reuse or explicit `modality`, `operation`, `provider`, `protocol`, `base_url`, or auth configuration is needed. + +Do not create app-local duplicates for Celeste-owned concepts unless the user explicitly asks for a temporary compatibility layer. Import and use Celeste types for roles, providers, modalities, operations, artifacts, MIME types, tools, and model discovery. + +## SDK Internal Rules + +Do not flatten Celeste into a single invented abstraction. The SDK intentionally separates: + +- provider auth registration +- modality provider maps +- provider API mixins +- modality-specific provider clients +- per-modality model aggregation +- parameter enums and `ParameterMapper`s +- constraints and optional input type inference +- protocol clients for compatible APIs + +Start from nearby existing providers/modalities and templates. Use focused tests as executable documentation. + +## Common Failure Modes + +Before adding new local code, check whether Celeste already owns the concept. +Watch for: + +- raw provider, role, MIME, modality, operation, or input-type strings where Celeste exposes enums +- app-side model allowlists where `list_models(...)` or the host app's existing Celeste discovery flow should be used +- runtime registry patching from app code +- provider SDK request shapes copied directly into app code +- new parameter names without modality enums, `TypedDict` fields, model constraints, and provider mappers +- provider registration changes that update one seam but miss auth, modality maps, model aggregation, or tests + +## Verification + +Pick focused tests from `references/verification.md` based on the seam touched. +For broader read-only checks, use the repo commands: + +- `make lint` +- `make typecheck` +- `make test` + +For finalization commands that may rewrite files, use them only when mutation is intended: + +- `make format` +- `make ci` (runs lint-fix and format internally) + +If you cannot run a relevant command, say so and explain the remaining risk. diff --git a/.agents/skills/celeste-python/evals/evals.json b/.agents/skills/celeste-python/evals/evals.json new file mode 100644 index 00000000..e2f3242d --- /dev/null +++ b/.agents/skills/celeste-python/evals/evals.json @@ -0,0 +1,93 @@ +{ + "skill_name": "celeste-python", + "evals": [ + { + "id": 1, + "prompt": "Integrate Celeste into an app endpoint that generates text, analyzes uploaded images, and lists available text generation models. Keep the integration thin and type-safe.", + "expected_output": "Uses namespace-first public APIs or create_client only where explicit reuse is needed; imports Celeste public types; uses list_models for discovery; avoids local provider/model/role/MIME registries.", + "files": [], + "expectations": [ + "Uses celeste.text or celeste.images namespace APIs for app-side operations unless create_client is justified.", + "Uses list_models with Modality and Operation instead of a hard-coded app-side model catalog.", + "Uses Celeste artifacts, MIME enums, Message, Role, Provider, Modality, and Operation where those concepts appear.", + "Does not patch or mutate Celeste internals from app code." + ] + }, + { + "id": 2, + "prompt": "Add support for a new hypothetical text provider named AcmeAI that exposes an OpenAI-compatible chat completions API. Describe or implement the files and tests needed in celeste-python.", + "expected_output": "Follows SDK seams: Provider enum, provider auth registration, provider API or protocol reuse, modality provider client/map, model definitions and aggregation, parameter mappers where needed, and focused tests.", + "files": [], + "expectations": [ + "Identifies Provider enum registration in src/celeste/core.py.", + "Identifies provider auth registration under src/celeste/providers/acmeai/__init__.py.", + "Uses a concrete provider API package or the chat completions protocol seam instead of inventing a flat request stack.", + "Updates modality text provider registration and model aggregation.", + "Defines models with operations and parameter_constraints.", + "Selects focused tests such as test_provider_api_templates.py, test_init.py, test_models.py, and parameter/client tests as relevant." + ] + }, + { + "id": 3, + "prompt": "Review this Celeste integration: it defines local Provider and Role enums as strings, keeps a hard-coded BACKEND_MODELS list, maps file extensions to raw MIME strings, and registers unknown models at runtime before calling Celeste. Identify what is wrong and how to fix it.", + "expected_output": "Flags app-side duplication and registry mutation, recommends Celeste public enums/types/model discovery/MIME enums, and scopes warnings so SDK-internal registry work is not incorrectly banned.", + "files": [], + "expectations": [ + "Flags local Provider and Role enums as duplicates of Celeste-owned concepts.", + "Recommends list_models or existing host discovery instead of a hard-coded model catalog.", + "Recommends Celeste MIME enums and artifact types instead of raw MIME string maps where applicable.", + "Rejects app-side runtime model registry patching while noting SDK-internal model definitions are legitimate.", + "Grounds review claims in current Celeste source or skill references rather than memory alone." + ] + }, + { + "id": 4, + "prompt": "A chat app wants to send an uploaded image and user text to a Celeste text model with native multimodal input. Show the Celeste message shape it should build.", + "expected_output": "Builds Message(content=[TextPart(...), ImagePart(image=ImageArtifact(...))]) or equivalent ordered MessagePart list; does not pass raw ImageArtifact in content; does not define app-local ImagePart or MessageContent types.", + "files": [], + "expectations": [ + "Uses Celeste Message, Role, TextPart, ImagePart, and ImageArtifact.", + "Represents media through ordered MessagePart wrappers inside Message.content.", + "Does not pass raw artifacts or mixed raw lists as Message.content.", + "Does not create app-local duplicate multimodal message models." + ] + }, + { + "id": 5, + "prompt": "Implement a multi-turn text conversation that sends a previous tool result back to the model. The tool result contains structured JSON.", + "expected_output": "Uses ToolResult as a separate message-list item with structured content in ToolResult.content; does not subclass or wrap it as Message; keeps normal Message.content narrow.", + "files": [], + "expectations": [ + "Uses ToolResult directly in the text messages list.", + "Keeps structured tool payloads in ToolResult.content.", + "Does not treat ToolResult as a Message subclass.", + "Does not put structured Pydantic payloads in normal Message.content." + ] + }, + { + "id": 6, + "prompt": "Configure Celeste to call a custom OpenAI-compatible text endpoint using base_url and protocol. Explain what modalities this supports today.", + "expected_output": "Uses create_client with Modality.TEXT plus protocol/base_url; states protocol-based compatible client support is currently text-only; distinguishes root protocol mixins from text modality protocol clients.", + "files": [], + "expectations": [ + "Uses create_client with modality=Modality.TEXT and protocol/base_url.", + "States compatible protocol client support is currently text-only.", + "Does not imply protocol= works for images, audio, videos, documents, or embeddings.", + "Mentions text-specific protocol clients under src/celeste/modalities/text/protocols when discussing SDK internals." + ] + }, + { + "id": 7, + "prompt": "Review a provider/template change that added extra_headers support only in one provider client file. What else should be checked?", + "expected_output": "Checks templates, provider API mixins, parameter mappers/constraints if public, focused tests, and streaming/telemetry preservation when relevant.", + "files": [], + "expectations": [ + "Points to templates as source of truth for generated provider/module shape.", + "Checks provider API mixin and concrete modality provider seams.", + "Considers parameter mapper and model constraint updates if extra_headers is public configuration.", + "Selects focused tests from references/verification.md instead of guessing from memory.", + "Does not run mutating commands unless mutation is intended." + ] + } + ] +} diff --git a/.agents/skills/celeste-python/references/anti-patterns.md b/.agents/skills/celeste-python/references/anti-patterns.md new file mode 100644 index 00000000..f65b9467 --- /dev/null +++ b/.agents/skills/celeste-python/references/anti-patterns.md @@ -0,0 +1,104 @@ +# Celeste Anti-Patterns + +Use this for reviews, debugging, and suspicious Celeste integrations. + +## Scope The Rule + +Many warnings apply differently depending on context. + +App-side integration: + +- consume Celeste public APIs +- avoid local duplicates of Celeste concepts +- keep wrappers thin +- prefer `list_models(...)` and existing model discovery + +SDK-internal work: + +- may update Celeste registries and model definitions +- must follow existing provider/modality/protocol/mapper/test patterns +- should not be blocked by app-side "do not register models" warnings + +## Do Not Duplicate Celeste Concepts + +Avoid local copies of: + +- provider strings when `Provider` applies +- role strings when `Role` applies +- MIME strings when Celeste MIME enums apply +- artifact shapes when `ImageArtifact`, `AudioArtifact`, `VideoArtifact`, or `DocumentArtifact` apply +- message part shapes when `MessageContent`, `MessagePart`, `TextPart`, `ImagePart`, `AudioPart`, `VideoPart`, or `DocumentPart` apply +- model catalogs when `list_models(...)` or existing registry flow applies +- tool result/error wrappers when `ToolOutput` or `ToolError` applies + +Local types are fine for truly app-specific transport events or persistence shapes. + +## Do Not Bypass Message Semantics + +Celeste owns semantic chat message content. + +Avoid: + +- raw artifacts directly in `Message.content` +- mixed raw lists such as `["text", ImageArtifact(...)]` +- structured Pydantic payloads as normal `Message.content` +- treating `ToolResult` as a `Message` subclass +- hand-rolled provider message serializers in app code or new SDK paths + +Use ordered message parts for chat input. Use `ToolResult.content` for structured tool outputs. + +## Do Not Invent A Flat Provider Registry + +Celeste provider support is intentionally split: + +- provider auth registration +- provider enum registration +- provider API mixins +- modality provider maps +- modality-specific provider clients +- per-modality model aggregation +- per-provider model definitions +- parameter mappers and constraints + +A new single registry that bypasses these seams is probably wrong. +Directory presence alone is not support; every active provider must be wired through the relevant enum, auth, client, model, map, and test seams. + +## Do Not Copy Vendor SDK Shapes Into App Code + +Celeste exists to hide provider-specific request/response differences. In app code, avoid OpenAI/Anthropic/Gemini request structures unless the user explicitly asks for provider-specific fallback behavior outside Celeste. + +Inside the SDK, provider wire shapes belong in provider API mixins, protocol clients, provider tools, and provider parameter mappers. + +## Do Not Add Parameters In Only One Place + +New SDK parameters usually need several updates: + +- modality parameter enum +- modality `TypedDict` +- model `parameter_constraints` +- provider or protocol mapper +- tests + +If only a request-body field was added, the change is probably incomplete. + +## Do Not Treat Notes As Canonical + +`common_agent_mistakes.md` is useful but not perfect. Verify every warning against: + +1. current source +2. current tests/templates +3. public exports and README + +Example: app code should not patch Celeste's model registry, but SDK code may legitimately update model definitions or registry behavior. + +## Review Checklist + +Ask: + +- Is this app code or SDK code? +- Is the code using public namespaces where app usage is enough? +- Did it invent local enums or string literals for Celeste-owned concepts? +- Did it duplicate model catalog logic? +- Did it update every provider/modality/model/mapper seam required by the change? +- Did it pick focused tests that match the touched seam? +- Is a warning based on current code, or only on stale memory/notes? diff --git a/.agents/skills/celeste-python/references/public-api.md b/.agents/skills/celeste-python/references/public-api.md new file mode 100644 index 00000000..81494b19 --- /dev/null +++ b/.agents/skills/celeste-python/references/public-api.md @@ -0,0 +1,184 @@ +# Celeste Public API Reference + +Use this for app-side integrations and examples. + +## Namespace-First Usage + +Normal app code should use public namespaces from `src/celeste/namespaces/domains.py`. + +```python +import celeste + +text_model = "..." # choose from celeste.list_models(modality=celeste.Modality.TEXT, ...) +image_model = "..." # choose from celeste.list_models(modality=celeste.Modality.IMAGES, ...) +audio_model = "..." # choose from celeste.list_models(modality=celeste.Modality.AUDIO, ...) +video_model = "..." # choose from celeste.list_models(modality=celeste.Modality.VIDEOS, ...) +video = "..." # use a video artifact or supported video input + +text = await celeste.text.generate("Explain quantum computing", model=text_model) +image = await celeste.images.generate("A mountain lake", model=image_model) +speech = await celeste.audio.speak("Welcome", model=audio_model) +summary = await celeste.videos.analyze(video, prompt="Summarize this clip", model=video_model) +embeddings = await celeste.text.embed(["lorem ipsum"], model=text_model) +``` + +Available namespace families include: + +- `celeste.text.generate` +- `celeste.text.embed` +- `celeste.images.generate` +- `celeste.images.edit` +- `celeste.images.analyze` +- `celeste.images.embed` +- `celeste.audio.speak` +- `celeste.audio.analyze` +- `celeste.audio.embed` +- `celeste.videos.generate` +- `celeste.videos.analyze` +- `celeste.videos.embed` +- `celeste.documents.analyze` + +Use sync and streaming namespaces when needed: + +- `celeste.text.sync.generate(...)` +- `celeste.text.stream.generate(...)` +- `celeste.images.sync.edit(...)` +- `celeste.audio.stream.speak(...)` + +Inspect `src/celeste/namespaces/domains.py` for exact signatures. + +## Explicit Client Usage + +Use `create_client(...)` for explicit client reuse or lower-level configuration. + +```python +from celeste import Modality, Operation, Provider, create_client + +model_id = "..." # choose from model discovery or current provider docs + +client = create_client( + modality=Modality.TEXT, + operation=Operation.GENERATE, + provider=Provider.OLLAMA, + model=model_id, +) +response = await client.generate("Extract user info") +``` + +Important details: + +- Prefer `modality` + `operation`; `capability` is deprecated compatibility. +- Use `protocol` and `base_url` for compatible text APIs. +- If `base_url` is provided without provider/protocol, Celeste defaults to the OpenResponses protocol path. +- Unknown provider models can create a fallback model only when provider and modality context make that intentional; check `src/celeste/__init__.py`. + +## Model Discovery + +Use Celeste model discovery instead of duplicating catalogs: + +```python +from celeste import Modality, Operation, Provider, list_models + +models = list_models( + provider=Provider.OPENAI, + modality=Modality.TEXT, + operation=Operation.GENERATE, +) +``` + +Relevant source: + +- `src/celeste/models.py` +- `src/celeste/__init__.py` +- `references/verification.md` for focused test selection + +## Public Types + +Use Celeste-owned types for Celeste concepts. Import paths matter. + +- Top-level `celeste`: `Capability`, `Provider`, `Protocol`, `Modality`, `Operation`, `Message`, `MessageContent`, `MessagePart`, `TextPart`, `ImagePart`, `AudioPart`, `VideoPart`, `DocumentPart`, `Role`, `Tool`, `ToolCall`, `WebSearch`, `XSearch`, `CodeExecution`, `ToolChoice`, `ToolResult`, `ToolResultContent`, `ToolOutput`, `ToolError`. +- `celeste.core`: `InputType`, `Domain`, `Parameter`, `UsageField`, and other core enums not exported from the root package. +- `celeste.artifacts`: `ImageArtifact`, `VideoArtifact`, `AudioArtifact`, `DocumentArtifact`. +- `celeste.mime_types`: `ImageMimeType`, `VideoMimeType`, `AudioMimeType`, `DocumentMimeType`, `ApplicationMimeType`. + +Do not replace these with app-local enums or raw strings unless the code is truly app-specific transport. + +## Artifacts And MIME Types + +Artifacts support `url`, `data`, and `path`. + +```python +from celeste.artifacts import ImageArtifact +from celeste.mime_types import ImageMimeType + +image = ImageArtifact(path="image.png", mime_type=ImageMimeType.PNG) +``` + +Relevant source: + +- `src/celeste/artifacts.py` +- `src/celeste/mime_types.py` +- `references/verification.md` for focused test selection + +## Messages And Tools + +Use `Message` and `Role` for conversation structures. `Message.content` is either plain text or an ordered `list[MessagePart]`. + +For native multimodal chat input, wrap media artifacts in message parts: + +```python +from celeste import ImagePart, Message, Role, TextPart +from celeste.artifacts import ImageArtifact + +message = Message( + role=Role.USER, + content=[ + TextPart(text="Describe this image"), + ImagePart(image=ImageArtifact(path="image.png")), + ], +) +``` + +Do not pass raw artifacts, mixed raw lists, or structured Pydantic payloads directly as `Message.content`. Structured tool payloads belong in `ToolResult.content`, and `ToolResult` is a separate message-list item, not a `Message` subclass. + +Use `tools=[WebSearch()]`, `tools=[XSearch()]`, or `tools=[CodeExecution()]` rather than deprecated boolean tool parameters. + +Tool choice uses `ToolChoice.AUTO`, `ToolChoice.REQUIRED`, or `ToolChoice.NONE`. + +Relevant source: + +- `src/celeste/types.py` +- `src/celeste/messages.py` +- `src/celeste/tools.py` +- `references/verification.md` for focused test selection + +## Structured Outputs + +Pass a Pydantic model class through `output_schema`. + +```python +from pydantic import BaseModel + +class User(BaseModel): + name: str + age: int + +model_id = "..." # choose from celeste.list_models(...) + +response = await celeste.text.generate( + "Extract user info: John is 30", + model=model_id, + output_schema=User, +) +user = response.content +``` + +Check provider-specific structured output behavior before changing SDK internals: + +- `src/celeste/structured_outputs.py` +- `src/celeste/protocols/openresponses/parameters.py` +- `src/celeste/protocols/chatcompletions/parameters.py` +- provider-specific parameter mapper files +- `references/verification.md` for focused test selection + +If notes such as `common_agent_mistakes.md` mention provider-specific structured-output constraints, verify the constraint against current source, tests, or provider behavior before enforcing it. Scope any warning to the affected provider and request path. diff --git a/.agents/skills/celeste-python/references/repo-map.md b/.agents/skills/celeste-python/references/repo-map.md new file mode 100644 index 00000000..ce403acb --- /dev/null +++ b/.agents/skills/celeste-python/references/repo-map.md @@ -0,0 +1,222 @@ +# Celeste Repo Map + +Use this map first when working with Celeste. It is an orientation layer, not a substitute for reading the current source before editing. + +## Public Surface + +Primary app-facing imports are exported from `src/celeste/__init__.py`. + +Use domain namespaces for normal app code: + +- `celeste.text.generate(...)` +- `celeste.text.embed(...)` +- `celeste.images.generate(...)` +- `celeste.images.edit(...)` +- `celeste.images.analyze(...)` +- `celeste.images.embed(...)` +- `celeste.audio.speak(...)` +- `celeste.audio.analyze(...)` +- `celeste.audio.embed(...)` +- `celeste.videos.generate(...)` +- `celeste.videos.analyze(...)` +- `celeste.videos.embed(...)` +- `celeste.documents.analyze(...)` + +Namespace implementations live in `src/celeste/namespaces/domains.py`. They route domain operations to modality clients through `create_client(...)`. + +Use `create_client(...)` when code needs explicit client reuse or lower-level control over: + +- `modality` +- `operation` +- `provider` +- `model` +- `protocol` +- `base_url` +- `api_key` or `auth` + +Model discovery lives in `src/celeste/models.py`: + +- `list_models(provider=..., modality=..., operation=...)` +- `get_model(model_id, provider=...)` +- `register_models(...)` for SDK registry work + +App integrations should normally consume model discovery rather than creating local model catalogs. + +## Core Concepts + +Core enums live in `src/celeste/core.py`: + +- `Provider`: backend vendor or local provider. +- `Protocol`: wire-compatible API format such as `openresponses` or `chatcompletions`. +- `Modality`: output/client family such as text, images, videos, audio, embeddings. +- `Operation`: action such as generate, edit, analyze, speak, transcribe, embed, upscale. +- `Domain`: resource the user works with, used by namespace routing. +- `InputType`: optional media input categories. +- `Parameter`: common parameter names shared across modalities. +- `UsageField`: normalized usage keys. + +I/O and content types: + +- `src/celeste/io.py`: `Input`, `Output`, `Chunk`, `Usage`, `FinishReason`, input-type inference. +- `src/celeste/types.py`: `Message`, `Role`, `MessageContent`, `MessagePart`, `TextPart`, `ImagePart`, `AudioPart`, `VideoPart`, `DocumentPart`, `TextContent`, `ToolResultContent`, media content aliases. +- `src/celeste/messages.py`: canonical chat-message helpers for request-message construction, content-part normalization, media detection, tool-result serialization, and provider part support checks. +- `src/celeste/artifacts.py`: `Artifact`, `ImageArtifact`, `VideoArtifact`, `AudioArtifact`, `DocumentArtifact`. +- `src/celeste/mime_types.py`: MIME `StrEnum`s such as `ImageMimeType.PNG`. +- `src/celeste/tools.py`: `Tool`, `WebSearch`, `XSearch`, `CodeExecution`, `ToolChoice`, `ToolResult`, `ToolOutput`, `ToolError`. + +Prefer these types over app-local strings or duplicate models. + +## Runtime Layering + +The core client layering is in `src/celeste/client.py`: + +```text +HTTPClient + ^ +APIMixin + ^ +ModalityClient + ^ +Concrete provider/modality client +``` + +`APIMixin` owns provider API behavior such as request transport, URL building, response parsing, and usage mapping. `ModalityClient` owns modality behavior such as `_predict`, `_stream`, parameter mapping, output construction, metadata, warnings for unsupported parameters, and common error handling. + +Modality clients live under: + +- `src/celeste/modalities/text/client.py` +- `src/celeste/modalities/images/client.py` +- `src/celeste/modalities/audio/client.py` +- `src/celeste/modalities/videos/client.py` +- `src/celeste/modalities/embeddings/client.py` + +They expose operations and sync/stream namespaces. Examples: + +- text: `generate`, `analyze`, `stream.generate`, `sync.generate` +- images: `generate`, `edit`, `stream.generate`, `sync.edit` +- audio: `speak`, `stream.speak`, `sync.speak` +- videos: `generate`, `edit`, `sync.generate` +- embeddings: `embed`, `sync.embed` + +## Provider And Modality Registration + +Provider support has multiple seams. Do not collapse them into one invented registry. +Directory presence alone is not active provider support. A provider is active only when its enum, auth registration, provider API or protocol seam, modality client/map, model aggregation, and tests are wired. + +Provider auth registration: + +- `src/celeste/providers//__init__.py` +- registers credentials with `register_auth(...)` +- examples: OpenAI, Anthropic, Google, Ollama + +Provider API mixins: + +- `src/celeste/providers///client.py` +- implement provider wire details such as `_make_request`, `_make_stream_request`, `_parse_usage`, `_parse_content`, `_parse_finish_reason` +- examples: `providers/openai/responses`, `providers/google/generate_content`, `providers/byteplus/videos` + +Protocol clients: + +- `src/celeste/protocols/openresponses/*` +- `src/celeste/protocols/chatcompletions/*` +- shared wire mixins, streaming parsers, tool mappers, and base parameter mappers for compatible APIs +- text-specific protocol clients live in `src/celeste/modalities/text/protocols/*` and are the concrete clients registered for `create_client(protocol=...)` +- compatible `protocol=` / `base_url` client support is currently text-only + +Modality-specific provider clients: + +- `src/celeste/modalities//providers//client.py` +- combine provider API mixins with modality clients +- define provider-specific `parameter_mappers` +- adapt parsed provider content into modality outputs + +Modality provider maps: + +- `src/celeste/modalities//providers/__init__.py` +- maps `Provider` enum values to modality client classes +- feeds `_CLIENT_MAP` in `src/celeste/__init__.py` + +Model aggregation: + +- `src/celeste/modalities//models.py` +- imports provider model lists and exports one modality-level `MODELS` +- `_models` is populated during `celeste` import from these lists + +Per-provider model definitions: + +- `src/celeste/modalities//providers//models.py` +- define `Model(id=..., provider=..., display_name=..., operations=..., parameter_constraints=..., streaming=...)` + +## Parameter And Constraint System + +Universal mapper primitives live in `src/celeste/parameters.py`: + +- `Parameters`: base `TypedDict`. +- `ParameterMapper`: maps a unified parameter into provider request shape and can transform output. +- `FieldMapper`: simple direct field mapping. + +Common parameter names live in `src/celeste/core.py` as `Parameter`. Modality-specific names live in each modality enum. Model constraints may use either common `Parameter.*` values such as `Parameter.TEMPERATURE` or modality-specific values such as `TextParameter.OUTPUT_SCHEMA`. + +Modality parameter names and `TypedDict`s live under: + +- `src/celeste/modalities/text/parameters.py` +- `src/celeste/modalities/images/parameters.py` +- `src/celeste/modalities/audio/parameters.py` +- `src/celeste/modalities/videos/parameters.py` +- `src/celeste/modalities/embeddings/parameters.py` + +Provider-specific mappers live under both protocol and modality provider packages. Examples: + +- `src/celeste/protocols/openresponses/parameters.py` +- `src/celeste/protocols/chatcompletions/parameters.py` +- `src/celeste/modalities/text/providers/google/parameters.py` +- `src/celeste/modalities/images/providers/openai/parameters.py` + +Model support for parameters comes from `Model.parameter_constraints`, not from a separate allowlist. Constraints live in `src/celeste/constraints.py` and include: + +- `Choice` +- `Range` +- `Pattern` +- `Schema` +- media constraints such as `ImageConstraint`, `ImagesConstraint`, `DocumentConstraint` +- tool constraints such as `ToolSupport`, `ToolChoiceSupport` + +Optional input support is inferred from constraints through `src/celeste/io.py`. + +## Templates + +Start new provider, protocol, modality, and test work from templates: + +- `templates/providers/{provider_slug}/` +- `templates/providers/{provider_slug}/{api_slug}/` +- `templates/modalities/{modality_slug}/` +- `templates/modalities/{modality_slug}/providers/{provider_slug}/` +- `templates/protocols/{protocol_slug}/` + +Provider API mixin templates are validated by focused tests. If changing provider API patterns, inspect the template and `references/verification.md` together. + +## Tests As Executable Documentation + +Use focused tests by seam. The canonical test routing matrix lives in `references/verification.md`; keep detailed test-file lists there to avoid duplicate drift. + +Common commands: + +- `make lint` +- `make typecheck` +- `make test` +- focused tests with `uv run pytest tests/unit_tests/.py -q` + +`make format` and current `make ci` may rewrite files; use them only when mutation is intended. + +## Source-Of-Truth Order + +When sources disagree, follow this order: + +1. Current source code in `src/celeste/`. +2. Current tests and templates for existing behavior. +3. README examples and public exports. +4. OpenSpec artifacts for the active change's intended behavior and acceptance criteria. +5. Notes such as `common_agent_mistakes.md`. +6. Model memory. + +`common_agent_mistakes.md` is advisory. Keep useful warnings, but verify every rule against current code and task context. diff --git a/.agents/skills/celeste-python/references/sdk-architecture.md b/.agents/skills/celeste-python/references/sdk-architecture.md new file mode 100644 index 00000000..a7a361bc --- /dev/null +++ b/.agents/skills/celeste-python/references/sdk-architecture.md @@ -0,0 +1,185 @@ +# Celeste SDK Architecture Reference + +Use this for changes inside `src/celeste`, templates, or Celeste tests. + +## Layering + +`src/celeste/client.py` defines the base layering: + +```text +HTTPClient + ^ +APIMixin + ^ +ModalityClient + ^ +Concrete provider/modality client +``` + +Provider API mixins handle wire details. Modality clients handle operation methods, common prediction/streaming flow, output construction, parameter mapping, metadata, and warnings. + +## Public Client Resolution + +`src/celeste/__init__.py` builds `_CLIENT_MAP` from modality provider maps and protocol clients. + +`create_client(...)` resolves: + +- deprecated `capability` into `modality` + `operation` +- model objects or string model IDs +- provider or protocol target +- provider credentials or BYOA protocol auth + +When changing client resolution, inspect: + +- `src/celeste/__init__.py` +- `references/verification.md` for focused test selection + +## Modality Clients + +Modality clients define operation-level Python APIs: + +- `src/celeste/modalities/text/client.py` +- `src/celeste/modalities/images/client.py` +- `src/celeste/modalities/audio/client.py` +- `src/celeste/modalities/videos/client.py` +- `src/celeste/modalities/embeddings/client.py` + +They should delegate request execution through `_predict(...)` and streaming through `_stream(...)`. + +## Provider Support Seams + +Provider auth registration: + +- `src/celeste/providers//__init__.py` +- registers credentials or no-auth behavior + +Provider API mixin: + +- `src/celeste/providers///client.py` +- owns endpoint routing, HTTP request shape, streaming request shape, response parsing, usage mapping, and finish reason parsing + +Modality provider client: + +- `src/celeste/modalities//providers//client.py` +- combines the provider API mixin with a modality client +- adapts provider content into `TextContent`, `ImageArtifact`, `AudioArtifact`, `VideoArtifact`, or embeddings +- returns provider-specific stream class where supported + +Modality provider map: + +- `src/celeste/modalities//providers/__init__.py` +- maps `Provider` to concrete modality client class + +Model aggregation: + +- `src/celeste/modalities//models.py` +- imports provider model lists + +Per-provider models: + +- `src/celeste/modalities//providers//models.py` +- define `Model` entries with `operations`, `streaming`, and `parameter_constraints` + +When adding provider support, check every relevant seam. Missing one seam usually creates runtime lookup or model discovery failures. + +## Protocol Clients + +Shared compatible API implementations live under: + +- `src/celeste/protocols/openresponses/` +- `src/celeste/protocols/chatcompletions/` + +These packages own shared wire mixins, streaming parsers, tool mappers, and base parameter mappers. + +Concrete text protocol clients live under: + +- `src/celeste/modalities/text/protocols/openresponses/` +- `src/celeste/modalities/text/protocols/chatcompletions/` + +`create_client(protocol=..., base_url=...)` currently resolves through those text-specific protocol clients. Provider clients can inherit shared protocol behavior instead of duplicating wire behavior when the provider is protocol-compatible. + +## Message Serialization + +Canonical chat-message helpers live in `src/celeste/messages.py`. + +Use those helpers for SDK-internal message work: + +- normalize `Message.content` into ordered content parts +- collect media input types from messages and top-level media kwargs +- build request messages from prompt/messages/media inputs +- serialize structured text content for text-only provider fields +- serialize `ToolResult.content` for provider fields that accept JSON objects +- raise clear errors when a provider serializer receives an unsupported message part + +Provider and protocol serializers should use these helpers rather than hand-rolled message traversal. + +## Parameters And Constraints + +Modality parameter enums and `TypedDict`s live under each modality's `parameters.py`. + +Provider-specific parameter mapping uses: + +- `ParameterMapper` +- `FieldMapper` +- provider/protocol mapper lists such as `OPENRESPONSES_PARAMETER_MAPPERS`, `GOOGLE_PARAMETER_MAPPERS` + +Model constraints define supported parameters: + +```python +Model( + id="...", + provider=Provider.OPENAI, + operations={Modality.TEXT: {Operation.GENERATE}}, + parameter_constraints={ + TextParameter.OUTPUT_SCHEMA: Schema(), + TextParameter.TOOLS: ToolSupport(tools=[WebSearch]), + }, +) +``` + +If adding a parameter, update the relevant modality enum/TypedDict, constraints in model definitions, provider mappers, and tests. + +## Artifacts And Media Support + +Media support is inferred from model constraints and message content parts. Do not add separate app-side media allowlists. + +Relevant source: + +- `src/celeste/artifacts.py` +- `src/celeste/mime_types.py` +- `src/celeste/constraints.py` +- `src/celeste/io.py` +- `references/verification.md` for focused test selection + +## Streaming + +Streaming support requires: + +- model `streaming=True` +- provider API `_make_stream_request(...)` +- a stream class for provider/protocol event parsing +- modality stream aggregation when needed + +Relevant source: + +- `src/celeste/streaming.py` +- modality `streaming.py` files +- protocol/provider streaming files +- `references/verification.md` for focused test selection + +## Templates + +Use templates before inventing structure: + +- `templates/providers/{provider_slug}/` +- `templates/providers/{provider_slug}/{api_slug}/` +- `templates/modalities/{modality_slug}/` +- `templates/modalities/{modality_slug}/providers/{provider_slug}/` +- `templates/protocols/{protocol_slug}/` + +Provider API templates are validated by focused tests. +Templates are source of truth for generated provider/module shape. Current source and message helpers are source of truth for multimodal message serialization semantics. + +## Tests To Consult + +Use `references/verification.md` for the current focused test matrix. Keep detailed test-file routing there to avoid duplicate drift. diff --git a/.agents/skills/celeste-python/references/verification.md b/.agents/skills/celeste-python/references/verification.md new file mode 100644 index 00000000..0f437a50 --- /dev/null +++ b/.agents/skills/celeste-python/references/verification.md @@ -0,0 +1,138 @@ +# Celeste Verification Reference + +Use focused tests first, then broader commands when the blast radius warrants it. + +## Commands + +Focused unit test: + +```bash +uv run pytest tests/unit_tests/.py -q +``` + +Read-only or failure-only repo checks: + +```bash +make lint +make typecheck +make test +``` + +Mutation-intended finalization commands: + +```bash +make format +make ci +``` + +`make format` rewrites files. Current `make ci` runs `lint-fix` and `format`, so treat it as mutating. + +Integration tests require provider credentials and are not default for local verification. + +## Test Matrix By Seam + +Public exports, client factory, model resolution: + +```bash +uv run pytest tests/unit_tests/test_init.py -q +``` + +Model registry and filtering: + +```bash +uv run pytest tests/unit_tests/test_models.py -q +``` + +ModalityClient request building, output construction, metadata, warnings, streaming: + +```bash +uv run pytest tests/unit_tests/test_client.py -q +``` + +Parameter `TypedDict`s and `ParameterMapper` behavior: + +```bash +uv run pytest tests/unit_tests/test_parameters.py -q +``` + +Constraints and optional input support: + +```bash +uv run pytest tests/unit_tests/test_constraints.py tests/unit_tests/test_io.py -q +``` + +Provider API mixin/template contract: + +```bash +uv run pytest tests/unit_tests/test_provider_api_templates.py -q +``` + +Protocol and base URL routing: + +```bash +uv run pytest tests/unit_tests/test_protocol_base_url.py -q +``` + +Vertex/provider auth routing: + +```bash +uv run pytest tests/unit_tests/test_vertex_routing.py -q +``` + +Structured outputs: + +```bash +uv run pytest tests/unit_tests/test_structured_outputs.py -q +``` + +Tools, tool choice, tool outputs, and tool-call validation: + +```bash +uv run pytest tests/unit_tests/test_tool_choice.py tests/unit_tests/test_tool_outputs.py tests/unit_tests/test_text_tool_results.py tests/unit_tests/test_google_tools_mapper.py tests/unit_tests/test_tool_call_validation.py -q +``` + +Artifacts and MIME types: + +```bash +uv run pytest tests/unit_tests/test_artifacts.py tests/unit_tests/test_mime_types.py -q +``` + +Multimodal text messages and media support validation: + +```bash +uv run pytest tests/unit_tests/test_text_multimodal_message_content.py tests/unit_tests/test_text_multimodal_message_request_building.py tests/unit_tests/test_text_media_support_validation.py -q +``` + +Analyze-media convenience paths: + +```bash +uv run pytest tests/unit_tests/test_text_modality_analyze_image.py tests/unit_tests/test_text_modality_analyze_document.py -q +``` + +Embeddings input and multimodal embeddings: + +```bash +uv run pytest tests/unit_tests/test_embeddings_input.py tests/unit_tests/test_embeddings_multimodal.py -q +``` + +Streaming metadata: + +```bash +uv run pytest tests/unit_tests/test_streaming.py tests/unit_tests/test_stream_metadata_from_response_data.py -q +``` + +Telemetry content and streaming: + +```bash +uv run pytest tests/unit_tests/test_telemetry_content_events.py tests/unit_tests/test_telemetry_streaming.py tests/unit_tests/test_telemetry_metrics.py -q +``` + +## When To Run Broader Checks + +Run `make test` when a change touches shared client behavior, model registry, constraints, parameters, artifacts, tools, or multiple providers. + +Run `make typecheck` when changing signatures, `TypedDict`s, generic client classes, or public exports. + +Run mutation-intended finalization commands only when file rewrites are acceptable. + +If a command cannot run because dependencies or credentials are missing, state that explicitly and describe the remaining risk. diff --git a/src/celeste/__init__.py b/src/celeste/__init__.py index df37b920..996c36e2 100644 --- a/src/celeste/__init__.py +++ b/src/celeste/__init__.py @@ -46,7 +46,18 @@ WebSearch, XSearch, ) -from celeste.types import Content, Message, Role +from celeste.types import ( + AudioPart, + DocumentPart, + ImagePart, + Message, + MessageContent, + MessagePart, + Role, + TextPart, + ToolResultContent, + VideoPart, +) logger = logging.getLogger(__name__) @@ -277,13 +288,17 @@ def create_client( __all__ = [ "APIKey", + "AudioPart", "Authentication", "Capability", "CodeExecution", - "Content", + "DocumentPart", "Error", + "ImagePart", "Input", "Message", + "MessageContent", + "MessagePart", "Modality", "Model", "Operation", @@ -291,13 +306,16 @@ def create_client( "Protocol", "Provider", "Role", + "TextPart", "Tool", "ToolCall", "ToolChoice", "ToolError", "ToolOutput", "ToolResult", + "ToolResultContent", "Usage", + "VideoPart", "WebSearch", "XSearch", "audio", diff --git a/src/celeste/messages.py b/src/celeste/messages.py new file mode 100644 index 00000000..0dc64bd0 --- /dev/null +++ b/src/celeste/messages.py @@ -0,0 +1,178 @@ +"""Helpers for canonical chat message content.""" + +import json + +from pydantic import BaseModel + +from celeste.core import InputType +from celeste.tools import ToolResult +from celeste.types import ( + AudioContent, + AudioPart, + DocumentContent, + DocumentPart, + ImageContent, + ImagePart, + Message, + MessageContent, + MessagePart, + Role, + TextPart, + VideoContent, + VideoPart, +) + + +def _as_list[T](value: T | list[T] | None) -> list[T]: + if value is None: + return [] + if isinstance(value, list): + return value + return [value] + + +def message_parts(content: MessageContent) -> list[MessagePart]: + """Return message content as an ordered list of content parts.""" + if isinstance(content, str): + return [TextPart(text=content)] + return content + + +def _message_media_types(messages: list[Message | ToolResult] | None) -> set[InputType]: + """Collect media input types present in normal chat messages.""" + media_types: set[InputType] = set() + for message in messages or []: + if isinstance(message, ToolResult): + continue + for part in message_parts(message.content): + if part.type != InputType.TEXT: + media_types.add(part.type) + return media_types + + +def media_types( + *, + messages: list[Message | ToolResult] | None = None, + image: ImageContent | None = None, + video: VideoContent | None = None, + audio: AudioContent | None = None, + document: DocumentContent | None = None, +) -> set[InputType]: + """Collect media input types from messages and top-level media kwargs.""" + media_types = _message_media_types(messages) + if _as_list(image): + media_types.add(InputType.IMAGE) + if _as_list(video): + media_types.add(InputType.VIDEO) + if _as_list(audio): + media_types.add(InputType.AUDIO) + if _as_list(document): + media_types.add(InputType.DOCUMENT) + return media_types + + +def _user_message( + *, + prompt: str | None = None, + image: ImageContent | None = None, + video: VideoContent | None = None, + audio: AudioContent | None = None, + document: DocumentContent | None = None, +) -> Message | None: + """Build a user message from prompt plus top-level media kwargs.""" + images = _as_list(image) + videos = _as_list(video) + audios = _as_list(audio) + documents = _as_list(document) + has_media = bool(images or videos or audios or documents) + + if not has_media: + if prompt is None: + return None + return Message(role=Role.USER, content=prompt) + + parts: list[MessagePart] = [] + parts.extend(ImagePart(image=item) for item in images) + parts.extend(VideoPart(video=item) for item in videos) + parts.extend(AudioPart(audio=item) for item in audios) + parts.extend(DocumentPart(document=item) for item in documents) + if prompt is not None: + parts.append(TextPart(text=prompt)) + return Message(role=Role.USER, content=parts) + + +def request_messages( + *, + prompt: str | None = None, + messages: list[Message | ToolResult] | None = None, + image: ImageContent | None = None, + video: VideoContent | None = None, + audio: AudioContent | None = None, + document: DocumentContent | None = None, +) -> list[Message | ToolResult]: + """Return full request messages, appending top-level media as a user turn.""" + result = list(messages or []) + extra_message = _user_message( + prompt=prompt, + image=image, + video=video, + audio=audio, + document=document, + ) + if extra_message is not None: + result.append(extra_message) + if not result: + msg = "Text request requires prompt, messages, or media input" + raise ValueError(msg) + return result + + +def _base_model_list(value: object) -> list[BaseModel] | None: + if not isinstance(value, list): + return None + models: list[BaseModel] = [] + for item in value: + if not isinstance(item, BaseModel): + return None + models.append(item) + return models + + +def content_to_text(content: object) -> str: + """Serialize structured content for text-only message fields.""" + if isinstance(content, str): + return content + if isinstance(content, BaseModel): + return content.model_dump_json() + models = _base_model_list(content) + if models is not None: + return json.dumps([item.model_dump(mode="json") for item in models]) + return json.dumps(content) + + +def tool_result_object(result: ToolResult) -> object: + """Serialize tool-result content for provider fields that accept JSON objects.""" + content = result.content + if isinstance(content, BaseModel): + return content.model_dump(mode="json") + models = _base_model_list(content) + if models is not None: + return [item.model_dump(mode="json") for item in models] + return content + + +def require_part(provider: str, part: MessagePart, allowed: tuple[type, ...]) -> None: + """Raise when a provider serializer receives an unsupported content part.""" + if not isinstance(part, allowed): + msg = f"{provider} text messages do not support {part.type} content parts" + raise ValueError(msg) + + +__all__ = [ + "content_to_text", + "media_types", + "message_parts", + "request_messages", + "require_part", + "tool_result_object", +] diff --git a/src/celeste/modalities/text/client.py b/src/celeste/modalities/text/client.py index 29b1e714..0acc48f1 100644 --- a/src/celeste/modalities/text/client.py +++ b/src/celeste/modalities/text/client.py @@ -6,8 +6,9 @@ from asgiref.sync import async_to_sync from celeste.client import ModalityClient -from celeste.core import InputType, Modality -from celeste.tools import CodeExecution, WebSearch, XSearch +from celeste.core import Modality +from celeste.messages import media_types +from celeste.tools import CodeExecution, ToolResult, WebSearch, XSearch from celeste.types import ( AudioContent, DocumentContent, @@ -48,12 +49,13 @@ async def generate( self, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, extra_body: dict[str, Any] | None = None, extra_headers: dict[str, str] | None = None, **parameters: Unpack[TextParameters], ) -> TextOutput: """Generate text from prompt.""" + self._check_media_support(messages=messages) inputs = TextInput(prompt=prompt, messages=messages) return await self._predict( inputs, extra_body=extra_body, extra_headers=extra_headers, **parameters @@ -63,7 +65,7 @@ async def analyze( self, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, image: ImageContent | None = None, video: VideoContent | None = None, audio: AudioContent | None = None, @@ -73,10 +75,13 @@ async def analyze( **parameters: Unpack[TextParameters], ) -> TextOutput: """Analyze image(s), video(s), audio, or document(s) with prompt or messages.""" - if messages is None: - self._check_media_support( - image=image, video=video, audio=audio, document=document - ) + self._check_media_support( + messages=messages, + image=image, + video=video, + audio=audio, + document=document, + ) inputs = TextInput( prompt=prompt, messages=messages, @@ -117,31 +122,28 @@ def _build_request( def _check_media_support( self, - image: ImageContent | None, - video: VideoContent | None, - audio: AudioContent | None, + image: ImageContent | None = None, + video: VideoContent | None = None, + audio: AudioContent | None = None, document: DocumentContent | None = None, + messages: list[Message | ToolResult] | None = None, ) -> None: """Check model supports the provided media types. Raises: NotImplementedError: If media type is provided but model doesn't support it. """ - if image is not None and InputType.IMAGE not in self.model.optional_input_types: - msg = f"Model {self.model.id} does not support image input" - raise NotImplementedError(msg) - if video is not None and InputType.VIDEO not in self.model.optional_input_types: - msg = f"Model {self.model.id} does not support video input" - raise NotImplementedError(msg) - if audio is not None and InputType.AUDIO not in self.model.optional_input_types: - msg = f"Model {self.model.id} does not support audio input" - raise NotImplementedError(msg) - if ( - document is not None - and InputType.DOCUMENT not in self.model.optional_input_types - ): - msg = f"Model {self.model.id} does not support document input" - raise NotImplementedError(msg) + provided_media = media_types( + messages=messages, + image=image, + video=video, + audio=audio, + document=document, + ) + for input_type in provided_media: + if input_type not in self.model.optional_input_types: + msg = f"Model {self.model.id} does not support {input_type.value} input" + raise NotImplementedError(msg) @property def stream(self) -> "TextStreamNamespace": @@ -167,7 +169,7 @@ def generate( self, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, extra_body: dict[str, Any] | None = None, extra_headers: dict[str, str] | None = None, **parameters: Unpack[TextParameters], @@ -178,6 +180,7 @@ def generate( async for chunk in client.stream.generate("Hello"): print(chunk.content) """ + self._client._check_media_support(messages=messages) inputs = TextInput(prompt=prompt, messages=messages) return self._client._stream( inputs, @@ -191,7 +194,7 @@ def analyze( self, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, image: ImageContent | None = None, video: VideoContent | None = None, audio: AudioContent | None = None, @@ -209,10 +212,13 @@ def analyze( async for chunk in client.stream.analyze("Summarize", document=doc): print(chunk.content) """ - if messages is None: - self._client._check_media_support( - image=image, video=video, audio=audio, document=document - ) + self._client._check_media_support( + messages=messages, + image=image, + video=video, + audio=audio, + document=document, + ) inputs = TextInput( prompt=prompt, messages=messages, @@ -243,7 +249,7 @@ def generate( self, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, extra_body: dict[str, Any] | None = None, extra_headers: dict[str, str] | None = None, **parameters: Unpack[TextParameters], @@ -254,6 +260,7 @@ def generate( result = client.sync.generate("Hello") print(result.content) """ + self._client._check_media_support(messages=messages) inputs = TextInput(prompt=prompt, messages=messages) return async_to_sync(self._client._predict)( inputs, @@ -266,7 +273,7 @@ def analyze( self, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, image: ImageContent | None = None, video: VideoContent | None = None, audio: AudioContent | None = None, @@ -284,10 +291,13 @@ def analyze( result = client.sync.analyze("Summarize", document=doc) print(result.content) """ - if messages is None: - self._client._check_media_support( - image=image, video=video, audio=audio, document=document - ) + self._client._check_media_support( + messages=messages, + image=image, + video=video, + audio=audio, + document=document, + ) inputs = TextInput( prompt=prompt, messages=messages, @@ -319,7 +329,7 @@ def generate( self, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, extra_body: dict[str, Any] | None = None, extra_headers: dict[str, str] | None = None, **parameters: Unpack[TextParameters], @@ -347,7 +357,7 @@ def analyze( self, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, image: ImageContent | None = None, video: VideoContent | None = None, audio: AudioContent | None = None, diff --git a/src/celeste/modalities/text/io.py b/src/celeste/modalities/text/io.py index e2951d44..dee073f9 100644 --- a/src/celeste/modalities/text/io.py +++ b/src/celeste/modalities/text/io.py @@ -12,6 +12,7 @@ from pydantic import Field from celeste.io import Chunk, FinishReason, Input, Output, Usage +from celeste.messages import content_to_text from celeste.tools import ToolResult from celeste.types import ( AudioContent, @@ -28,7 +29,7 @@ class TextInput(Input): """Input for text operations.""" prompt: str | None = None - messages: list[ToolResult | Message] | None = None + messages: list[Message | ToolResult] | None = None text: str | list[str] | None = None image: ImageContent | None = None video: VideoContent | None = None @@ -66,7 +67,7 @@ def message(self) -> Message: """The assistant message for multi-turn conversations.""" return Message( role=Role.ASSISTANT, - content=self.content, + content=content_to_text(self.content), tool_calls=self.tool_calls if self.tool_calls else None, reasoning=self.reasoning, signature=self.signature, diff --git a/src/celeste/modalities/text/protocols/chatcompletions/client.py b/src/celeste/modalities/text/protocols/chatcompletions/client.py index a71b1ac9..f529726c 100644 --- a/src/celeste/modalities/text/protocols/chatcompletions/client.py +++ b/src/celeste/modalities/text/protocols/chatcompletions/client.py @@ -3,8 +3,12 @@ import json from typing import Any -from pydantic import BaseModel - +from celeste.messages import ( + content_to_text, + message_parts, + request_messages, + require_part, +) from celeste.parameters import ParameterMapper from celeste.protocols.chatcompletions.client import ( ChatCompletionsClient as ChatCompletionsMixin, @@ -14,7 +18,7 @@ ) from celeste.protocols.chatcompletions.tools import parse_tool_calls from celeste.tools import ToolCall, ToolResult -from celeste.types import Message, TextContent +from celeste.types import DocumentPart, ImagePart, Message, Role, TextContent, TextPart from celeste.utils import build_document_data_url, build_image_data_url from ...client import TextClient @@ -25,32 +29,56 @@ from .parameters import CHATCOMPLETIONS_PARAMETER_MAPPERS -def _chat_completions_text_messages( - messages: list[Message], +def _serialize_content(content: Any) -> Any: + if isinstance(content, str): + return content + items: list[dict[str, Any]] = [] + for part in message_parts(content): + require_part( + "Chat Completions", + part, + (TextPart, ImagePart, DocumentPart), + ) + if isinstance(part, TextPart): + items.append({"type": "text", "text": part.text}) + elif isinstance(part, ImagePart): + items.append( + { + "type": "image_url", + "image_url": {"url": build_image_data_url(part.image)}, + } + ) + elif isinstance(part, DocumentPart): + items.append( + { + "type": "document_url", + "document_url": build_document_data_url(part.document), + } + ) + return items + + +def _serialize_messages( + messages: list[Message | ToolResult], ) -> list[dict[str, Any]]: items: list[dict[str, Any]] = [] for msg in messages: if isinstance(msg, ToolResult): - content = msg.content - if isinstance(content, BaseModel): - content = content.model_dump_json() - elif not isinstance(content, str): - content = json.dumps(content, default=str) items.append( { "role": "tool", "tool_call_id": msg.tool_call_id, - "content": content, + "content": content_to_text(msg.content), } ) - elif msg.role == "assistant" and msg.tool_calls: + elif msg.role == Role.ASSISTANT and msg.tool_calls: msg_dict = msg.model_dump( + exclude={"content", "tool_calls", "reasoning", "signature"}, exclude_none=True, mode="json", serialize_as_any=True, ) - msg_dict.pop("reasoning", None) - msg_dict.pop("signature", None) + msg_dict["content"] = _serialize_content(msg.content) msg_dict["tool_calls"] = [ { "id": tc.id, @@ -65,13 +93,12 @@ def _chat_completions_text_messages( items.append(msg_dict) else: msg_dict = msg.model_dump( + exclude={"content", "tool_calls", "reasoning", "signature"}, exclude_none=True, mode="json", serialize_as_any=True, ) - msg_dict.pop("tool_calls", None) - msg_dict.pop("reasoning", None) - msg_dict.pop("signature", None) + msg_dict["content"] = _serialize_content(msg.content) items.append(msg_dict) return items @@ -89,40 +116,15 @@ def parameter_mappers(cls) -> list[ParameterMapper[TextContent]]: def _init_request(self, inputs: TextInput) -> dict[str, Any]: """Initialize request with Chat Completions message format.""" - if inputs.messages is not None: - return {"messages": _chat_completions_text_messages(inputs.messages)} - - if inputs.image is None and inputs.document is None: - content: str | list[dict[str, Any]] = inputs.prompt or "" - else: - content = [] - if inputs.image is not None: - images = ( - inputs.image if isinstance(inputs.image, list) else [inputs.image] - ) - for img in images: - content.append( - { - "type": "image_url", - "image_url": {"url": build_image_data_url(img)}, - } - ) - if inputs.document is not None: - docs = ( - inputs.document - if isinstance(inputs.document, list) - else [inputs.document] - ) - for doc in docs: - content.append( - { - "type": "document_url", - "document_url": build_document_data_url(doc), - } - ) - content.append({"type": "text", "text": inputs.prompt or ""}) - - return {"messages": [{"role": "user", "content": content}]} + messages = request_messages( + prompt=inputs.prompt, + messages=inputs.messages, + image=inputs.image, + video=inputs.video, + audio=inputs.audio, + document=inputs.document, + ) + return {"messages": _serialize_messages(messages)} def _parse_content( self, diff --git a/src/celeste/modalities/text/protocols/openresponses/client.py b/src/celeste/modalities/text/protocols/openresponses/client.py index fd46f6e0..3a0e61d0 100644 --- a/src/celeste/modalities/text/protocols/openresponses/client.py +++ b/src/celeste/modalities/text/protocols/openresponses/client.py @@ -3,8 +3,13 @@ import json from typing import Any -from pydantic import BaseModel - +from celeste.artifacts import DocumentArtifact +from celeste.messages import ( + content_to_text, + message_parts, + request_messages, + require_part, +) from celeste.parameters import ParameterMapper from celeste.protocols.openresponses.client import ( OpenResponsesClient as OpenResponsesMixin, @@ -18,7 +23,7 @@ parse_tool_calls, ) from celeste.tools import ToolCall, ToolResult -from celeste.types import Message, TextContent +from celeste.types import DocumentPart, ImagePart, Message, Role, TextContent, TextPart from celeste.utils import build_document_data_url, build_image_data_url from ...client import TextClient @@ -30,28 +35,61 @@ from .parameters import OPENRESPONSES_PARAMETER_MAPPERS -def _openresponses_text_input_messages( - messages: list[Message], +def _input_file(document: DocumentArtifact) -> dict[str, Any]: + if document.url and not document.data and not document.path: + return {"type": "input_file", "file_url": document.url} + return { + "type": "input_file", + "filename": document.path.rsplit("/", 1)[-1] if document.path else "document", + "file_data": build_document_data_url(document), + } + + +def _serialize_content(content: Any) -> Any: + if isinstance(content, str): + return content + items: list[dict[str, Any]] = [] + for part in message_parts(content): + require_part( + "OpenResponses", + part, + (TextPart, ImagePart, DocumentPart), + ) + if isinstance(part, TextPart): + items.append({"type": "input_text", "text": part.text}) + elif isinstance(part, ImagePart): + items.append( + {"type": "input_image", "image_url": build_image_data_url(part.image)} + ) + elif isinstance(part, DocumentPart): + items.append(_input_file(part.document)) + return items + + +def _serialize_messages( + messages: list[Message | ToolResult], ) -> list[dict[str, Any]]: items: list[dict[str, Any]] = [] for msg in messages: if isinstance(msg, ToolResult): - content = msg.content - if isinstance(content, BaseModel): - content = content.model_dump_json() - elif not isinstance(content, str): - content = json.dumps(content, default=str) items.append( { "type": "function_call_output", "call_id": msg.tool_call_id, - "output": content, + "output": content_to_text(msg.content), } ) - elif msg.role == "assistant" and (msg.tool_calls or msg.signature): + elif msg.role == Role.ASSISTANT and (msg.tool_calls or msg.signature): sig_blocks = msg.signature if sig_blocks: items.extend(sig_blocks) + if msg.content: + items.append( + { + "role": msg.role, + "content": _serialize_content(msg.content), + } + ) if msg.tool_calls: for tc in msg.tool_calls: items.append( @@ -64,13 +102,12 @@ def _openresponses_text_input_messages( ) else: msg_dict = msg.model_dump( + exclude={"content", "tool_calls", "reasoning", "signature"}, exclude_none=True, mode="json", serialize_as_any=True, ) - msg_dict.pop("tool_calls", None) - msg_dict.pop("reasoning", None) - msg_dict.pop("signature", None) + msg_dict["content"] = _serialize_content(msg.content) items.append(msg_dict) return items @@ -126,39 +163,15 @@ def parameter_mappers(cls) -> list[ParameterMapper[TextContent]]: def _init_request(self, inputs: TextInput) -> dict[str, Any]: """Initialize request with input content.""" - if inputs.messages is not None: - return {"input": _openresponses_text_input_messages(inputs.messages)} - - content: list[dict[str, Any]] = [] - if inputs.image is not None: - images = inputs.image if isinstance(inputs.image, list) else [inputs.image] - for img in images: - content.append( - {"type": "input_image", "image_url": build_image_data_url(img)} - ) - - if inputs.document is not None: - docs = ( - inputs.document - if isinstance(inputs.document, list) - else [inputs.document] - ) - for doc in docs: - if doc.url and not doc.data and not doc.path: - content.append({"type": "input_file", "file_url": doc.url}) - else: - content.append( - { - "type": "input_file", - "filename": doc.path.rsplit("/", 1)[-1] - if doc.path - else "document", - "file_data": build_document_data_url(doc), - } - ) - - content.append({"type": "input_text", "text": inputs.prompt or ""}) - return {"input": [{"role": "user", "content": content}]} + messages = request_messages( + prompt=inputs.prompt, + messages=inputs.messages, + image=inputs.image, + video=inputs.video, + audio=inputs.audio, + document=inputs.document, + ) + return {"input": _serialize_messages(messages)} def _parse_content( self, diff --git a/src/celeste/modalities/text/providers/anthropic/client.py b/src/celeste/modalities/text/providers/anthropic/client.py index d72d7a09..b513dcfb 100644 --- a/src/celeste/modalities/text/providers/anthropic/client.py +++ b/src/celeste/modalities/text/providers/anthropic/client.py @@ -2,12 +2,15 @@ import base64 import contextlib -import json from typing import Any -from pydantic import BaseModel - from celeste.artifacts import DocumentArtifact, ImageArtifact +from celeste.messages import ( + content_to_text, + message_parts, + request_messages, + require_part, +) from celeste.mime_types import ImageMimeType from celeste.parameters import ParameterMapper from celeste.providers.anthropic.messages.client import AnthropicMessagesClient @@ -15,7 +18,7 @@ AnthropicMessagesStream as _AnthropicMessagesStream, ) from celeste.tools import ToolCall, ToolResult -from celeste.types import TextContent +from celeste.types import DocumentPart, ImagePart, Role, TextContent, TextPart from celeste.utils import detect_mime_type from ...client import TextClient @@ -119,102 +122,96 @@ def parameter_mappers(cls) -> list[ParameterMapper[TextContent]]: def _init_request(self, inputs: TextInput) -> dict[str, Any]: """Initialize request from Anthropic Messages API format.""" - if inputs.messages is not None: - system_blocks: list[dict[str, Any]] = [] - messages: list[dict[str, Any]] = [] - pending_tool_results: list[dict[str, Any]] = [] - - for message in inputs.messages: - role = message.role - content = message.content - - if role in {"system", "developer"}: - if isinstance(content, list): - for block in content: - if isinstance(block, dict): - system_blocks.append(block) - else: - system_blocks.append( - {"type": "text", "text": str(block)} - ) - elif isinstance(content, dict): - system_blocks.append(content) - elif content is not None: - system_blocks.append({"type": "text", "text": str(content)}) - continue - - if isinstance(message, ToolResult): - if isinstance(content, BaseModel): - content = content.model_dump_json() - elif not isinstance(content, str): - content = json.dumps(content, default=str) - pending_tool_results.append( + + def content_to_blocks(content: Any) -> list[dict[str, Any]]: + blocks: list[dict[str, Any]] = [] + for part in message_parts(content): + require_part( + "Anthropic", + part, + (TextPart, ImagePart, DocumentPart), + ) + if isinstance(part, TextPart): + blocks.append({"type": "text", "text": part.text}) + elif isinstance(part, ImagePart): + blocks.append( { - "type": "tool_result", - "tool_use_id": message.tool_call_id, - "content": content, + "type": "image", + "source": self._build_image_source(part.image), } ) - continue - - # Flush pending tool results as a single user message - if pending_tool_results: - messages.append({"role": "user", "content": pending_tool_results}) - pending_tool_results = [] - - if role == "assistant" and (message.tool_calls or message.signature): - content_blocks: list[dict[str, Any]] = [] - sig_blocks = message.signature - if sig_blocks: - content_blocks.extend(sig_blocks) - if content: - content_blocks.append({"type": "text", "text": str(content)}) - if message.tool_calls: - for tc in message.tool_calls: - content_blocks.append( - { - "type": "tool_use", - "id": tc.id, - "name": tc.name, - "input": tc.arguments, - } - ) - messages.append({"role": "assistant", "content": content_blocks}) - else: - messages.append({"role": role, "content": content}) - - # Flush remaining tool results - if pending_tool_results: - messages.append({"role": "user", "content": pending_tool_results}) - - request: dict[str, Any] = {"messages": messages} - if system_blocks: - request["system"] = system_blocks - return request - - if inputs.image is None and inputs.document is None: - prompt_content: str | list[dict[str, Any]] = inputs.prompt or "" - else: - prompt_content = [] - if inputs.image is not None: - images = ( - inputs.image if isinstance(inputs.image, list) else [inputs.image] - ) - for img in images: - source = self._build_image_source(img) - prompt_content.append({"type": "image", "source": source}) - if inputs.document is not None: - docs = ( - inputs.document - if isinstance(inputs.document, list) - else [inputs.document] + elif isinstance(part, DocumentPart): + blocks.append( + { + "type": "document", + "source": self._build_document_source(part.document), + } + ) + return blocks + + system_blocks: list[dict[str, Any]] = [] + messages: list[dict[str, Any]] = [] + pending_tool_results: list[dict[str, Any]] = [] + + for message in request_messages( + prompt=inputs.prompt, + messages=inputs.messages, + image=inputs.image, + video=inputs.video, + audio=inputs.audio, + document=inputs.document, + ): + role = message.role + content = message.content + + if role in {Role.SYSTEM, Role.DEVELOPER}: + system_blocks.extend(content_to_blocks(content)) + continue + + if isinstance(message, ToolResult): + pending_tool_results.append( + { + "type": "tool_result", + "tool_use_id": message.tool_call_id, + "content": content_to_text(message.content), + } ) - for doc in docs: - source = self._build_document_source(doc) - prompt_content.append({"type": "document", "source": source}) - prompt_content.append({"type": "text", "text": inputs.prompt or ""}) + continue - return {"messages": [{"role": "user", "content": prompt_content}]} + # Flush pending tool results as a single user message + if pending_tool_results: + messages.append({"role": "user", "content": pending_tool_results}) + pending_tool_results = [] + + if role == Role.ASSISTANT and (message.tool_calls or message.signature): + content_blocks: list[dict[str, Any]] = [] + sig_blocks = message.signature + if sig_blocks: + content_blocks.extend(sig_blocks) + if content: + content_blocks.extend(content_to_blocks(content)) + if message.tool_calls: + for tc in message.tool_calls: + content_blocks.append( + { + "type": "tool_use", + "id": tc.id, + "name": tc.name, + "input": tc.arguments, + } + ) + messages.append({"role": "assistant", "content": content_blocks}) + else: + messages.append({"role": role, "content": content_to_blocks(content)}) + + # Flush remaining tool results + if pending_tool_results: + messages.append({"role": "user", "content": pending_tool_results}) + + request: dict[str, Any] = {"messages": messages} + if system_blocks: + request["system"] = system_blocks + return request def _build_document_source(self, doc: DocumentArtifact) -> dict[str, Any]: """Build Anthropic document source dict from DocumentArtifact.""" diff --git a/src/celeste/modalities/text/providers/cohere/client.py b/src/celeste/modalities/text/providers/cohere/client.py index 4bf64f6b..74d03ad3 100644 --- a/src/celeste/modalities/text/providers/cohere/client.py +++ b/src/celeste/modalities/text/providers/cohere/client.py @@ -2,12 +2,19 @@ from typing import Any +from celeste.messages import ( + message_parts, + request_messages, + require_part, + tool_result_object, +) from celeste.parameters import ParameterMapper from celeste.providers.cohere.chat.client import CohereChatClient from celeste.providers.cohere.chat.streaming import ( CohereChatStream as _CohereChatStream, ) -from celeste.types import TextContent +from celeste.tools import ToolResult +from celeste.types import ImagePart, TextContent, TextPart from celeste.utils import build_image_data_url from ...client import TextClient @@ -31,34 +38,51 @@ def parameter_mappers(cls) -> list[ParameterMapper[TextContent]]: def _init_request(self, inputs: TextInput) -> dict[str, Any]: """Initialize request from Cohere v2 Chat API messages array format.""" - # If messages provided, use them directly (messages take precedence) - if inputs.messages is not None: - return { - "messages": [ - message.model_dump( - exclude_none=True, - mode="json", - serialize_as_any=True, - ) - for message in inputs.messages - ] - } - # Fall back to prompt-based input - if inputs.image is None: - content: str | list[dict[str, Any]] = inputs.prompt or "" - else: - images = inputs.image if isinstance(inputs.image, list) else [inputs.image] - content = [ - { - "type": "image_url", - "image_url": {"url": build_image_data_url(img)}, - } - for img in images - ] - content.append({"type": "text", "text": inputs.prompt or ""}) + def cohere_content(content: Any) -> Any: + if isinstance(content, str): + return content + items: list[dict[str, Any]] = [] + for part in message_parts(content): + require_part("Cohere", part, (TextPart, ImagePart)) + if isinstance(part, TextPart): + items.append({"type": "text", "text": part.text}) + elif isinstance(part, ImagePart): + items.append( + { + "type": "image_url", + "image_url": {"url": build_image_data_url(part.image)}, + } + ) + return items - return {"messages": [{"role": "user", "content": content}]} + messages = [] + for message in request_messages( + prompt=inputs.prompt, + messages=inputs.messages, + image=inputs.image, + video=inputs.video, + audio=inputs.audio, + document=inputs.document, + ): + if isinstance(message, ToolResult): + msg = message.model_dump( + exclude={"content"}, + exclude_none=True, + mode="json", + serialize_as_any=True, + ) + msg["content"] = tool_result_object(message) + else: + msg = message.model_dump( + exclude={"content", "tool_calls", "reasoning", "signature"}, + exclude_none=True, + mode="json", + serialize_as_any=True, + ) + msg["content"] = cohere_content(message.content) + messages.append(msg) + return {"messages": messages} def _parse_content( self, diff --git a/src/celeste/modalities/text/providers/google/client.py b/src/celeste/modalities/text/providers/google/client.py index 34926dea..ad4e1430 100644 --- a/src/celeste/modalities/text/providers/google/client.py +++ b/src/celeste/modalities/text/providers/google/client.py @@ -3,8 +3,11 @@ from typing import Any from uuid import uuid4 -from pydantic import BaseModel - +from celeste.messages import ( + message_parts, + request_messages, + tool_result_object, +) from celeste.parameters import ParameterMapper from celeste.providers.google.generate_content.client import GoogleGenerateContentClient from celeste.providers.google.generate_content.streaming import ( @@ -12,7 +15,15 @@ ) from celeste.providers.google.utils import build_media_part from celeste.tools import ToolCall, ToolResult -from celeste.types import TextContent +from celeste.types import ( + AudioPart, + DocumentPart, + ImagePart, + Role, + TextContent, + TextPart, + VideoPart, +) from ...client import TextClient from ...io import TextInput @@ -67,102 +78,73 @@ def parameter_mappers(cls) -> list[ParameterMapper[TextContent]]: def _init_request(self, inputs: TextInput) -> dict[str, Any]: """Initialize request from Google contents array format.""" - # If messages provided, use them with special handling for system/developer - if inputs.messages is not None: - - def normalize_part(part: Any) -> dict[str, Any]: - """Normalize a content part to Google's format.""" - if isinstance(part, str): - return {"text": part} - if isinstance(part, dict): - return part - return {"text": str(part)} - - def content_to_parts(content: Any) -> list[dict[str, Any]]: - """Convert message content to Google parts array.""" - if isinstance(content, str): - return [{"text": content}] - if isinstance(content, list): - return [normalize_part(p) for p in content] - return [normalize_part(content)] - - system_parts: list[dict[str, Any]] = [] - contents: list[dict[str, Any]] = [] - - for msg in inputs.messages: - if msg.role in ("system", "developer"): - system_parts.extend(content_to_parts(msg.content)) - elif isinstance(msg, ToolResult): - content = msg.content - if isinstance(content, BaseModel): - content = content.model_dump(mode="json") - contents.append( - { - "role": "user", - "parts": [ - { - "functionResponse": { - "name": msg.name, - "response": {"result": content}, - } - } - ], - } - ) - else: - role = "model" if msg.role == "assistant" else msg.role - sig_blocks = msg.signature - msg_parts = list(sig_blocks) if sig_blocks else [] - msg_parts.extend(content_to_parts(msg.content)) - if msg.tool_calls: - for tc in msg.tool_calls: - part: dict[str, Any] = { - "functionCall": { - "name": tc.name, - "args": tc.arguments, + + def content_to_parts(content: Any) -> list[dict[str, Any]]: + """Convert message content to Google parts array.""" + parts: list[dict[str, Any]] = [] + for part in message_parts(content): + if isinstance(part, TextPart): + parts.append({"text": part.text}) + elif isinstance(part, ImagePart): + parts.append(build_media_part(part.image)) + elif isinstance(part, VideoPart): + parts.append(build_media_part(part.video)) + elif isinstance(part, AudioPart): + parts.append(build_media_part(part.audio)) + elif isinstance(part, DocumentPart): + parts.append(build_media_part(part.document)) + return parts + + system_parts: list[dict[str, Any]] = [] + contents: list[dict[str, Any]] = [] + + for msg in request_messages( + prompt=inputs.prompt, + messages=inputs.messages, + image=inputs.image, + video=inputs.video, + audio=inputs.audio, + document=inputs.document, + ): + if msg.role in {Role.SYSTEM, Role.DEVELOPER}: + system_parts.extend(content_to_parts(msg.content)) + elif isinstance(msg, ToolResult): + contents.append( + { + "role": "user", + "parts": [ + { + "functionResponse": { + "name": msg.name, + "response": {"result": tool_result_object(msg)}, } } - thought_sig = getattr(tc, "thoughtSignature", None) - if thought_sig: - part["thoughtSignature"] = thought_sig - msg_parts.append(part) - contents.append({"role": role, "parts": msg_parts}) - - result: dict[str, Any] = {"contents": contents} - if system_parts: - result["system_instruction"] = {"parts": system_parts} - return result - - # Fall back to prompt-based input - parts: list[dict[str, Any]] = [] - - if inputs.image is not None: - images = inputs.image if isinstance(inputs.image, list) else [inputs.image] - for img in images: - parts.append(build_media_part(img)) - - if inputs.video is not None: - videos = inputs.video if isinstance(inputs.video, list) else [inputs.video] - for vid in videos: - parts.append(build_media_part(vid)) - - if inputs.audio is not None: - audios = inputs.audio if isinstance(inputs.audio, list) else [inputs.audio] - for aud in audios: - parts.append(build_media_part(aud)) - - if inputs.document is not None: - docs = ( - inputs.document - if isinstance(inputs.document, list) - else [inputs.document] - ) - for doc in docs: - parts.append(build_media_part(doc)) - - parts.append({"text": inputs.prompt or ""}) - - return {"contents": [{"role": "user", "parts": parts}]} + ], + } + ) + else: + role = "model" if msg.role == Role.ASSISTANT else msg.role + sig_blocks = msg.signature + msg_parts = list(sig_blocks) if sig_blocks else [] + msg_parts.extend(content_to_parts(msg.content)) + if msg.tool_calls: + for tc in msg.tool_calls: + part: dict[str, Any] = { + "functionCall": { + "name": tc.name, + "args": tc.arguments, + } + } + thought_sig = getattr(tc, "thoughtSignature", None) + if thought_sig: + part["thoughtSignature"] = thought_sig + msg_parts.append(part) + contents.append({"role": role, "parts": msg_parts}) + + result: dict[str, Any] = {"contents": contents} + if system_parts: + result["system_instruction"] = {"parts": system_parts} + return result def _parse_content( self, diff --git a/src/celeste/modalities/text/providers/xai/client.py b/src/celeste/modalities/text/providers/xai/client.py index 64ad1170..24ca7d11 100644 --- a/src/celeste/modalities/text/providers/xai/client.py +++ b/src/celeste/modalities/text/providers/xai/client.py @@ -33,7 +33,11 @@ def parameter_mappers(cls) -> list[ParameterMapper[TextContent]]: def _init_request(self, inputs: TextInput) -> dict[str, Any]: """xAI accepts plain string input for text-only requests.""" - if inputs.messages is None and inputs.image is None: + has_media = any( + media is not None + for media in (inputs.image, inputs.video, inputs.audio, inputs.document) + ) + if inputs.messages is None and not has_media: return {"input": inputs.prompt or ""} return super()._init_request(inputs) diff --git a/src/celeste/namespaces/domains.py b/src/celeste/namespaces/domains.py index d6a40fd7..bc281a04 100644 --- a/src/celeste/namespaces/domains.py +++ b/src/celeste/namespaces/domains.py @@ -8,7 +8,7 @@ from pydantic import SecretStr -from celeste import Authentication, create_client +from celeste import Authentication, ToolResult, create_client from celeste.artifacts import ImageArtifact from celeste.core import Modality, Operation, Protocol, Provider from celeste.modalities.audio.io import AudioOutput @@ -40,7 +40,7 @@ def generate( self, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, protocol: Protocol | None = None, @@ -76,7 +76,7 @@ def generate( self, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, protocol: Protocol | None = None, @@ -112,7 +112,7 @@ def generate( self, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, protocol: Protocol | None = None, @@ -182,7 +182,7 @@ async def generate( self, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, protocol: Protocol | None = None, @@ -326,7 +326,7 @@ def analyze( image: ImageContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, api_key: str | SecretStr | None = None, @@ -398,7 +398,7 @@ def analyze( image: ImageContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, api_key: str | SecretStr | None = None, @@ -468,7 +468,7 @@ def analyze( image: ImageContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, api_key: str | SecretStr | None = None, @@ -592,7 +592,7 @@ async def analyze( image: ImageContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, api_key: str | SecretStr | None = None, @@ -699,7 +699,7 @@ def analyze( audio: AudioContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, api_key: str | SecretStr | None = None, @@ -749,7 +749,7 @@ def analyze( audio: AudioContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, api_key: str | SecretStr | None = None, @@ -797,7 +797,7 @@ def analyze( audio: AudioContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, api_key: str | SecretStr | None = None, @@ -886,7 +886,7 @@ async def analyze( audio: AudioContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, api_key: str | SecretStr | None = None, @@ -972,7 +972,7 @@ def analyze( video: VideoContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, api_key: str | SecretStr | None = None, @@ -1001,7 +1001,7 @@ def analyze( video: VideoContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, api_key: str | SecretStr | None = None, @@ -1049,7 +1049,7 @@ def analyze( video: VideoContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, api_key: str | SecretStr | None = None, @@ -1138,7 +1138,7 @@ async def analyze( video: VideoContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, api_key: str | SecretStr | None = None, @@ -1224,7 +1224,7 @@ def analyze( document: DocumentContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, protocol: Protocol | None = None, @@ -1257,7 +1257,7 @@ def analyze( document: DocumentContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, protocol: Protocol | None = None, @@ -1290,7 +1290,7 @@ def analyze( document: DocumentContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, protocol: Protocol | None = None, @@ -1331,7 +1331,7 @@ async def analyze( document: DocumentContent, prompt: str | None = None, *, - messages: list[Message] | None = None, + messages: list[Message | ToolResult] | None = None, model: str, provider: Provider | None = None, protocol: Protocol | None = None, diff --git a/src/celeste/telemetry.py b/src/celeste/telemetry.py index f2a078b4..bd3e9297 100644 --- a/src/celeste/telemetry.py +++ b/src/celeste/telemetry.py @@ -16,10 +16,18 @@ from celeste.core import Modality, Protocol, Provider, UsageField from celeste.exceptions import StreamNotExhaustedError from celeste.io import Input, Output, Usage +from celeste.messages import request_messages from celeste.models import Model from celeste.streaming import Stream -from celeste.tools import ToolCall -from celeste.types import Message +from celeste.tools import ToolCall, ToolResult +from celeste.types import ( + AudioPart, + DocumentPart, + ImagePart, + Message, + TextPart, + VideoPart, +) _PROVIDER_NAME_MAP: dict[Provider, str] = { Provider.OPENAI: "openai", @@ -291,6 +299,16 @@ def _content_to_parts(content: Any) -> list[dict[str, Any]]: for item in content: parts.extend(_content_to_parts(item)) return parts + if isinstance(content, TextPart): + return [{"type": "text", "content": content.text}] + if isinstance(content, ImagePart): + return [_artifact_part(content.image)] + if isinstance(content, AudioPart): + return [_artifact_part(content.audio)] + if isinstance(content, VideoPart): + return [_artifact_part(content.video)] + if isinstance(content, DocumentPart): + return [_artifact_part(content.document)] if isinstance(content, Artifact): return [_artifact_part(content)] if isinstance(content, str): @@ -318,22 +336,38 @@ def _message_to_dict(message: Message) -> dict[str, Any]: return {"role": message.role.value, "parts": parts} +def _tool_result_to_dict(result: ToolResult) -> dict[str, Any]: + """Convert a celeste ToolResult into a semconv ``{role, parts}`` dict.""" + message = {"role": "tool", "parts": _content_to_parts(result.content)} + if result.tool_call_id: + message["tool_call_id"] = result.tool_call_id + if result.name: + message["name"] = result.name + return message + + def _input_messages_event(inputs: Input) -> dict[str, Any] | None: """Build the ``gen_ai.input.messages`` event attributes, or None when capture is off.""" if not _CAPTURE_CONTENT: return None + try: + input_messages = request_messages( + prompt=getattr(inputs, "prompt", None), + messages=getattr(inputs, "messages", None), + image=getattr(inputs, "image", None), + video=getattr(inputs, "video", None), + audio=getattr(inputs, "audio", None), + document=getattr(inputs, "document", None), + ) + except ValueError: + return None + messages: list[dict[str, Any]] = [] - for message in getattr(inputs, "messages", None) or []: + for message in input_messages: if isinstance(message, Message): messages.append(_message_to_dict(message)) - prompt = getattr(inputs, "prompt", None) - if prompt is not None: - parts: list[dict[str, Any]] = [{"type": "text", "content": str(prompt)}] - for media_field in ("image", "video", "audio", "document"): - media = getattr(inputs, media_field, None) - if media is not None: - parts.extend(_content_to_parts(media)) - messages.append({"role": "user", "parts": parts}) + elif isinstance(message, ToolResult): + messages.append(_tool_result_to_dict(message)) if not messages: return None return {"messages": json.dumps(messages, default=str)} diff --git a/src/celeste/tools.py b/src/celeste/tools.py index 6dad22c6..5d4fda86 100644 --- a/src/celeste/tools.py +++ b/src/celeste/tools.py @@ -8,7 +8,7 @@ from pydantic import ValidationError as PydanticValidationError from celeste.exceptions import ValidationError -from celeste.types import Message, Role, ToolCall +from celeste.types import Role, ToolCall, ToolResultContent class Tool(BaseModel): @@ -123,10 +123,13 @@ class ToolChoice(StrEnum): type ToolChoiceOption = ToolChoice | ToolDefinition -class ToolResult(Message): +class ToolResult(BaseModel): """A tool result for multi-turn tool use.""" + model_config = ConfigDict(extra="allow") + role: Role = Role.USER + content: ToolResultContent tool_call_id: str name: str | None = None @@ -157,6 +160,7 @@ class ToolError[Content](BaseModel): "ToolMapper", "ToolOutput", "ToolResult", + "ToolResultContent", "WebSearch", "XSearch", "validate_tool_calls", diff --git a/src/celeste/types.py b/src/celeste/types.py index d2b29388..f044bc05 100644 --- a/src/celeste/types.py +++ b/src/celeste/types.py @@ -1,9 +1,9 @@ """Type definitions for Celeste.""" from enum import StrEnum -from typing import Any +from typing import Annotated, Any, Literal -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field from celeste.artifacts import ( AudioArtifact, @@ -11,25 +11,65 @@ ImageArtifact, VideoArtifact, ) +from celeste.core import InputType type JsonValue = ( str | int | float | bool | None | dict[str, JsonValue] | list[JsonValue] ) type TextContent = str | JsonValue | BaseModel | list[BaseModel] +type ToolResultContent = TextContent type AudioContent = AudioArtifact | list[AudioArtifact] type DocumentContent = DocumentArtifact | list[DocumentArtifact] type ImageContent = ImageArtifact | list[ImageArtifact] type VideoContent = VideoArtifact | list[VideoArtifact] type EmbeddingsContent = list[float] | list[list[float]] -type Content = ( - TextContent | ImageContent | VideoContent | AudioContent | DocumentContent -) - type RawUsage = dict[str, int | float | None] +class TextPart(BaseModel): + """Text block inside a chat message.""" + + type: Literal[InputType.TEXT] = InputType.TEXT + text: str + + +class ImagePart(BaseModel): + """Image block inside a chat message.""" + + type: Literal[InputType.IMAGE] = InputType.IMAGE + image: ImageArtifact + + +class AudioPart(BaseModel): + """Audio block inside a chat message.""" + + type: Literal[InputType.AUDIO] = InputType.AUDIO + audio: AudioArtifact + + +class VideoPart(BaseModel): + """Video block inside a chat message.""" + + type: Literal[InputType.VIDEO] = InputType.VIDEO + video: VideoArtifact + + +class DocumentPart(BaseModel): + """Document block inside a chat message.""" + + type: Literal[InputType.DOCUMENT] = InputType.DOCUMENT + document: DocumentArtifact + + +type MessagePart = Annotated[ + TextPart | ImagePart | AudioPart | VideoPart | DocumentPart, + Field(discriminator="type"), +] +type MessageContent = str | list[MessagePart] + + class Role(StrEnum): """Message role in a conversation.""" @@ -55,7 +95,7 @@ class Message(BaseModel): model_config = ConfigDict(extra="allow") role: Role - content: Content + content: MessageContent tool_calls: list[ToolCall] | None = None reasoning: str | None = None signature: list[dict[str, Any]] | None = None @@ -63,15 +103,22 @@ class Message(BaseModel): __all__ = [ "AudioContent", - "Content", + "AudioPart", "DocumentContent", + "DocumentPart", "EmbeddingsContent", "ImageContent", + "ImagePart", "JsonValue", "Message", + "MessageContent", + "MessagePart", "RawUsage", "Role", "TextContent", + "TextPart", "ToolCall", + "ToolResultContent", "VideoContent", + "VideoPart", ] diff --git a/templates/modalities/{modality_slug}/client.py.template b/templates/modalities/{modality_slug}/client.py.template index 13971ccd..ef50aa32 100644 --- a/templates/modalities/{modality_slug}/client.py.template +++ b/templates/modalities/{modality_slug}/client.py.template @@ -6,7 +6,7 @@ from asgiref.sync import async_to_sync from celeste.client import ModalityClient from celeste.core import InputType, Modality -from celeste.types import AudioContent, ImageContent, {Content}, VideoContent +from celeste.types import AudioContent, DocumentContent, ImageContent, {Content}, VideoContent from .io import {Modality}Chunk, {Modality}Input, {Modality}Output from .parameters import {Modality}Parameters @@ -31,6 +31,7 @@ class {Modality}Client(ModalityClient[{Modality}Input, {Modality}Output, {Modali image: ImageContent | None, video: VideoContent | None, audio: AudioContent | None, + document: DocumentContent | None = None, ) -> None: """Check model supports the provided media types. @@ -46,6 +47,9 @@ class {Modality}Client(ModalityClient[{Modality}Input, {Modality}Output, {Modali if audio is not None and InputType.AUDIO not in self.model.optional_input_types: msg = f"Model {self.model.id} does not support audio input" raise NotImplementedError(msg) + if document is not None and InputType.DOCUMENT not in self.model.optional_input_types: + msg = f"Model {self.model.id} does not support document input" + raise NotImplementedError(msg) @property def stream(self) -> "{Modality}StreamNamespace": @@ -97,6 +101,7 @@ class {Modality}StreamNamespace: image: ImageContent | None = None, video: VideoContent | None = None, audio: AudioContent | None = None, + document: DocumentContent | None = None, extra_body: dict[str, Any] | None = None, extra_headers: dict[str, str] | None = None, **parameters: Unpack[{Modality}Parameters], @@ -107,8 +112,8 @@ class {Modality}StreamNamespace: async for chunk in client.stream.analyze("Describe", image=img): print(chunk.content) """ - self._client._check_media_support(image=image, video=video, audio=audio) - inputs = {Modality}Input(prompt=prompt, image=image, video=video, audio=audio) + self._client._check_media_support(image=image, video=video, audio=audio, document=document) + inputs = {Modality}Input(prompt=prompt, image=image, video=video, audio=audio, document=document) return self._client._stream( inputs, stream_class=self._client._stream_class(), @@ -151,6 +156,7 @@ class {Modality}SyncNamespace: image: ImageContent | None = None, video: VideoContent | None = None, audio: AudioContent | None = None, + document: DocumentContent | None = None, extra_body: dict[str, Any] | None = None, extra_headers: dict[str, str] | None = None, **parameters: Unpack[{Modality}Parameters], @@ -161,8 +167,8 @@ class {Modality}SyncNamespace: result = client.sync.analyze("Describe", image=img) print(result.content) """ - self._client._check_media_support(image=image, video=video, audio=audio) - inputs = {Modality}Input(prompt=prompt, image=image, video=video, audio=audio) + self._client._check_media_support(image=image, video=video, audio=audio, document=document) + inputs = {Modality}Input(prompt=prompt, image=image, video=video, audio=audio, document=document) return async_to_sync(self._client._predict)(inputs, extra_body=extra_body, extra_headers=extra_headers, **parameters) @property @@ -205,6 +211,7 @@ class {Modality}SyncStreamNamespace: image: ImageContent | None = None, video: VideoContent | None = None, audio: AudioContent | None = None, + document: DocumentContent | None = None, extra_body: dict[str, Any] | None = None, extra_headers: dict[str, str] | None = None, **parameters: Unpack[{Modality}Parameters], @@ -221,7 +228,7 @@ class {Modality}SyncStreamNamespace: """ # Return same stream as async version - __iter__/__next__ handle sync iteration return self._client.stream.analyze( - prompt, image=image, video=video, audio=audio, extra_body=extra_body, extra_headers=extra_headers, **parameters + prompt, image=image, video=video, audio=audio, document=document, extra_body=extra_body, extra_headers=extra_headers, **parameters ) diff --git a/templates/modalities/{modality_slug}/io.py.template b/templates/modalities/{modality_slug}/io.py.template index 015da8da..0c638c61 100644 --- a/templates/modalities/{modality_slug}/io.py.template +++ b/templates/modalities/{modality_slug}/io.py.template @@ -10,7 +10,7 @@ Types are unified per-modality since generate and analyze produce identical outp from pydantic import Field from celeste.io import Chunk, FinishReason, Input, Output, Usage -from celeste.types import AudioContent, ImageContent, {Content}, VideoContent +from celeste.types import AudioContent, DocumentContent, ImageContent, {Content}, VideoContent class {Modality}Input(Input): @@ -21,6 +21,7 @@ class {Modality}Input(Input): image: ImageContent | None = None video: VideoContent | None = None audio: AudioContent | None = None + document: DocumentContent | None = None class {Modality}FinishReason(FinishReason): diff --git a/templates/protocols/{protocol_slug}/tools.py.template b/templates/protocols/{protocol_slug}/tools.py.template new file mode 100644 index 00000000..9c6a4b57 --- /dev/null +++ b/templates/protocols/{protocol_slug}/tools.py.template @@ -0,0 +1,40 @@ +"""{Protocol} protocol tool mappers and shared parsing helpers.""" + +from typing import Any + +from celeste.tools import Tool, ToolMapper + + +# Add ToolMapper subclasses for protocol-native tools (web_search, code_execution, etc.). +# Providers that override the wire format define their own tools.py. +# +# class WebSearchMapper(ToolMapper): +# tool_type = WebSearch +# +# def map_tool(self, tool: Tool) -> dict[str, Any]: +# assert isinstance(tool, WebSearch) +# return {"type": "web_search"} + +TOOL_MAPPERS: list[ToolMapper] = [] + + +# Shared parsing helpers (optional) +# Add when multiple providers share identical response/message formats. +# Short names — file path provides context. +# +# import json +# from celeste.tools import ToolCall, ToolResult +# from celeste.types import Message +# +# def parse_tool_calls(response_data: dict[str, Any]) -> list[ToolCall]: +# """Parse tool calls from {Protocol} API response.""" +# ... +# +# def serialize_messages(messages: list[Message | ToolResult]) -> list[dict[str, Any]]: +# """Serialize text-modality messages to {Protocol} API format.""" +# ... + + +__all__ = [ + "TOOL_MAPPERS", +] diff --git a/templates/providers/{provider_slug}/{api_slug}/tools.py.template b/templates/providers/{provider_slug}/{api_slug}/tools.py.template new file mode 100644 index 00000000..7e05d0e2 --- /dev/null +++ b/templates/providers/{provider_slug}/{api_slug}/tools.py.template @@ -0,0 +1,21 @@ +"""{Provider} {Api} API tool mappers.""" + +from typing import Any + +from celeste.tools import Tool, ToolMapper, WebSearch + + +class WebSearchMapper(ToolMapper): + """Map WebSearch to {Provider} {Api} wire format.""" + + tool_type = WebSearch + + def map_tool(self, tool: Tool) -> dict[str, Any]: + assert isinstance(tool, WebSearch) + # TODO: Map WebSearch fields to provider's wire format. + return {"type": "web_search"} + + +TOOL_MAPPERS: list[ToolMapper] = [WebSearchMapper()] + +__all__ = ["TOOL_MAPPERS", "WebSearchMapper"] diff --git a/tests/unit_tests/test_io.py b/tests/unit_tests/test_io.py index ad16850f..708ab968 100644 --- a/tests/unit_tests/test_io.py +++ b/tests/unit_tests/test_io.py @@ -4,7 +4,9 @@ from celeste.artifacts import AudioArtifact, ImageArtifact, VideoArtifact from celeste.constraints import ( + AudioConstraint, Bool, + DocumentsConstraint, ImageConstraint, Str, VideoConstraint, @@ -80,6 +82,18 @@ def test_get_constraint_input_type_with_videos_constraint(self) -> None: result = get_constraint_input_type(constraint) assert result == InputType.VIDEO + def test_get_constraint_input_type_with_audio_constraint(self) -> None: + """Test that get_constraint_input_type extracts InputType from AudioConstraint.""" + constraint = AudioConstraint() + result = get_constraint_input_type(constraint) + assert result == InputType.AUDIO + + def test_get_constraint_input_type_with_documents_constraint(self) -> None: + """Test that get_constraint_input_type extracts InputType from DocumentsConstraint.""" + constraint = DocumentsConstraint() + result = get_constraint_input_type(constraint) + assert result == InputType.DOCUMENT + def test_get_constraint_input_type_with_str_constraint_returns_text(self) -> None: """Test that get_constraint_input_type returns TEXT for Str constraint.""" constraint = Str(min_length=1) diff --git a/tests/unit_tests/test_telemetry_content_events.py b/tests/unit_tests/test_telemetry_content_events.py index 1d7cd6bb..18cc2264 100644 --- a/tests/unit_tests/test_telemetry_content_events.py +++ b/tests/unit_tests/test_telemetry_content_events.py @@ -74,11 +74,38 @@ def test_image_input_emitted_as_reference(self, capture_enabled: None) -> None: assert result is not None parts = json.loads(result["messages"])[0]["parts"] - assert parts[0]["type"] == "text" - image_parts = [p for p in parts if p["type"] == "image"] - assert len(image_parts) == 1 - assert image_parts[0]["uri"] == "https://example.com/img.png" - assert image_parts[0]["mime_type"] == "image/png" + assert parts[0]["type"] == "image" + assert parts[0]["uri"] == "https://example.com/img.png" + assert parts[0]["mime_type"] == "image/png" + assert parts[1]["type"] == "text" + assert parts[1]["content"] == "describe this" + + def test_media_only_input_emitted_as_user_message( + self, capture_enabled: None + ) -> None: + """Media without a prompt still produces a user message.""" + result = telemetry._input_messages_event( + TextInput( + image=ImageArtifact( + url="https://example.com/img.png", mime_type=ImageMimeType.PNG + ) + ) + ) + + assert result is not None + messages = json.loads(result["messages"]) + assert messages == [ + { + "role": "user", + "parts": [ + { + "type": "image", + "uri": "https://example.com/img.png", + "mime_type": "image/png", + } + ], + } + ] class TestOutputMessagesEvent: diff --git a/tests/unit_tests/test_text_media_support_validation.py b/tests/unit_tests/test_text_media_support_validation.py index 11229c17..a11ed41a 100644 --- a/tests/unit_tests/test_text_media_support_validation.py +++ b/tests/unit_tests/test_text_media_support_validation.py @@ -4,12 +4,23 @@ from pydantic import SecretStr from celeste import Model -from celeste.artifacts import DocumentArtifact, ImageArtifact, VideoArtifact +from celeste.artifacts import ( + AudioArtifact, + DocumentArtifact, + ImageArtifact, + VideoArtifact, +) from celeste.auth import AuthHeader -from celeste.constraints import DocumentsConstraint, ImagesConstraint, VideosConstraint +from celeste.constraints import ( + AudioConstraint, + DocumentsConstraint, + ImagesConstraint, + VideosConstraint, +) from celeste.core import InputType, Modality, Operation, Provider from celeste.modalities.text.parameters import TextParameter from celeste.modalities.text.providers.google.client import GoogleTextClient +from celeste.types import ImagePart, Message, Role, TextPart @pytest.fixture @@ -57,6 +68,21 @@ def model_with_document_support() -> Model: ) +@pytest.fixture +def model_with_audio_support() -> Model: + """Model that declares audio support.""" + return Model( + id="test-audio", + provider=Provider.GOOGLE, + display_name="Test Audio", + operations={Modality.TEXT: {Operation.GENERATE, Operation.ANALYZE}}, + streaming=True, + parameter_constraints={ + TextParameter.AUDIO: AudioConstraint(), + }, + ) + + @pytest.fixture def model_without_media_support() -> Model: """Model that declares no media support.""" @@ -79,6 +105,7 @@ def google_auth() -> AuthHeader: def test_model_optional_input_types_computed_from_constraints( model_with_image_support: Model, model_with_video_support: Model, + model_with_audio_support: Model, model_without_media_support: Model, ) -> None: """Verify optional_input_types is correctly computed from parameter_constraints.""" @@ -88,6 +115,9 @@ def test_model_optional_input_types_computed_from_constraints( assert InputType.VIDEO in model_with_video_support.optional_input_types assert InputType.IMAGE not in model_with_video_support.optional_input_types + assert InputType.AUDIO in model_with_audio_support.optional_input_types + assert InputType.IMAGE not in model_with_audio_support.optional_input_types + assert InputType.IMAGE not in model_without_media_support.optional_input_types assert InputType.VIDEO not in model_without_media_support.optional_input_types @@ -207,6 +237,24 @@ def test_check_media_support_allows_document_when_declared( ) +def test_check_media_support_allows_audio_when_declared( + model_with_audio_support: Model, + google_auth: AuthHeader, +) -> None: + """Audio input should be allowed when model declares AudioConstraint.""" + client = GoogleTextClient( + model=model_with_audio_support, + provider=Provider.GOOGLE, + auth=google_auth, + ) + + client._check_media_support( + image=None, + video=None, + audio=AudioArtifact(data=b"test"), + ) + + def test_check_media_support_rejects_document_when_not_declared( model_without_media_support: Model, google_auth: AuthHeader, @@ -240,3 +288,40 @@ def test_check_media_support_allows_none_values( # Should not raise - no media provided client._check_media_support(image=None, video=None, audio=None) + + +def test_check_media_support_rejects_message_image_when_not_declared( + model_without_media_support: Model, + google_auth: AuthHeader, +) -> None: + """Media inside messages should be validated like top-level media.""" + client = GoogleTextClient( + model=model_without_media_support, + provider=Provider.GOOGLE, + auth=google_auth, + ) + message = Message( + role=Role.USER, + content=[TextPart(text="look"), ImagePart(image=ImageArtifact(data=b"test"))], + ) + + with pytest.raises(NotImplementedError, match="does not support image input"): + client._check_media_support(messages=[message]) + + +def test_check_media_support_allows_message_image_when_declared( + model_with_image_support: Model, + google_auth: AuthHeader, +) -> None: + """Message media should pass when the model declares support.""" + client = GoogleTextClient( + model=model_with_image_support, + provider=Provider.GOOGLE, + auth=google_auth, + ) + message = Message( + role=Role.USER, + content=[TextPart(text="look"), ImagePart(image=ImageArtifact(data=b"test"))], + ) + + client._check_media_support(messages=[message]) diff --git a/tests/unit_tests/test_text_multimodal_message_content.py b/tests/unit_tests/test_text_multimodal_message_content.py new file mode 100644 index 00000000..ad827f78 --- /dev/null +++ b/tests/unit_tests/test_text_multimodal_message_content.py @@ -0,0 +1,86 @@ +"""Unit tests for canonical multimodal message content.""" + +import pytest +from pydantic import BaseModel, ValidationError + +from celeste import ( + DocumentPart, + ImagePart, + Message, + Role, + TextPart, +) +from celeste.artifacts import DocumentArtifact, ImageArtifact +from celeste.mime_types import DocumentMimeType, ImageMimeType +from celeste.tools import ToolResult + + +class StructuredPayload(BaseModel): + ok: bool + + +def test_message_content_accepts_ordered_text_and_image_parts() -> None: + message = Message( + role=Role.USER, + content=[ + TextPart(text="describe"), + ImagePart(image=ImageArtifact(data=b"abc", mime_type=ImageMimeType.PNG)), + ], + ) + + dumped = message.model_dump(mode="json") + + assert dumped["content"][0] == {"type": "text", "text": "describe"} + assert dumped["content"][1]["type"] == "image" + assert dumped["content"][1]["image"]["data"] == "YWJj" + assert dumped["content"][1]["image"]["mime_type"] == "image/png" + + +def test_message_content_round_trips_discriminated_document_part() -> None: + message = Message( + role=Role.USER, + content=[ + TextPart(text="summarize"), + DocumentPart( + document=DocumentArtifact( + data=b"abc", + mime_type=DocumentMimeType.PDF, + ) + ), + ], + ) + + restored = Message.model_validate_json(message.model_dump_json()) + + assert isinstance(restored.content, list) + assert isinstance(restored.content[0], TextPart) + assert isinstance(restored.content[1], DocumentPart) + assert restored.content[1].document.data == b"abc" + + +def test_message_content_rejects_raw_artifact() -> None: + with pytest.raises(ValidationError): + Message(role=Role.USER, content=ImageArtifact(data=b"abc")) + + +def test_message_content_rejects_mixed_raw_list() -> None: + with pytest.raises(ValidationError): + Message( + role=Role.USER, + content=["describe", ImageArtifact(data=b"abc")], # type: ignore[list-item] + ) + + +def test_message_content_rejects_structured_payload() -> None: + with pytest.raises(ValidationError): + Message(role=Role.USER, content=StructuredPayload(ok=True)) + + +def test_tool_result_content_remains_structured_payload() -> None: + result = ToolResult( + content=StructuredPayload(ok=True), + tool_call_id="call_123", + name="structured_tool", + ) + + assert result.content == StructuredPayload(ok=True) diff --git a/tests/unit_tests/test_text_multimodal_message_request_building.py b/tests/unit_tests/test_text_multimodal_message_request_building.py new file mode 100644 index 00000000..8254fd84 --- /dev/null +++ b/tests/unit_tests/test_text_multimodal_message_request_building.py @@ -0,0 +1,261 @@ +"""Unit tests for provider request building from multimodal messages.""" + +from pydantic import SecretStr + +from celeste import ( + AudioPart, + DocumentPart, + ImagePart, + Message, + MessagePart, + Model, + Role, + TextPart, + ToolCall, + VideoPart, +) +from celeste.artifacts import ( + AudioArtifact, + DocumentArtifact, + ImageArtifact, + VideoArtifact, +) +from celeste.auth import AuthHeader +from celeste.core import Modality, Operation, Provider +from celeste.mime_types import ( + AudioMimeType, + DocumentMimeType, + ImageMimeType, + VideoMimeType, +) +from celeste.modalities.text.io import TextInput +from celeste.modalities.text.providers.anthropic.client import AnthropicTextClient +from celeste.modalities.text.providers.cohere.client import CohereTextClient +from celeste.modalities.text.providers.google.client import GoogleTextClient +from celeste.modalities.text.providers.mistral.client import MistralTextClient +from celeste.modalities.text.providers.openai.client import OpenAITextClient + + +def _model(provider: Provider) -> Model: + return Model( + id="test-model", + provider=provider, + display_name="Test Model", + operations={Modality.TEXT: {Operation.GENERATE}}, + ) + + +def _auth(provider: Provider) -> AuthHeader: + if provider == Provider.GOOGLE: + return AuthHeader(secret=SecretStr("test"), header="x-goog-api-key", prefix="") + if provider == Provider.ANTHROPIC: + return AuthHeader(secret=SecretStr("test"), header="x-api-key", prefix="") + return AuthHeader(secret=SecretStr("test")) + + +def _image() -> ImageArtifact: + return ImageArtifact(data=b"img", mime_type=ImageMimeType.PNG) + + +def _document() -> DocumentArtifact: + return DocumentArtifact(data=b"doc", mime_type=DocumentMimeType.PDF) + + +def _message(*parts: MessagePart) -> Message: + return Message(role=Role.USER, content=list(parts)) + + +def test_openai_message_parts_include_image_and_document_blocks() -> None: + client = OpenAITextClient( + model=_model(Provider.OPENAI), + provider=Provider.OPENAI, + auth=_auth(Provider.OPENAI), + ) + + request = client._init_request( + TextInput( + messages=[ + _message( + TextPart(text="inspect"), + ImagePart(image=_image()), + DocumentPart(document=_document()), + ) + ] + ) + ) + + content = request["input"][0]["content"] + assert content[0] == {"type": "input_text", "text": "inspect"} + assert content[1]["type"] == "input_image" + assert content[1]["image_url"].startswith("data:image/png;base64,") + assert content[2]["type"] == "input_file" + assert content[2]["file_data"].startswith("data:application/pdf;base64,") + + +def test_chat_completions_message_parts_include_image_and_document_blocks() -> None: + client = MistralTextClient( + model=_model(Provider.MISTRAL), + provider=Provider.MISTRAL, + auth=_auth(Provider.MISTRAL), + ) + + request = client._init_request( + TextInput( + messages=[ + _message( + ImagePart(image=_image()), + DocumentPart(document=_document()), + TextPart(text="inspect"), + ) + ] + ) + ) + + content = request["messages"][0]["content"] + assert content[0]["type"] == "image_url" + assert content[0]["image_url"]["url"].startswith("data:image/png;base64,") + assert content[1]["type"] == "document_url" + assert content[1]["document_url"].startswith("data:application/pdf;base64,") + assert content[2] == {"type": "text", "text": "inspect"} + + +def test_chat_completions_assistant_tool_call_serializes_message_parts() -> None: + client = MistralTextClient( + model=_model(Provider.MISTRAL), + provider=Provider.MISTRAL, + auth=_auth(Provider.MISTRAL), + ) + + request = client._init_request( + TextInput( + messages=[ + Message( + role=Role.ASSISTANT, + content=[TextPart(text="checking")], + tool_calls=[ + ToolCall( + id="call_1", + name="lookup", + arguments={"query": "weather"}, + ) + ], + ) + ] + ) + ) + + message = request["messages"][0] + assert message["content"] == [{"type": "text", "text": "checking"}] + assert message["tool_calls"][0]["function"]["name"] == "lookup" + + +def test_google_message_parts_include_all_media_blocks() -> None: + client = GoogleTextClient( + model=_model(Provider.GOOGLE), + provider=Provider.GOOGLE, + auth=_auth(Provider.GOOGLE), + ) + + request = client._init_request( + TextInput( + messages=[ + _message( + ImagePart(image=_image()), + VideoPart( + video=VideoArtifact(data=b"vid", mime_type=VideoMimeType.MP4) + ), + AudioPart( + audio=AudioArtifact(data=b"aud", mime_type=AudioMimeType.MP3) + ), + DocumentPart(document=_document()), + TextPart(text="inspect"), + ) + ] + ) + ) + + parts = request["contents"][0]["parts"] + assert [part.get("inline_data", {}).get("mime_type") for part in parts[:4]] == [ + "image/png", + "video/mp4", + "audio/mpeg", + "application/pdf", + ] + assert parts[-1] == {"text": "inspect"} + + +def test_anthropic_message_parts_include_image_and_document_sources() -> None: + client = AnthropicTextClient( + model=_model(Provider.ANTHROPIC), + provider=Provider.ANTHROPIC, + auth=_auth(Provider.ANTHROPIC), + ) + + request = client._init_request( + TextInput( + messages=[ + _message( + ImagePart(image=_image()), + DocumentPart(document=_document()), + TextPart(text="inspect"), + ) + ] + ) + ) + + content = request["messages"][0]["content"] + assert content[0]["type"] == "image" + assert content[0]["source"]["data"] == "aW1n" + assert content[1]["type"] == "document" + assert content[1]["source"]["data"] == "ZG9j" + assert content[2] == {"type": "text", "text": "inspect"} + + +def test_cohere_message_parts_include_image_block() -> None: + client = CohereTextClient( + model=_model(Provider.COHERE), + provider=Provider.COHERE, + auth=_auth(Provider.COHERE), + ) + + request = client._init_request( + TextInput(messages=[_message(ImagePart(image=_image()), TextPart(text="look"))]) + ) + + content = request["messages"][0]["content"] + assert content[0]["type"] == "image_url" + assert content[0]["image_url"]["url"].startswith("data:image/png;base64,") + assert content[1] == {"type": "text", "text": "look"} + + +def test_openresponses_assistant_tool_call_preserves_message_content() -> None: + client = OpenAITextClient( + model=_model(Provider.OPENAI), + provider=Provider.OPENAI, + auth=_auth(Provider.OPENAI), + ) + + request = client._init_request( + TextInput( + messages=[ + Message( + role=Role.ASSISTANT, + content=[TextPart(text="checking")], + tool_calls=[ + ToolCall( + id="call_1", + name="lookup", + arguments={"query": "weather"}, + ) + ], + ) + ] + ) + ) + + assert request["input"][0] == { + "role": "assistant", + "content": [{"type": "input_text", "text": "checking"}], + } + assert request["input"][1]["type"] == "function_call" + assert request["input"][1]["name"] == "lookup" diff --git a/tests/unit_tests/test_text_tool_results.py b/tests/unit_tests/test_text_tool_results.py index 538344f8..51f4557d 100644 --- a/tests/unit_tests/test_text_tool_results.py +++ b/tests/unit_tests/test_text_tool_results.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, SecretStr -from celeste import Message, Model, Role +from celeste import Model from celeste.auth import AuthHeader from celeste.core import InputType, Modality, Operation, Provider from celeste.modalities.text.io import TextInput @@ -122,7 +122,7 @@ def test_google_tool_result_uses_pydantic_json_object() -> None: assert result == output.model_dump(mode="json") -def test_cohere_message_content_preserves_pydantic_shape() -> None: +def test_cohere_tool_result_preserves_pydantic_shape() -> None: output = _artifact_output() client = CohereTextClient( model=_text_model(Provider.COHERE), @@ -130,8 +130,6 @@ def test_cohere_message_content_preserves_pydantic_shape() -> None: auth=AuthHeader(secret=SecretStr("test")), ) - request = client._init_request( - TextInput(messages=[Message(role=Role.USER, content=output)]) - ) + request = client._init_request(TextInput(messages=[_tool_result(output)])) assert request["messages"][0]["content"] == output.model_dump(mode="json") diff --git a/tests/unit_tests/test_tool_outputs.py b/tests/unit_tests/test_tool_outputs.py new file mode 100644 index 00000000..b73f4387 --- /dev/null +++ b/tests/unit_tests/test_tool_outputs.py @@ -0,0 +1,47 @@ +from celeste.core import InputType +from celeste.mime_types import ImageMimeType +from celeste.tools import ToolError, ToolOutput + + +def test_tool_output_wraps_structured_content_and_metadata() -> None: + output = ToolOutput[list[dict[str, str | None]]]( + content=[ + { + "id": "image-1", + "artifact_type": InputType.IMAGE, + "mime_type": ImageMimeType.PNG, + } + ], + metadata={"provider": "test"}, + ) + + assert output.model_dump(mode="json") == { + "content": [ + { + "id": "image-1", + "artifact_type": "image", + "mime_type": "image/png", + } + ], + "metadata": {"provider": "test"}, + } + + +def test_tool_output_metadata_defaults_to_empty_dict() -> None: + output = ToolOutput[dict[str, bool]](content={"ok": True}) + + assert output.metadata == {} + + +def test_tool_error_wraps_error_content_code_and_metadata() -> None: + error = ToolError[str]( + content="quota exceeded", + code="provider_call_failed", + metadata={"retryable": False}, + ) + + assert error.model_dump(mode="json") == { + "content": "quota exceeded", + "code": "provider_call_failed", + "metadata": {"retryable": False}, + }