From 6583dbc99d9998387dcd20d57134420cf8faa978 Mon Sep 17 00:00:00 2001 From: Zain Hasan Date: Mon, 13 Jul 2026 14:55:46 -0700 Subject: [PATCH 01/11] add DMI skills --- .../SKILL.md | 127 ++++++ .../agents/openai.yaml | 4 + .../references/api-reference.md | 402 ++++++++++++++++++ .../references/cli-reference.md | 306 +++++++++++++ .../references/models-and-configs.md | 265 ++++++++++++ .../references/traffic-routing.md | 343 +++++++++++++++ .../scripts/deploy_model.py | 257 +++++++++++ .../scripts/rollout_new_version.py | 199 +++++++++ .../scripts/upload_custom_model.py | 143 +++++++ 9 files changed, 2046 insertions(+) create mode 100644 skills/together-dedicated-model-inference/SKILL.md create mode 100644 skills/together-dedicated-model-inference/agents/openai.yaml create mode 100644 skills/together-dedicated-model-inference/references/api-reference.md create mode 100644 skills/together-dedicated-model-inference/references/cli-reference.md create mode 100644 skills/together-dedicated-model-inference/references/models-and-configs.md create mode 100644 skills/together-dedicated-model-inference/references/traffic-routing.md create mode 100644 skills/together-dedicated-model-inference/scripts/deploy_model.py create mode 100644 skills/together-dedicated-model-inference/scripts/rollout_new_version.py create mode 100644 skills/together-dedicated-model-inference/scripts/upload_custom_model.py diff --git a/skills/together-dedicated-model-inference/SKILL.md b/skills/together-dedicated-model-inference/SKILL.md new file mode 100644 index 0000000..291dc3e --- /dev/null +++ b/skills/together-dedicated-model-inference/SKILL.md @@ -0,0 +1,127 @@ +--- +name: together-dedicated-model-inference +description: "Deploy and operate models on dedicated GPUs with Together AI's Dedicated Model Inference (DMI, the v2 dedicated endpoints API): beta endpoints, deployments, hardware configs, autoscaling with auto-shutdown, traffic splitting, canary/blue-green/rolling rollouts, A/B tests, shadow experiments, and custom model or LoRA adapter uploads. Reach for it whenever the user mentions together beta endpoints or tg beta commands, client.beta.endpoints, DMI resources like ep_/dep_/cr_/ml_ IDs, or wants production model serving with traffic management on Together AI. Use together-dedicated-endpoints only for the legacy v1 endpoints API." +--- + +# Together Dedicated Model Inference + +## Overview + +Dedicated model inference (DMI) serves a model on reserved single-tenant GPUs. It bills per +minute per running replica (by hardware, not by token), has no hard rate limits, and uses the +same inference API as serverless models. + +The resource model has six parts: **Project → Model → Config → Endpoint → Deployment → Replica**. + +- **Endpoint** — stable inference name; routes traffic across its deployments by weight. +- **Deployment** — binds one model + one config to an endpoint with an autoscaling policy; + runs the replicas. +- **Config** — immutable published revision describing how the model runs (engine, GPU type + and count, optimization profile). The model determines the available configs. +- **Replica** — one model instance on its own dedicated hardware. + +Inference goes to `https://api-inference.together.ai/v1` with the endpoint's **qualified name** +(`/`) as the `model` parameter. Management goes through the +Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; `tg` and +`together` are interchangeable), the `client.beta.*` SDK namespaces, or the `/v2` REST API at +`https://api.together.ai`. + +## When This Skill Wins + +- Deploying a model (public, fine-tuned, or uploaded) on dedicated GPUs via the v2 API +- Anything involving `tg beta endpoints` / `tg beta models` (a.k.a. `together beta ...`) CLI commands +- Anything involving `client.beta.endpoints` / `client.beta.models` SDK namespaces +- Traffic management: splits, rollouts (canary/blue-green/rolling), A/B tests, shadow experiments +- Autoscaling, scale-to-zero/auto-shutdown, deployment lifecycle, monitoring and events +- Uploading custom model weights or LoRA adapters for dedicated serving + +## Hand Off To Another Skill + +- Use `together-dedicated-endpoints` for the **legacy v1 API** (`client.endpoints.create` with + `model=` + `hardware=`, hardware IDs like `1x_nvidia_h100_80gb_sxm`, `together endpoints` + CLI without `beta`). v1 stays supported through end of 2026; new work should target v2. +- Use `together-chat-completions` for serverless inference and request-shaping questions + (the request shape is identical once the endpoint is up). +- Use `together-fine-tuning` for training the model you'll deploy here. +- Use `together-dedicated-containers` for custom Docker runtimes (Sprocket/Jig). +- Use `together-gpu-clusters` for raw multi-node compute. + +## Quick Routing + +- **Deploy a model end to end** + - Start with [scripts/deploy_model.py](scripts/deploy_model.py) + - Read [references/cli-reference.md](references/cli-reference.md) for the one-command CLI path +- **Endpoint/deployment lifecycle from the SDK or API** (create, poll, scale, stop, delete, list filters) + - Read [references/api-reference.md](references/api-reference.md) +- **Split traffic, rollouts, A/B tests, shadow experiments** + - Read [references/traffic-routing.md](references/traffic-routing.md) + - Start with [scripts/rollout_new_version.py](scripts/rollout_new_version.py) for rollouts +- **Choose a model or config, pricing, upload custom weights or LoRA adapters** + - Read [references/models-and-configs.md](references/models-and-configs.md) + - Start with [scripts/upload_custom_model.py](scripts/upload_custom_model.py) +- **Autoscaling and monitoring** + - Read [references/api-reference.md](references/api-reference.md) (Autoscaling, Monitoring sections) + +## Workflow + +1. Pick a model: `tg beta models public` (or upload custom weights first). +2. Optionally pick a config: `tg beta models configs ` (the CLI's `deploy` + auto-picks when there's exactly one). +3. Deploy: `tg beta endpoints deploy --endpoint ` — creates the + endpoint, attaches a deployment, and routes 100% of traffic in one step. +4. Poll until `status.state` is `DEPLOYMENT_STATE_READY` (re-run `tg beta endpoints get `). +5. Send requests to `https://api-inference.together.ai/v1` with the qualified name as `model`. +6. Scale or reconfigure with `tg beta endpoints update `; split traffic, roll out new + versions, or experiment as needed. +7. Clean up: `tg beta endpoints update --min-replicas 0 --max-replicas 0` to stop + billing, or `tg beta endpoints rm ` to delete. + +## High-Signal Rules + +- **Billing runs while replicas run.** Per minute, per replica, by hardware. Always scale to + zero (`min_replicas: 0, max_replicas: 0`) or delete when the user is done, and suggest + `--inactive-timeout` (minutes; auto-stop) for dev workloads. +- **A `READY` deployment serves nothing until it's in the endpoint's traffic split.** The CLI's + `deploy` routes automatically; SDK flows must call `endpoints.update(traffic_split=[...])`. + A `routing_error`/503 on a READY deployment almost always means a missing/zero weight. +- **Management IDs vs inference name.** Management calls take IDs (`ep_`, `dep_`, `cr_`, `ml_`, + `rol_`, `abx_`, `exp_`); inference takes the qualified name `/`. +- **The SDK requires `project_id` on every method** — derive it with `client.whoami().project_id`. + The SDK also takes models/configs as resource names (`projects/{p}/models/{ml}`, + `projects/{p}/configs/{cr}`), while the CLI takes bare IDs. Use the config's own `projectId` + (often a platform project) when building its resource name. +- **Traffic weights are relative capacity, not percentages.** Share = weight × ready replicas. + Prefer shifting traffic by changing replica counts; keep weights stable. +- **Scale-to-zero is all-or-nothing:** `min_replicas: 0` with a positive `max_replicas` is + rejected. `0/0` stops; on a running deployment the floor is otherwise `1`. +- **Metric names come in two disjoint catalogs.** Autoscaling uses `gpu_utilization`, + `inflight_requests`, `ttft`, etc.; rollout gates use `serving_latency`, `router_error_rate`, + etc. The sets are not interchangeable — a name from the wrong catalog is rejected. Charts + live in the dashboard (`https://api.together.ai/endpoints`); there's no metrics query API. +- **Deletion order matters:** stop the deployment (wait for `STOPPED`), remove it from the + traffic split, delete it, then delete the endpoint. The CLI's `rm` smart-deletes by ID + prefix and handles detaching; deleting an endpoint with deployments needs `--force`. +- **Rollouts are the safe way to replace a deployment on live traffic** — create the target + stopped (`0/0`), then `tg beta endpoints rollout --from --canary`. + Metric gates are canary-only and need live traffic to evaluate. +- The `client.beta.*` SDK surface and `together beta` CLI are **beta**: pin a current SDK + release (`uv pip install --upgrade together`) and expect the surface to evolve. + +## Resource Map + +- **SDK/API lifecycle reference**: [references/api-reference.md](references/api-reference.md) +- **CLI reference**: [references/cli-reference.md](references/cli-reference.md) +- **Traffic routing guide**: [references/traffic-routing.md](references/traffic-routing.md) +- **Models, configs, pricing, uploads**: [references/models-and-configs.md](references/models-and-configs.md) +- **Deploy workflow script**: [scripts/deploy_model.py](scripts/deploy_model.py) +- **Rollout workflow script**: [scripts/rollout_new_version.py](scripts/rollout_new_version.py) +- **Custom model upload script**: [scripts/upload_custom_model.py](scripts/upload_custom_model.py) + +## Official Docs + +- [Dedicated Model Inference overview](https://docs.together.ai/docs/dedicated-endpoints/overview) +- [Quickstart](https://docs.together.ai/docs/dedicated-endpoints/quickstart) +- [Concepts](https://docs.together.ai/docs/dedicated-endpoints/concepts) +- [CLI reference: beta endpoints](https://docs.together.ai/reference/cli/endpoints-beta) +- [CLI reference: beta models](https://docs.together.ai/reference/cli/models-beta) +- [Migrate from v1](https://docs.together.ai/docs/dedicated-endpoints/migrate-from-v1) diff --git a/skills/together-dedicated-model-inference/agents/openai.yaml b/skills/together-dedicated-model-inference/agents/openai.yaml new file mode 100644 index 0000000..3ea3ee8 --- /dev/null +++ b/skills/together-dedicated-model-inference/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Together Dedicated Model Inference" + short_description: "Together AI dedicated model inference (v2 endpoints)" + default_prompt: "Use $together-dedicated-model-inference to deploy a model on dedicated Together AI GPUs (beta endpoints/deployments), manage autoscaling, split traffic, run rollouts, A/B tests, or shadow experiments, or upload a custom model or LoRA adapter." diff --git a/skills/together-dedicated-model-inference/references/api-reference.md b/skills/together-dedicated-model-inference/references/api-reference.md new file mode 100644 index 0000000..6bc652f --- /dev/null +++ b/skills/together-dedicated-model-inference/references/api-reference.md @@ -0,0 +1,402 @@ +# Dedicated Model Inference — SDK / API Reference + +## Contents + +- [Basics](#basics) +- [Resource IDs and names](#resource-ids-and-names) +- [Create an Endpoint](#create-an-endpoint) +- [Create a Deployment](#create-a-deployment) +- [Poll Deployment Status](#poll-deployment-status) +- [Deployment States](#deployment-states) +- [Autoscaling](#autoscaling) +- [Scaling Metrics](#scaling-metrics) +- [Stop / Restart / Auto-Shutdown](#stop--restart--auto-shutdown) +- [Route Traffic](#route-traffic) +- [List, Paginate, Filter](#list-paginate-filter) +- [Delete Resources](#delete-resources) +- [Monitoring](#monitoring) +- [Events Feed](#events-feed) +- [Send Inference Requests](#send-inference-requests) +- [Troubleshooting](#troubleshooting) + +## Basics + +- **Management API**: `https://api.together.ai` under `/v2/` (for example + `/v2/projects/{project_id}/endpoints/{endpoint_id}/deployments`). Auth is + `Authorization: Bearer $TOGETHER_API_KEY`. +- **Inference API**: `https://api-inference.together.ai/v1` — same request shapes as serverless. +- **SDK namespaces**: `client.beta.endpoints` (with `.deployments`, `.rollouts`, + `.ab_experiments`, `.shadow_experiments`) and `client.beta.models` (with `.configs`, + `.remote_uploads`). TypeScript uses the camelCase equivalents (`client.beta.endpoints`, + `abExperiments`, `shadowExperiments`, `remoteUploads`). +- **Every SDK method requires `project_id`** (the CLI infers it from the API key): + +```python +from together import Together + +client = Together() +project_id = client.whoami().project_id +``` + +- Wire format is camelCase JSON; the Python SDK uses snake_case arguments and attributes. + +## Resource IDs and names + +| Resource | ID format | Notes | +| --- | --- | --- | +| Project | `proj_...` | Scoped to your API key; get it from `whoami`. | +| Model | `ml_...` | Public base models and uploaded models both use this format. | +| Model revision | `rv_...` | Optional pin; omitting it pins the latest revision at create time. | +| Config | `cr_...` | A config revision. Configs are immutable; each revision has a new ID. | +| Endpoint | `ep_...` | Inference uses the qualified name `/`. | +| Deployment | `dep_...` | Qualified name `//`. | +| Rollout | `rol_...` | See [traffic-routing.md](traffic-routing.md). | +| A/B experiment | `abx_...` | See [traffic-routing.md](traffic-routing.md). | +| Shadow experiment / target | `exp_...` / `shet_...` | See [traffic-routing.md](traffic-routing.md). | +| Upload job | `job_...` | See [models-and-configs.md](models-and-configs.md). | +| Architecture (catalog) | `arch_...` | Supported-models catalog entries. | + +The SDK references models and configs by **resource name**, not bare ID: + +- Model: `projects/{project_id}/models/{model_id}[/revisions/{revision_id}]` +- Config: `projects/{project_id}/configs/{config_revision_id}` + +`{project_id}` is the **owning** project — for public models and published configs that's +usually a platform project, not yours. List/retrieve responses return these resource names, so +copy them rather than reassembling from IDs. + +## Create an Endpoint + +An endpoint is a logical grouping of deployments and the stable name your application calls. +The platform prepends your project slug: an endpoint named `my-endpoint` in project slug +`acme` gets the qualified name `acme/my-endpoint`. Names are immutable. + +```python +endpoint = client.beta.endpoints.create( + project_id=project_id, + name="my-endpoint", +) +print(endpoint.id, endpoint.name) # ep_abc123 acme/my-endpoint +``` + +```typescript +const endpoint = await client.beta.endpoints.create({ + projectId, + name: 'my-endpoint', +}); +``` + +There is no standalone create-endpoint CLI command — `tg beta endpoints deploy` creates +the endpoint and its first deployment together. + +## Create a Deployment + +A deployment binds a model and config to an endpoint with an autoscaling policy. + +```python +model = f"projects/{project_id}/models/ml_abc123" # owning project's ID +config = "projects/proj_platform/configs/cr_abc123" # use the config's own projectId + +deployment = client.beta.endpoints.deployments.create( + "ep_abc123", + project_id=project_id, + name="my-deployment", + model=model, + config=config, + autoscaling={"min_replicas": 1, "max_replicas": 2}, +) +``` + +```shell +curl -X POST "https://api.together.ai/v2/projects/$PROJECT_ID/endpoints/ep_abc123/deployments" \ + -H "Authorization: Bearer $TOGETHER_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "my-deployment", + "model": "projects/'"$PROJECT_ID"'/models/ml_abc123", + "config": "projects/proj_platform/configs/cr_abc123", + "autoscaling": {"minReplicas": 1, "maxReplicas": 2} + }' +``` + +### Create request fields + +| Field | Required | Description | +| --- | --- | --- | +| `name` | Yes | Deployment name within the endpoint. | +| `model` | Yes | Model resource name. Omit `/revisions/{rv}` to pin the latest revision at create time. | +| `config` | Yes | Config-revision resource name. Hardware and engine are fixed for the deployment's life. | +| `autoscaling` | Yes | Replica bounds + optional scaling metrics. See [Autoscaling](#autoscaling). | +| `enable_lora` | No | Enable LoRA adapter loading. Toggling later requires a redeploy. | +| `inactive_timeout` | No | Minutes of inactivity before auto-stop; `0` disables. | + +The speculator (draft model for speculative decoding) comes from the config and is pinned at +creation — it cannot be set on the deployment. + +**A new deployment receives no traffic until it's in the endpoint's traffic split** — see +[Route Traffic](#route-traffic). + +## Poll Deployment Status + +The CLI's `get` prints each deployment's state and ready/desired replica counts alongside the +endpoint, so re-running `tg beta endpoints get ep_abc123` works for quick polling. The full +status fields below are SDK/API. + +```python +d = client.beta.endpoints.deployments.retrieve( + "dep_abc123", project_id=project_id, endpoint_id="ep_abc123" +) +print(d.status.state, d.status.message) +``` + +| Field | Description | +| --- | --- | +| `desiredReplicas` | Target replica count from autoscaling. | +| `status.scheduledReplicas` | Replicas placed on clusters (can trail desired while capacity is found). | +| `status.readyReplicas` | Replicas actively serving. | +| `status.message` | Human-readable explanation of the current stage or cause. | +| `status.state` | See below. | + +## Deployment States + +The API returns fully qualified enums (`DEPLOYMENT_STATE_READY`); filters must use the full name. + +| State | Meaning | +| --- | --- | +| `PROVISIONING` | Scheduler is placing replicas (`Scheduling replicas`). | +| `SCALING` | Replicas starting or draining toward the desired count. | +| `READY` | All replicas healthy and serving. Still needs a traffic-split entry to receive requests. | +| `DEGRADED` | Below requested capacity or blocked by a transient issue; usually self-resolves. | +| `STOPPING` | Draining after a stop; settles to `STOPPED` (or `FAILED` on teardown failure). | +| `STOPPED` | Zero replicas. Not billing, not serving. Does not restart on its own. | +| `FAILED` | Terminal; `status.message` explains why. A deployment that never reaches `READY` within six hours is marked `FAILED`. | + +## Autoscaling + +Two bounds control replica count: + +- `min_replicas` — floor; runs (and bills) continuously. At least `1` on a running deployment. +- `max_replicas` — ceiling; caps maximum cost. + +Rules: + +- `min == max` — fixed size, autoscaling off. +- `min < max` — autoscaling on. With no metric specified, the platform scales on concurrent + in-flight requests per replica (default target 8). +- `0 / 0` — stops the deployment. `min 0` with a positive max is **rejected**. + +```python +client.beta.endpoints.deployments.update( + "dep_abc123", + project_id=project_id, + endpoint_id="ep_abc123", + autoscaling={"min_replicas": 1, "max_replicas": 4}, +) +``` + +CLI equivalent: `tg beta endpoints update dep_abc123 --min-replicas 1 --max-replicas 4`. + +Optional windows (settable at deploy or update time via CLI flags): `scale_up_window` (metric must stay +above target this long before adding replicas), `scale_down_window` (cooldown between +scale-downs), `scale_to_zero_window` (idle time before scaling to zero). Scale-up reacts fast; +scale-down is deliberately slow so bursty traffic doesn't pay a cold start on every trough. + +New replicas **cold start**: placement → weight download → engine load → warmup. Minutes for +small models, longer for large ones. Keep `min_replicas >= 1` to avoid cold starts on the +first request; raise it ahead of known traffic spikes. + +## Scaling Metrics + +Each `scaling_metrics` entry sets `name`, `type`, and `target`: + +| `name` | `type` | `target` meaning | +| --- | --- | --- | +| `inflight_requests` | `METRIC_TARGET_TYPE_AVERAGE_VALUE` | Concurrent in-flight requests per replica (default metric, target 8). | +| `gpu_utilization` | `METRIC_TARGET_TYPE_UTILIZATION` | GPU compute %, 0–100. | +| `token_utilization` | `METRIC_TARGET_TYPE_UTILIZATION` | KV-cache utilization %, 0–100. | +| `cache_hit_rate` | `METRIC_TARGET_TYPE_UTILIZATION` | Prompt-cache hit rate %, 0–100. | +| `throughput_per_replica` | `METRIC_TARGET_TYPE_AVERAGE_VALUE` | Tokens/sec per replica. | +| `ttft` | `METRIC_TARGET_TYPE_VALUE` | Time to first token, ms. | +| `decoding_speed` | `METRIC_TARGET_TYPE_VALUE` | Time per output token, ms. | +| `e2e_latency` | `METRIC_TARGET_TYPE_VALUE` | End-to-end request latency, ms. | + +```python +client.beta.endpoints.deployments.update( + "dep_abc123", + project_id=project_id, + endpoint_id="ep_abc123", + autoscaling={ + "min_replicas": 1, + "max_replicas": 4, + "scaling_metrics": [ + {"name": "gpu_utilization", "type": "METRIC_TARGET_TYPE_UTILIZATION", "target": 70} + ], + }, +) +``` + +CLI equivalent: `tg beta endpoints update dep_abc123 --min-replicas 1 --max-replicas 4 +--scaling-metrics '[{"name":"gpu_utilization","type":"METRIC_TARGET_TYPE_UTILIZATION","target":70}]'`. + +Caveats: + +- `throughput_per_replica`, `ttft`, and `decoding_speed` are recorded **only for streaming + responses** — scaling on them without `"stream": true` traffic won't work. +- Latency metrics default to p95 over the measurement window; set `"percentile"` to `"p50"`, + `"p90"`, `"p95"`, or `"p99"` to change it. +- These names are the **autoscaling catalog**, distinct from the rollout-gate catalog + (`serving_latency`, `router_error_rate`, ...) — the two are not interchangeable. + +## Stop / Restart / Auto-Shutdown + +```python +# Stop (drains, then STOPPED; billing stops) +client.beta.endpoints.deployments.update( + "dep_abc123", project_id=project_id, endpoint_id="ep_abc123", + autoscaling={"min_replicas": 0, "max_replicas": 0}, +) + +# Restart (first replica cold-starts) +client.beta.endpoints.deployments.update( + "dep_abc123", project_id=project_id, endpoint_id="ep_abc123", + autoscaling={"min_replicas": 1, "max_replicas": 2}, +) + +# Auto-shutdown after 30 idle minutes +client.beta.endpoints.deployments.update( + "dep_abc123", project_id=project_id, endpoint_id="ep_abc123", + inactive_timeout=30, +) +``` + +CLI equivalents: `tg beta endpoints update dep_abc123 --min-replicas 0 --max-replicas 0` +(stop), `--min-replicas 1 --max-replicas 2` (restart), `--inactive-timeout 30` (auto-shutdown). + +A stopped deployment keeps its configuration and traffic-split weight (the weight reapplies +when it scales back up), but never restarts on its own. + +## Route Traffic + +Traffic is routed by updating the endpoint's `traffic_split` (SDK/API only; the CLI's `deploy` +sets it automatically for the first deployment): + +```python +client.beta.endpoints.update( + "ep_abc123", + project_id=project_id, + traffic_split=[{"deployment_id": "dep_abc123", "weight": 1}], +) +``` + +Weights are relative capacity: share = weight × ready replicas. Full semantics, plus rollouts, +A/B tests, and shadow experiments, are in [traffic-routing.md](traffic-routing.md). + +## List, Paginate, Filter + +```python +client.beta.endpoints.list(project_id=project_id) +client.beta.endpoints.retrieve("ep_abc123", project_id=project_id) +client.beta.endpoints.deployments.list("ep_abc123", project_id=project_id) +``` + +| Parameter | Description | +| --- | --- | +| `limit` | Max results (max 500, default 50). | +| `after` | Opaque cursor from the previous response's `next_cursor`. | +| `order_by` | `created_at` or `updated_at`, optionally with ` asc` / ` desc`. | +| `filter` | Expression; see below. | + +Filter expressions support `=`, `!=`, `<`, `<=`, `>`, `>=`, `AND`, `OR`, `NOT`, double-quoted +strings, and RFC-3339 timestamps. Endpoint lists filter on `created_at` / `updated_at`; +deployment lists also support `state` (fully qualified enum) and `model` (resource name, +matched regardless of revision): + +```python +client.beta.endpoints.deployments.list( + "ep_abc123", + project_id=project_id, + filter='state = "DEPLOYMENT_STATE_READY" AND created_at > "2026-01-01T00:00:00Z"', + order_by="created_at desc", +) +``` + +Invalid syntax or unknown fields return `400` with `invalid filter expression`. + +## Delete Resources + +Deletion is permanent, and a deployment must be stopped first. Order: + +1. Scale the deployment to `0/0`; wait for `DEPLOYMENT_STATE_STOPPED`. +2. Remove it from the endpoint's traffic split. +3. Delete the deployment. +4. Delete the endpoint once it has no deployments. + +```python +client.beta.endpoints.update("ep_abc123", project_id=project_id, traffic_split=[]) +client.beta.endpoints.deployments.delete("dep_abc123", project_id=project_id, endpoint_id="ep_abc123") +client.beta.endpoints.delete("ep_abc123", project_id=project_id) +``` + +The CLI's `rm` smart-deletes by ID prefix and auto-detaches from traffic splits and +experiments; see [cli-reference.md](cli-reference.md). + +## Monitoring + +DMI records latency, throughput, replica-count, and utilization metrics for every endpoint +and deployment — the same series that drive autoscaling and rollout gates. Two surfaces: + +- **Analytics dashboard** — `https://api.together.ai/endpoints` shows per-endpoint charts. + Use it to monitor at a glance and to compare deployments during a rollout or A/B test. +- **Events feed** — the audit trail of lifecycle changes (below). + +There is no user-facing metrics query API. Metric *names* appear only in the two write-side +catalogs — autoscaling metrics (`gpu_utilization`, `inflight_requests`, ...) and rollout gate +metrics (`serving_latency`, `router_error_rate`, ...) — and the two sets are not +interchangeable. + +## Events Feed + +Each endpoint has an audit feed (newest first) merging endpoint- and deployment-scoped events: +scale-ups, traffic shifts, readiness changes, rollout pauses. + +```python +events = client.beta.endpoints.list_events("ep_abc123", project_id=project_id, limit=50) +``` + +Optional filters: `types` (event-type strings), `since` / `until` (time range), `subject_id` +(for example `rol_abc123` for one rollout's audit trail), `deployment_ids` (scope to specific +deployments), `limit` / `after` (max 500, default 50). + +## Send Inference Requests + +Same shared inference API as serverless — function calling, structured outputs, and streaming +work if the underlying model supports them. Prompt caching is on by default. + +```python +from together import Together + +client = Together(base_url="https://api-inference.together.ai/v1") + +response = client.chat.completions.create( + model="your-project-slug/my-endpoint", # qualified name, not ep_ ID + messages=[{"role": "user", "content": "What is 2+2?"}], + max_tokens=30, +) +``` + +## Troubleshooting + +- **`routing_error` / 503 on a `READY` deployment** — the deployment isn't in the traffic + split with a non-zero weight, or it has zero ready replicas. +- **`DEGRADED` with `Cannot place replicas: insufficient GPU capacity`** — hardware for the + config is constrained. The scheduler keeps retrying; compare `status.scheduledReplicas` to + `desiredReplicas`. Request fewer replicas or pick a smaller-footprint config. +- **`DEGRADED` with `Startup stalled` / `Not ready`** — a placed replica is booting or hit a + startup failure; the detail follows the colon in `status.message`. +- **`FAILED` with `Timed out waiting for readiness`** — no replica provisioned within six + hours. Read the stall cause in `status.message`; restart for a fresh budget. +- **Model not deployable** — not every model is supported; a fine-tune deploys only if its + base model is supported. See [models-and-configs.md](models-and-configs.md). +- **`400 invalid filter expression`** — check filter field names and quoting. +- **`409` on update/delete** — stale `etag` (shadow experiments and other etag-guarded + resources). Re-read to get the current etag and retry. diff --git a/skills/together-dedicated-model-inference/references/cli-reference.md b/skills/together-dedicated-model-inference/references/cli-reference.md new file mode 100644 index 0000000..9c82e4c --- /dev/null +++ b/skills/together-dedicated-model-inference/references/cli-reference.md @@ -0,0 +1,306 @@ +# Dedicated Model Inference — CLI Reference + +The Together CLI is the fastest path for DMI. It's intent-based: each command expresses a goal +and wires up the underlying resources. + +## Contents + +- [Installation and invocation](#installation-and-invocation) +- [Command tree](#command-tree) +- [endpoints deploy](#endpoints-deploy) +- [endpoints ls / get](#endpoints-ls--get) +- [endpoints update](#endpoints-update) +- [endpoints rm (smart delete)](#endpoints-rm-smart-delete) +- [endpoints rollout](#endpoints-rollout) +- [endpoints ab (A/B tests)](#endpoints-ab-ab-tests) +- [endpoints shadow (shadow experiments)](#endpoints-shadow-shadow-experiments) +- [models commands](#models-commands) +- [What the CLI can't do](#what-the-cli-cant-do) + +## Installation and invocation + +```bash +uv tool install "together[cli]" +tg --version # check; upgrade with: uv tool upgrade together +``` + +`tg` and `together` are interchangeable entry points (docs use `tg`). Project scoping: the CLI +uses the project associated with your API key, overridable with the `TOGETHER_PROJECT_ID` env +var or `--project` on any command. Every command accepts `--json` for machine-readable output. + +## Command tree + +```text +tg beta endpoints + deploy Deploy a model: create endpoint + deployment + route traffic in one step + ls List endpoints in the project + get Get endpoint details (also the default command: tg beta endpoints ep_abc123) + update Update a deployment: replica bounds, scaling metrics, auto-shutdown + rm Smart-delete any endpoint/deployment/experiment by ID prefix + rollout Start or control a rollout (canary / blue-green / rolling) + ab Start an A/B test (creates the variant deployment) + shadow Start a shadow experiment (creates the shadow deployment) + +tg beta models + public List platform-supported public models (the deployable catalog) + list | ls List models in your project (uploaded/fine-tuned) + get Get a model + configs List published configs for a model + create Register a custom model/adapter record + upload Upload local weights to a model record + ls-files List files in a model or adapter + ls-revisions List revisions for a model + update / delete + remote-uploads create | retrieve | list (import from Hugging Face or presigned URL) +``` + +## endpoints deploy + +Creates (or reuses) an endpoint, attaches a new deployment, and routes 100% of traffic to it. +Returns as soon as resources are created; the deployment provisions in the background. + +```bash +tg beta endpoints deploy --endpoint [flags] +``` + +`` (positional) accepts a model ID (`ml_...`) or a model name (public catalog name or a +model in your project). `--endpoint` takes a new name (creates the endpoint) or an existing +name/ID (adds a deployment to it). + +| Flag | Default | Description | +| --- | --- | --- | +| `--config` | auto | Config ID (`cr_...`). Auto-selected when the model has exactly one; use `tg beta models configs ` to list. | +| `--deployment-name` | derived | Deployment name; defaults to model name + short suffix. | +| `--min-replicas` / `--max-replicas` | 1 / 1 | Replica bounds. Pass a range (e.g. 1/10) to autoscale. | +| `--scale-up-window` | — | Seconds the metric must stay above target before adding replicas. | +| `--scale-down-window` | — | Cooldown seconds between scale-downs. | +| `--scale-to-zero-window` | — | Idle time before scaling to zero replicas. | +| `--inactive-timeout` | — | Minutes of inactivity before auto-stop; `0` disables. | +| `--model-revision` | latest | Model revision ID (`rv_...`) to pin. | +| `--placement` | — | Placement profile to use. | +| `--placement.regions` | — | Comma-separated inline placement regions (alternative to a profile). | +| `--placement.constraint` | — | `required` or `preferred` — how strictly to enforce inline placement. | +| `--placement.hipaa` | off | Require HIPAA-eligible placement. | +| `--enable-lora` | off | Run the multi-LoRA kernel so adapters hot-load. Toggling later needs a redeploy. | + +```bash +# Deploy a public model, single replica +tg beta endpoints deploy ml_CbJNwQC2ZqCU2iFT3mrCh --endpoint my-endpoint + +# Explicit config, autoscaling 1-10, auto-stop after 30 idle minutes +tg beta endpoints deploy ml_CbJNwQC2ZqCU2iFT3mrCh \ + --endpoint my-endpoint \ + --config cr_CbzGdmn14t3HYrXXitmKa \ + --min-replicas 1 --max-replicas 10 \ + --inactive-timeout 30 +``` + +Output includes the endpoint ID (`ep_...`), deployment ID (`dep_...`), and the **qualified +name** (`your-project-slug/my-endpoint`) — the value for the inference `model` parameter. + +## endpoints ls / get + +```bash +tg beta endpoints ls [--limit N] [--after CURSOR] [--org] [--public] +tg beta endpoints get ep_abc123 +``` + +`get` shows endpoint detail including the traffic split and, for each deployment, its state +and ready/desired replica counts — re-running it is a quick way to poll a deployment as it +comes up. Passing an endpoint ID with no subcommand (`tg beta endpoints ep_abc123`) runs `get`. +For the full status fields (scheduled replicas, status message) or deployment lists with +`filter`/`order_by`, use the SDK/API — see [api-reference.md](api-reference.md). + +## endpoints update + +Updates a **deployment's** parameters. Pass the deployment ID (`dep_...`); the CLI resolves +its parent endpoint automatically. At least one option is required. + +```bash +# Scale replica bounds +tg beta endpoints update dep_abc123 --min-replicas 2 --max-replicas 4 + +# Stop (scale to zero; billing stops) / restart +tg beta endpoints update dep_abc123 --min-replicas 0 --max-replicas 0 +tg beta endpoints update dep_abc123 --min-replicas 1 --max-replicas 2 + +# Auto-shutdown after 30 idle minutes +tg beta endpoints update dep_abc123 --inactive-timeout 30 + +# Scale on a specific metric +tg beta endpoints update dep_abc123 --min-replicas 1 --max-replicas 4 \ + --scaling-metrics '[{"name":"gpu_utilization","type":"METRIC_TARGET_TYPE_UTILIZATION","target":70}]' +``` + +| Flag | Description | +| --- | --- | +| `--name` | Rename the deployment. | +| `--min-replicas` / `--max-replicas` | Updated replica bounds. | +| `--scale-up-window` / `--scale-down-window` / `--scale-to-zero-window` | Autoscaling stabilization windows. | +| `--scaling-metrics` | Autoscaling metrics as a JSON array (see api-reference.md, Scaling Metrics). | +| `--inactive-timeout` | Minutes of inactivity before auto-stop; `0` disables. | +| `--etag` | ETag for optimistic concurrency. | + +Notes: + +- `update` targets deployments, not endpoints — there is still no CLI command to edit an + endpoint's traffic split. +- LoRA loading can't be changed after a deployment is created — redeploy with + `deploy --enable-lora` to turn it on or off. + +## endpoints rm (smart delete) + +Resolves the resource by ID prefix and deletes it: endpoint (`ep_`), deployment (`dep_`), A/B +experiment (`abx_`), shadow experiment (`exp_`). + +```bash +tg beta endpoints rm dep_abc123 # delete a deployment (must be stopped) +tg beta endpoints rm ep_abc123 # delete an endpoint with no deployments +tg beta endpoints rm ep_abc123 --force # tear down deployments too +tg beta endpoints rm abx_abc123 # end an A/B test (traffic returns to control) +tg beta endpoints rm exp_abc123 # stop a shadow experiment +``` + +Deleting a deployment auto-detaches it from the traffic split and any experiments. A running +deployment must be stopped first (scale to `0/0`, wait for `STOPPED`). `rm` does not accept +rollout IDs (`rol_`) — delete rollouts from the SDK/API. + +## endpoints rollout + +One command both starts and controls rollouts. (This command was previously named `promote`; +older examples showing `tg beta endpoints promote` map one-to-one onto `rollout`.) To start: +pass the **target** deployment ID +(create it stopped, `0/0` bounds) and a strategy flag. + +```bash +# Canary (default ladder 10,50,100 over 10m intervals) +tg beta endpoints rollout dep_target456 --from dep_source123 --canary + +# Custom canary ladder +tg beta endpoints rollout dep_target456 --from dep_source123 \ + --canary --steps 25,50,75,100 --interval 10m + +# Blue-green (single cutover; the default strategy) / rolling (capacity-preserving swap) +tg beta endpoints rollout dep_target456 --from dep_source123 --blue-green +tg beta endpoints rollout dep_target456 --from dep_source123 --rolling + +# Keep the source scaled up after completion instead of draining it +tg beta endpoints rollout dep_target456 --from dep_source123 --canary --source-cleanup keep +``` + +`--from` is inferred from the traffic split when omitted. To control a running rollout, pass +the **rollout** ID (`rol_...`) and a control flag: + +```bash +tg beta endpoints rollout rol_abc123 --pause --reason "holding for review" +tg beta endpoints rollout rol_abc123 --continue # resume +tg beta endpoints rollout rol_abc123 --complete # fast-forward, all traffic to target +tg beta endpoints rollout rol_abc123 --abort --reason "tail latency regression" +``` + +Metric-gated canaries are SDK/API only (`rollouts.create` with a `metrics` block) — see +[traffic-routing.md](traffic-routing.md). + +## endpoints ab (A/B tests) + +Creates a variant deployment for a model and starts an A/B experiment against a control +deployment that's already serving traffic. The CLI assigns the remainder to the control. + +```bash +tg beta endpoints ab ml_CbJNwQC2ZqCU2iFT3mrCh \ + --control dep_control123 \ + --percent 5 \ + --name sampling-tweak-v1 +``` + +The variant model is the positional argument (same forms as `deploy`). + +| Flag | Description | +| --- | --- | +| `--control` | Control deployment ID currently serving live traffic. Required. | +| `--percent` | Traffic percent for the variant, integer 1–100 (default 1). | +| `--config` | Config for the variant (auto-selected like `deploy`). | +| `--name` | Variant deployment name (defaults to model name + suffix). | +| `--enable-lora` | Enable the multi-LoRA kernel on the variant. | + +Ramping or editing an existing experiment is SDK/API only (`ab_experiments.update` replaces +the whole member set). End the test with `rm abx_...`. + +## endpoints shadow (shadow experiments) + +Creates a shadow deployment from a model and mirrors a sampled fraction of live endpoint +traffic to it. Responses from the shadow are measured and discarded — callers never see them. + +```bash +# Uniform 10% mirror +tg beta endpoints shadow --endpoint ep_abc123 \ + --model ml_CbJNwQC2ZqCU2iFT3mrCh --rate 0.1 --name candidate-v2 + +# Sticky per-user sampling +tg beta endpoints shadow --endpoint ep_abc123 \ + --model ml_CbJNwQC2ZqCU2iFT3mrCh --rate 0.05 --key body.user --name candidate-v2 + +# Adaptive: throttle toward ~5 QPS +tg beta endpoints shadow --endpoint ep_abc123 \ + --model ml_CbJNwQC2ZqCU2iFT3mrCh --target-qps 5.0 --name candidate-v2 +``` + +| Flag | Description | +| --- | --- | +| `--endpoint` | Endpoint serving the live traffic to mirror — accepts an ID or a name. | +| `--model` / `--config` | Model (and optional config) for the shadow deployment. | +| `--rate` | Fraction of traffic to mirror, 0.0–1.0 (uniform / key_based). | +| `--key` | Request-body field for sticky sampling (e.g. `body.user`). | +| `--target-qps` | Target mirrored QPS (adaptive strategies; approximate throttle). | +| `--window` | Sliding window for adaptive QPS observation (default 60s). | +| `--name` | Shadow deployment name. | + +Pass `--rate` or `--target-qps` (one is required). Stop with `rm exp_...`. + +## models commands + +```bash +# Browse the deployable public catalog +tg beta models public --product DEDICATED +tg beta models public --modality TEXT --search qwen + +# List configs published for a model (hardware, GPU count, optimization) +tg beta models configs ml_CbJNwQC2ZqCU2iFT3mrCh + +# Register a custom model or adapter record (name it bare, no org prefix) +tg beta models create gemma-4-31b-it --base-model ml_CbJNwQC2ZqCU2iFT3mrCh + +# Upload local weights (use --type adapter for a LoRA adapter) +tg beta models upload ml_abc123 ./path/to/model-dir +tg beta models upload ml_abc123 ./path/to/adapter-dir --type adapter + +# Import from Hugging Face or a presigned S3 URL (server-side streaming) +tg beta models remote-uploads create ml_abc123 \ + --from https://huggingface.co/your-org/your-repo --token hf_your_token +tg beta models remote-uploads retrieve job_abc123 +tg beta models remote-uploads list + +# Inspect your project's models +tg beta models list +tg beta models ls-files ml_abc123 +tg beta models ls-revisions ml_abc123 +``` + +`--type` values are asymmetric: **write** commands (`upload`, `remote-uploads create`) take +singular `model` / `adapter` (default `model`); **read** commands (`ls-files`, +`remote-uploads retrieve` / `list`) take plural `models` / `adapters` (default `models`). + +Full upload flows (requirements, S3 archive format, polling, deploying the result) are in +[models-and-configs.md](models-and-configs.md). + +## What the CLI can't do + +These are SDK/API-only operations; don't hunt for CLI flags: + +- Edit an endpoint's traffic split (`endpoints update` targets deployments only). +- List deployments or use `filter` / `order_by` (though `get` shows each deployment's state + and replica counts for quick polling). +- Create a metric-gated canary rollout, or delete a rollout. +- Ramp/edit an existing A/B experiment, or update a shadow experiment's sampling. +- Read the events feed. +- Inference itself — point the SDK or curl at `https://api-inference.together.ai/v1`. diff --git a/skills/together-dedicated-model-inference/references/models-and-configs.md b/skills/together-dedicated-model-inference/references/models-and-configs.md new file mode 100644 index 0000000..946f32c --- /dev/null +++ b/skills/together-dedicated-model-inference/references/models-and-configs.md @@ -0,0 +1,265 @@ +# Dedicated Model Inference — Models, Configs, Pricing, and Uploads + +## Contents + +- [Choose a model](#choose-a-model) +- [Deployment profiles](#deployment-profiles) +- [Configs](#configs) +- [Pricing and hardware](#pricing-and-hardware) +- [Upload a custom model](#upload-a-custom-model) +- [Upload a LoRA adapter](#upload-a-lora-adapter) +- [Upload troubleshooting](#upload-troubleshooting) + +## Choose a model + +List the platform-supported catalog (public base models deployable for DMI): + +```bash +tg beta models public --product DEDICATED +tg beta models public --modality TEXT --search qwen +``` + +```python +models = client.beta.models.list_supported(product="PRODUCT_DEDICATED") +for m in models.data: + print(m.id, m.name) + +model = client.beta.models.retrieve_supported("arch_abc123") +``` + +Catalog entries are architectures (`arch_...`) with a catalog-controlled Hugging Face `name` +(e.g. `Qwen/...`), a `displayName`, a `displayType` badge (`chat`, `language`, `code`, +`image`, `embedding`, `rerank`, `moderation`, `audio`, `video`, `transcribe`), and +`deploymentProfiles`. + +Models in **your project** (uploads, fine-tunes) are listed separately: + +```bash +tg beta models list +``` + +```python +models = client.beta.models.list(project_id=project_id) +``` + +Not every model is deployable, and a fine-tuned model deploys only if its base architecture is +supported. Don't hardcode model names — list the catalog and use real IDs. + +## Deployment profiles + +Each supported-model entry carries `deploymentProfiles`: certified model-and-config pairs +Together publishes. Each profile gives you `model` and `config` **resource names you can pass +straight into a deployment create**, plus `gpuType`, `gpuCount`, and `parallelism` (free-form: +`TP8`, `TP4`, `EP`, `PD` — not always a tensor-parallel degree). + +```json +{ + "profileId": "cfg_a", + "config": "projects/proj_cfg/configs/cr_certified", + "model": "projects/proj_weights/models/ml_weight/revisions/rv_snap", + "parallelism": "TP8", + "gpuType": "H100", + "gpuCount": 8 +} +``` + +Note the owning projects in these resource names are platform projects — copy the strings +verbatim rather than substituting your own project ID. + +## Configs + +A config describes how a model runs: inference engine, hardware selectors, optimization +profile. Together publishes configs per model; you pick one when creating a deployment. The +CLI's `deploy` auto-picks when the model has exactly one config. + +```bash +tg beta models configs ml_CbJNwQC2ZqCU2iFT3mrCh +``` + +```python +configs = client.beta.models.configs.list( + project_id=project_id, + reference_model=f"projects/{project_id}/models/{model_id}", +) +``` + +Responses include the config `id` (`cr_...`), `referenceModel`, the owning `projectId` (use it +when building the config resource name), and `selectors`: + +| Selector | Description | Example | +| --- | --- | --- | +| `accelerator_type` | GPU SKU the config targets. | `nvidia-h100-80gb` | +| `accelerator_count` | GPUs per replica. | `1`, `2`, `4`, `8` | +| `optimization` | Serving profile. | `balanced`, `throughput`, `latency` | +| `topology` | Model layout across GPUs. | `aggregated` | + +Picking a config: + +- **Single-GPU, balanced** — good default; lowest per-replica cost. +- **Multi-GPU** — higher throughput / lower latency for large models or heavy traffic, at a + proportionally higher per-replica price. +- **Latency-optimized** — when time to first token matters most. + +Key properties: + +- **Configs are immutable.** New revisions get new `cr_...` IDs; a deployment pins the + revision you selected, so hardware and engine never change underneath it. +- **Hardware is fixed for a deployment's life.** To change hardware, create a new deployment + with a different config and shift traffic over (ideally with a rollout). +- **Speculative decoding** is declared by the config (`draftModel` in the response when + enabled); the deployment's speculator is derived and pinned at create — you can't set it + yourself. Speculative decoding raises average throughput but can add occasional tail-latency + spikes; latency-strict workloads should pick a latency-profile config. + +## Pricing and hardware + +DMI bills **per minute, per running replica, by hardware** — model and token volume don't +affect cost. A replica stops billing as soon as it scales down; a stopped deployment costs +nothing. + +Single-GPU per-hour rates (multi-GPU configs cost proportionally more; check +[the pricing page](https://docs.together.ai/docs/dedicated-endpoints/pricing) for current rates): + +| GPU | Hardware ID | Cost/hour | +| --- | --- | --- | +| H100 80GB | `1xnvidia-h100-80gb` | $5.49 | +| B200 180GB | `1xnvidia-b200-180gb` | $8.99 | + +As a scale reference: one H100 replica running continuously is about $132/day, or roughly +$3,950 over a 30-day month. + +Cost levers: + +- `min_replicas` sets the cost floor (always running); `max_replicas` sets the ceiling. +- `inactive_timeout` (auto-shutdown) stops idle deployments; the next request pays a cold start. +- **On-demand** (per-minute, no commitment) vs **reserved** (committed term, lower effective + rate, guaranteed hardware — contact Together sales). + +DMI vs serverless rule of thumb: DMI wins when a replica stays busy most of the day (fixed +cost spread over high throughput, plus reserved capacity and predictable latency); serverless +wins for low or bursty traffic where a dedicated replica would idle. + +## Upload a custom model + +Serve your own fine-tuned weights. Requirements: + +- **Source**: Hugging Face Hub or an S3 presigned URL (or local files via CLI upload). +- **Architecture**: a fine-tuned variant of a base model Together supports for dedicated + inference — uploads cannot introduce new architectures. +- **Type**: text generation or embedding; **must fit on a single GPU** (multi-GPU uploads not + supported). +- **Format**: standard Hugging Face repo layout compatible with `from_pretrained` + (`config.json`, `*.safetensors`, `tokenizer.json`, ...). +- **S3 archives**: a single `.zip` / `.tar.gz` with the files at the **archive root** (no + nested top-level directory); presigned URL valid for at least 100 minutes. + From inside the model dir: `tar -czvf ../model.tar.gz .` + +### Step 1 — Register the model record + +Every custom model references a supported base model via `base_model_id`. **Name it bare** +(`gemma-4-31b-it`), not org-prefixed (`google/gemma-4-31b-it`) — the platform prepends your +project slug, an org prefix produces a doubled slug, and a doubled name can't be renamed +(delete and re-upload is the only fix). + +```bash +tg beta models create gemma-4-31b-it --base-model ml_CbJNwQC2ZqCU2iFT3mrCh +``` + +```python +model = client.beta.models.create( + project_id=project_id, + model={"name": "gemma-4-31b-it", "base_model_id": "ml_CbJNwQC2ZqCU2iFT3mrCh"}, +) +``` + +Save the returned `id` (`ml_...`). + +### Step 2 — Upload the weights + +Local directory (CLI handles the multipart upload): + +```bash +tg beta models upload ml_abc123 ./path/to/model-dir +``` + +Remote (server-side streaming from Hugging Face or presigned S3 URL; `--token` for gated or +private HF repos): + +```bash +tg beta models remote-uploads create ml_abc123 \ + --from https://huggingface.co/your-org/your-repo --token hf_your_token +``` + +```python +job = client.beta.models.remote_uploads.create( + project_id=project_id, + type="model", + model_id="ml_abc123", + remote_url="https://huggingface.co/your-org/your-repo", + token="hf_your_token", +) +``` + +### Step 3 — Poll until succeeded + +```bash +tg beta models remote-uploads retrieve job_abc123 +tg beta models ls-files ml_abc123 # confirm files landed +``` + +Poll `status` until `REMOTE_UPLOAD_STATUS_SUCCEEDED`. + +### Step 4 — Deploy + +Same as any model — list your project's configs for it and deploy: + +```bash +tg beta endpoints deploy ml_abc123 --endpoint my-custom-model --config cr_abc123 +``` + +## Upload a LoRA adapter + +Same flow as a custom model with `--type adapter` (`type="adapter"` in the SDK). Adapter-specific +requirements: + +- The adapter directory must contain `adapter_config.json` and `adapter_model.safetensors`. +- The adapter must target a supported base model (set via `base_model_id` on the record). +- Adapter versioning isn't supported — re-upload under a new name. + +```bash +tg beta models create my-stsb-lora --base-model ml_CbJNwQC2ZqCU2iFT3mrCh + +# Local +tg beta models upload ml_abc123 ./path/to/adapter-dir --type adapter + +# Remote +tg beta models remote-uploads create ml_abc123 \ + --from https://huggingface.co/your-org/your-adapter --type adapter --token hf_your_token + +# Poll / verify — read commands take the PLURAL --type adapters +tg beta models remote-uploads retrieve job_abc123 --type adapters +tg beta models ls-files ml_abc123 --type adapters +``` + +Note the `--type` asymmetry: write commands (`upload`, `remote-uploads create`) take singular +`model`/`adapter`; read commands (`ls-files`, `remote-uploads retrieve`/`list`) take plural +`models`/`adapters`. + +Deploy the adapter's `ml_...` ID like a base model, using a config for its base model. To +hot-load adapters onto a running deployment, the deployment must have been created with +`--enable-lora` (toggling later requires a redeploy). + +## Upload troubleshooting + +- **"Model not found" during upload** — create the record first (`models create`) and pass + the returned `ml_...` ID to the upload. +- **`base_model_id is required`** — every custom model/adapter must reference a supported + base model; get the ID from `tg beta models public`. +- **"Model name already exists"** — names are unique; pick a new one (no versioning). +- **Missing required files** — adapters need both `adapter_config.json` and + `adapter_model.safetensors` at the source root. +- **Job stuck in `Processing`** — the source usually can't be reached: expired presigned URL, + or HF token lacking repo access. +- **`401`/`403`** — check `TOGETHER_API_KEY`, HF token permissions, and presigned URL expiry. +- **Doubled slug in the catalog** (`your-project/google/gemma...`) — the record was created + with an org-prefixed name; delete and re-create with the bare name. diff --git a/skills/together-dedicated-model-inference/references/traffic-routing.md b/skills/together-dedicated-model-inference/references/traffic-routing.md new file mode 100644 index 0000000..bb3fb52 --- /dev/null +++ b/skills/together-dedicated-model-inference/references/traffic-routing.md @@ -0,0 +1,343 @@ +# Dedicated Model Inference — Traffic Routing + +How an endpoint decides which deployment serves each request, and the four tools for moving +traffic: weighted splits, rollouts, A/B tests, and shadow experiments. + +## Contents + +- [How routing works](#how-routing-works) +- [Stickiness](#stickiness) +- [Traffic splits (weights)](#traffic-splits-weights) +- [Rollouts](#rollouts) + - [Strategies](#strategies) + - [Create and start](#create-and-start) + - [Metric gates (canary only)](#metric-gates-canary-only) + - [Lifecycle controls](#lifecycle-controls) + - [Rollout states](#rollout-states) + - [Failure categories](#failure-categories) +- [A/B tests](#ab-tests) +- [Shadow experiments](#shadow-experiments) +- [Choosing a strategy](#choosing-a-strategy) + +## How routing works + +A deployment — even `READY` — receives no traffic until it's routed to. The endpoint resolves +each request through sampling stages: + +1. **Traffic split** — sample a candidate in proportion to capacity (weight × ready replicas). +2. **A/B test** — if the candidate is an A/B control, re-sample among control + variants by + the test percentages. +3. **Rollout** — if the candidate is an active rollout's source, re-sample between source and + target by the current rollout percentage. +4. **Route** — send to the final deployment. + +The A/B stage runs before the rollout stage, so a deployment can be both an A/B control and a +rollout source. Shadow experiments sit outside this path entirely (copies, not routing). + +## Stickiness + +Routing is deterministic per request: the endpoint derives a **sampling key** and always +routes the same key to the same deployment (while the split is stable). This keeps prompt +caches warm across a conversation. Key precedence: + +1. `prompt_cache_key` request field, if set. +2. Otherwise the `user` field, if set. +3. Otherwise a key derived from request content. + +Set `prompt_cache_key` on requests sharing a prompt prefix (e.g. every turn of one +conversation) to control stickiness. Editing weights, adding/removing deployments, or a +rollout shifting its percentage reassigns some keys. + +## Traffic splits (weights) + +Set by updating the endpoint (SDK/API only): + +```python +client.beta.endpoints.update( + "ep_abc123", + project_id=project_id, + traffic_split=[ + {"deployment_id": "dep_abc123", "weight": 70}, + {"deployment_id": "dep_def456", "weight": 30}, + ], +) +``` + +Semantics that trip people up: + +- **Weights are relative ratios, not percentages** — `.7`/`.3` equals `700`/`300`. +- **Share = weight × ready replicas.** Weight 1 with two replicas draws the same traffic as + weight 2 with one replica. Equal weights with unequal replica counts are not a 50/50 split. +- **Prefer shifting traffic by scaling replicas**, keeping weights as a stable capacity + definition — a deployment's share tracks its ready replicas automatically. +- A deployment gets no traffic if its weight is 0, it's absent from the split, or it has zero + ready replicas. Scaling to zero takes it out of rotation but its weight is remembered and + reapplies on scale-up. +- An endpoint with no routable deployment returns `routing_error`. + +## Rollouts + +A rollout migrates traffic from a **source** deployment to a **target** deployment under the +same endpoint, scaling the target up and draining the source. Use it instead of hand-editing +the split when replacing a deployment on live traffic (new model version, new config/hardware). + +Requirements: + +- Source and target under the same endpoint. +- The source must be referenced in the traffic split (a rollout only shifts traffic already + routed to the source; otherwise it runs but moves nothing). +- Create the target **stopped** (`0/0` bounds) so the rollout scales it up from zero. +- One active rollout per endpoint (`409` on create otherwise). + +### Strategies + +| Strategy | Traffic movement | Use when | +| --- | --- | --- | +| **Canary** | Staged ladder (e.g. 25→50→75→100%), source drains stepwise. | Gradual, metric-gated exposure. Safest for production. | +| **Blue-green** | Single 0→100 cutover once the target is ready. Default. | Atomic flip/rollback; you have capacity to run both at full size briefly. | +| **Rolling** | Replica-by-replica swap (target +1 / source −1 per batch). | Preserve total GPU capacity; no ramp or gate needed. | + +Metric gates are **canary-only** — blue-green and rolling have no soak window, and a +`metrics` block is rejected at create time. + +### Create and start + +CLI (`rollout` creates and starts in one step): + +```bash +tg beta endpoints rollout dep_target456 --from dep_source123 --canary --steps 25,50,75,100 --interval 10m +``` + +SDK (two steps — `create` returns `PENDING`, `start` begins shifting): + +```python +rollout = client.beta.endpoints.rollouts.create( + "ep_abc123", + project_id=project_id, + source_deployment_id="dep_source123", + target_deployment_id="dep_target456", + blue_green={}, # or canary={...} / rolling={...} + source_cleanup="SOURCE_CLEANUP_POLICY_DRAIN", # or keep the source scaled up + final_source_replicas=0, + final_target_replicas=4, # must be >= 1 +) +client.beta.endpoints.rollouts.start(rollout.id, project_id=project_id, endpoint_id="ep_abc123") +``` + +### Metric gates (canary only) + +SDK/API only. After each step's soak (`step_interval`), the gate evaluates the rules and +either advances or pauses (`SYSTEM_PAUSED`) for review. Rules are ANDed. + +```python +rollout = client.beta.endpoints.rollouts.create( + "ep_abc123", + project_id=project_id, + source_deployment_id="dep_source123", + target_deployment_id="dep_target456", + canary={ + "steps": [ + {"traffic": 25, "replicas": 1}, + {"traffic": 50, "replicas": 2}, + {"traffic": 100, "replicas": 4}, + ], + "step_interval": "390s", + }, + metrics=[ + { + "name": "serving_latency", + "stat": "METRIC_STAT_TYPE_PERCENTILE", + "percentile": 95, + "regression_check": { + "max_regression_percent": 50, + "direction": "REGRESSION_DIRECTION_HIGHER_IS_WORSE", + }, + "window": "300s", + } + ], + source_cleanup="SOURCE_CLEANUP_POLICY_DRAIN", + final_source_replicas=0, + final_target_replicas=4, +) +``` + +Gate catalog (`name` must be one of these — anything else, including autoscaling metric +names like `gpu_utilization`, is rejected with `400`): + +| `name` | Type | Measures | `stat` | +| --- | --- | --- | --- | +| `serving_latency` | Histogram | Engine request duration — the preferred latency gate. | `AVG` or `PERCENTILE` | +| `router_latency` | Histogram | Router per-attempt latency. Bimodal — gate on `PERCENTILE` ≥ 95, not the mean. | `AVG` or `PERCENTILE` | +| `router_error_rate` | Ratio | Router 5xx / all responses. | `AVG` | +| `inflight_requests` | Gauge | In-flight requests across the target's pods. | `AVG` | + +Each rule uses exactly one criterion: + +- **`regression_check`** — target vs source; fails when the target is worse by more than + `max_regression_percent` in the given direction. Relative safety ("no more than 50% slower"). +- **`threshold_check`** — target vs an absolute `value` with `THRESHOLD_OPERATOR_GT|GTE|LT|LTE`. + Latency values are **milliseconds** (`30000` = 30 s). Fixed SLOs ("p95 under 30 s"). + +Practical rules: + +- **Gates need live traffic.** With no requests on the target, the gate reports + `METRICS_UNAVAILABLE` and the rollout pauses instead of completing. Keep the endpoint under + load for the whole rollout. +- Use `window: "300s"` for histogram percentiles — shorter windows sit inside ingestion lag + and hold too few samples, causing spurious pauses. +- Keep `step_interval >= window + ~90s` so metrics settle and new replicas warm past their + cold start before the gate samples. + +### Lifecycle controls + +| CLI | SDK | Effect | +| --- | --- | --- | +| `rollout --pause` | `rollouts.pause` | Hold at the current step (honored at the next step boundary). | +| `rollout --continue` | `rollouts.resume` | Resume from the current step (re-evaluates a paused step). | +| `rollout --complete` | `rollouts.promote` | Fast-forward: all traffic to the target. | +| `rollout --abort` | `rollouts.abort` | Roll traffic back to the source. Terminal; only while active. | + +Pass `--reason` / `reason=` on pause and abort to record why. Monitor with +`rollouts.retrieve` (`state`, `current_step`, `current_traffic_percent`, per-step metric +verdicts) or the dashboard's Rollouts tab. + +**After `COMPLETED` there is no rollback** — the source is drained. Revert by running a new +rollout in the reverse direction. Delete pending/terminal rollouts via `rollouts.delete` +(SDK/API; `rm` doesn't take `rol_` IDs). + +### Rollout states + +| State | Meaning | +| --- | --- | +| `PENDING` | Created, not started. | +| `RUNNING` | Actively shifting traffic. | +| `STABILIZING` | Soaking at a step's traffic level. | +| `PAUSED` | Operator-paused; split holds. | +| `SYSTEM_PAUSED` | Platform-paused for review — read `status.condition.category` before acting. | +| `ABORTING` / `ABORTED` | Reverting / reverted to the source (terminal). | +| `COMPLETED` | All traffic on the target (terminal). | + +### Failure categories + +On `SYSTEM_PAUSED` or terminal failure, `status.condition.category` is typed — trust it over +the free-text message: + +| Category | Meaning | Action | +| --- | --- | --- | +| `METRIC_REGRESSION` | Gate found the target worse than the source. | Abort. | +| `HEALTH_REGRESSION` | Health check failed during soak. | Abort. | +| `METRICS_UNAVAILABLE` | Too few samples (usually no live traffic). | Restore load, resume. | +| `TARGET_NOT_READY` | Target didn't become ready in time. | Investigate target, resume or abort. | +| `SOURCE_NOT_DRAINED` | Source replicas didn't drain. | Wait/investigate, resume. | +| `CAPACITY_EXHAUSTED` | Not enough fleet capacity (checked in-rollout, not at create). | Wait; often auto-resumes. | +| `ROUTING_ERROR` | Invalid routing configuration mid-shift. | Fix routing, resume or abort. | +| `DEPENDENCY_OUTAGE` | Platform dependency unreachable. | Wait for recovery, resume. | +| `POLICY_INFEASIBLE` | An autoscaling ceiling dropped below the step's replica floor. | Raise the ceiling or lower the step target, resume. | +| `UNDER_SERVED` | Ready capacity fell below what the split requires. | Restore capacity or reduce the split, resume. | +| `ENTITLEMENT_LAPSED` | Billing/org entitlement lapsed mid-run. | Restore it, resume or abort. | +| `ABORTED_BY_OPERATOR` | Someone aborted. | Terminal; run a new rollout. | +| `INTERNAL` | Unexpected platform error. | Contact support with the rollout ID. | + +## A/B tests + +An A/B experiment holds a **fixed** split between a control deployment and 1–19 variants, +subdividing only the control's share of traffic. Use it to measure a candidate before +promoting; use a rollout to actually migrate. + +Rules: + +- 2–20 members, exactly one control; integer percents in [1, 100] summing to 100. +- **Only the control belongs in the endpoint's traffic split.** A variant with a non-zero + split weight fails the test at start. A control with weight 0 (or absent) means the test + receives no traffic. +- Updating `members` replaces the whole set — resend every member each time. + +CLI (creates the variant deployment too): + +```bash +tg beta endpoints ab ml_CbJNwQC2ZqCU2iFT3mrCh --control dep_control123 --percent 5 --name candidate-v1 +``` + +SDK (create the variant deployment first): + +```python +experiment = client.beta.endpoints.ab_experiments.create( + "ep_abc123", + project_id=project_id, + name="sampling-tweak-v1", + members=[ + {"deployment_id": "dep_control123", "role": "AB_EXPERIMENT_MEMBER_ROLE_CONTROL", "percent": 95}, + {"deployment_id": "dep_variant456", "role": "AB_EXPERIMENT_MEMBER_ROLE_VARIANT", "percent": 5}, + ], +) +``` + +Ramp / add variants / remove variants: `ab_experiments.update` with the full new member set +(SDK/API only). Promote a winner with a rollout +(`rollout dep_variant456 --from dep_control123 --blue-green`), then delete the test +(`rm abx_abc123`) — deletion returns all traffic to the regular split immediately. + +## Shadow experiments + +A shadow experiment mirrors a sampled fraction of live endpoint traffic to one or more target +deployments. Copies are fire-and-forget: measured, discarded, never returned to callers. Use +it to warm a candidate under real load, stress-test a config, or gather latency data risk-free. + +Structure: + +- **Source** — the endpoint plus a sampling strategy. +- **Targets** — deployments under the same endpoint, **excluded from the traffic split** + (weight 0 or unset). Each sampled request is copied to *every* target, so adding targets + multiplies mirrored volume. Up to 100 inline targets at create. + +Sampling strategies (exactly one): + +| Strategy | Behavior | Parameters | +| --- | --- | --- | +| `uniform` | Fixed random fraction of all requests. Predictable volume. | `rate` (0.0–1.0) | +| `key_based` | Fixed fraction of distinct key values; same key always same decision. | `rate`, `key` | +| `adaptive_uniform` | Auto-adjusts rate toward a target mirrored QPS. | `target_qps`, optional `window` | +| `adaptive_key_based` | Sticky per-key sampling throttled toward target QPS. | `target_qps`, `key`, optional `window` | + +`key` names a top-level request-body field (`body.user`, `body.prompt_cache_key`) — not a +nested path. `target_qps` is an approximate throttle that can run higher and grows as the +endpoint scales; use `uniform` when you need a predictable fraction. + +```python +experiment = client.beta.endpoints.shadow_experiments.create( + "ep_abc123", + project_id=project_id, + name="candidate-shadow", # immutable, unique in project, <= 256 chars + source={"endpoint": {"sampling": {"uniform": {"rate": 0.1}}}}, + targets=[{"name": "candidate-v2", "target_deployment_id": "dep_target456"}], +) +``` + +Operating notes: + +- Changes (create/update/delete) take effect within ~30–60 seconds. +- **Pause without deleting**: update `source` with `rate: 0`. +- Updates to `source` are etag-guarded: retrieve first, pass `etag` back, handle `409 ABORTED` + by re-reading. Use `update_mask` (`"source"`, `"description"`). +- Targets are sub-resources (`shadow_experiments.targets.create/list/retrieve/update/delete`). + Removing the last target makes the experiment `INACTIVE` (mirroring stops, config kept). +- If a target is provisioning/stopped/unhealthy, copies are silently dropped (visible only in + the target's shadow metrics) — callers are never affected, and mirroring self-recovers. +- Mirrored requests are never re-sampled; shadow loops are prevented automatically. +- Delete the experiment (`rm exp_abc123`) to stop all mirroring (cascade-deletes targets). + Remove a deployment from experiments before deleting the deployment. + +Multiple experiments can run on one endpoint, each sampling independently (volumes add up). +Prefer more targets in one experiment for apples-to-apples comparisons on identical sampled +requests; prefer separate experiments when candidates need different rates, strategies, or +lifetimes. + +## Choosing a strategy + +| Goal | Tool | +| --- | --- | +| Serve two deployments for redundancy | Traffic split with weights. | +| Replace a deployment on live traffic | Rollout (canary if you want metric gates). | +| Measure a candidate on a fixed slice of user-visible traffic | A/B test. | +| Test a candidate with zero user impact / warm it up | Shadow experiment. | +| Ship the A/B or shadow winner | Rollout from the current deployment to the winner. | diff --git a/skills/together-dedicated-model-inference/scripts/deploy_model.py b/skills/together-dedicated-model-inference/scripts/deploy_model.py new file mode 100644 index 0000000..f8609d6 --- /dev/null +++ b/skills/together-dedicated-model-inference/scripts/deploy_model.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +""" +Together AI Dedicated Model Inference -- Deploy, Poll, Infer, Scale, Clean Up + +Full DMI lifecycle from the SDK: list supported models and configs, create an +endpoint + deployment, route traffic, wait for READY, send a test request, +scale, stop, and delete. + +Usage: + python deploy_model.py models [--search qwen] + python deploy_model.py configs --model ml_abc123 + python deploy_model.py deploy --model ml_abc123 --config cr_abc123 --endpoint my-endpoint + python deploy_model.py status --endpoint ep_abc123 --deployment dep_abc123 + python deploy_model.py infer --name your-project-slug/my-endpoint + python deploy_model.py scale --endpoint ep_abc123 --deployment dep_abc123 --min 1 --max 4 + python deploy_model.py stop --endpoint ep_abc123 --deployment dep_abc123 + python deploy_model.py rm --endpoint ep_abc123 --deployment dep_abc123 + +Requires: + uv pip install --upgrade together # a release with the beta DMI surface (client.beta.*) + export TOGETHER_API_KEY=your_key + +Billing note: replicas bill per minute while running. `stop` (scale to 0/0) +halts billing without deleting anything. +""" + +import argparse +import sys +import time + +from together import Together + +client = Together() +PROJECT_ID = client.whoami().project_id + +INFERENCE_BASE_URL = "https://api-inference.together.ai/v1" + + +def list_supported_models(search: str | None = None) -> list: + """List public base models deployable for dedicated inference.""" + models = client.beta.models.list_supported(product="PRODUCT_DEDICATED") + rows = [m for m in models.data if not search or search.lower() in m.name.lower()] + for m in rows: + print(f" {m.id} {m.name}") + return rows + + +def list_configs(model_id: str) -> list: + """List published configs (hardware/engine/optimization) for a model.""" + configs = client.beta.models.configs.list( + project_id=PROJECT_ID, + reference_model=f"projects/{PROJECT_ID}/models/{model_id}", + ) + for c in configs.data: + selectors = {s.key: s.value for s in (c.selectors or [])} + print( + f" {c.id} {selectors.get('accelerator_count', '?')}x " + f"{selectors.get('accelerator_type', '?')} " + f"optimization={selectors.get('optimization', '?')}" + ) + return configs.data + + +def deploy( + model_id: str, + config_id: str, + config_project_id: str, + endpoint_name: str, + min_replicas: int = 1, + max_replicas: int = 1, + inactive_timeout: int | None = None, +): + """Create an endpoint, attach a deployment, and route 100% of traffic to it. + + This is what the CLI's `tg beta endpoints deploy` does in one command. + """ + endpoint = client.beta.endpoints.create(project_id=PROJECT_ID, name=endpoint_name) + print(f"Created endpoint: {endpoint.id} ({endpoint.name})") + + deployment = client.beta.endpoints.deployments.create( + endpoint.id, + project_id=PROJECT_ID, + name=f"{endpoint_name}-deployment", + model=f"projects/{PROJECT_ID}/models/{model_id}", + config=f"projects/{config_project_id}/configs/{config_id}", + autoscaling={"min_replicas": min_replicas, "max_replicas": max_replicas}, + inactive_timeout=inactive_timeout, + ) + print(f"Created deployment: {deployment.id}") + + # A READY deployment still serves nothing until it's in the traffic split. + client.beta.endpoints.update( + endpoint.id, + project_id=PROJECT_ID, + traffic_split=[{"deployment_id": deployment.id, "weight": 1}], + ) + print("Routed 100% of traffic to the deployment.") + + wait_for_ready(endpoint.id, deployment.id) + print(f"\nEndpoint ready. Inference model name: {endpoint.name}") + return endpoint, deployment + + +def wait_for_ready(endpoint_id: str, deployment_id: str, timeout: int = 3600, poll: int = 15): + """Poll deployment status until READY (cold start can take minutes for large models).""" + elapsed = 0 + while elapsed < timeout: + d = client.beta.endpoints.deployments.retrieve( + deployment_id, project_id=PROJECT_ID, endpoint_id=endpoint_id + ) + state = d.status.state + print( + f" {state} ready={d.status.ready_replicas}/{d.desired_replicas}" + f" ({d.status.message or ''})" + ) + if state == "DEPLOYMENT_STATE_READY": + return d + if state == "DEPLOYMENT_STATE_FAILED": + raise RuntimeError(f"Deployment failed: {d.status.message}") + time.sleep(poll) + elapsed += poll + raise TimeoutError(f"Deployment not READY after {timeout}s") + + +def show_status(endpoint_id: str, deployment_id: str): + """Print the deployment's current state and replica counts.""" + d = client.beta.endpoints.deployments.retrieve( + deployment_id, project_id=PROJECT_ID, endpoint_id=endpoint_id + ) + print(f" state: {d.status.state}") + print(f" desired: {d.desired_replicas}") + print(f" scheduled: {d.status.scheduled_replicas}") + print(f" ready: {d.status.ready_replicas}") + print(f" message: {d.status.message}") + return d + + +def infer(qualified_name: str, prompt: str = "What is 2+2?"): + """Send a chat completion to the endpoint via the shared inference API. + + `qualified_name` is `/` (from endpoint.name), + not the ep_ ID. + """ + inference_client = Together(base_url=INFERENCE_BASE_URL) + response = inference_client.chat.completions.create( + model=qualified_name, + messages=[{"role": "user", "content": prompt}], + max_tokens=64, + ) + print(response.choices[0].message.content) + return response + + +def scale(endpoint_id: str, deployment_id: str, min_replicas: int, max_replicas: int): + """Update replica bounds. 0/0 stops the deployment; min 0 with max > 0 is rejected.""" + d = client.beta.endpoints.deployments.update( + deployment_id, + project_id=PROJECT_ID, + endpoint_id=endpoint_id, + autoscaling={"min_replicas": min_replicas, "max_replicas": max_replicas}, + ) + print(f"Scaled {deployment_id} to bounds {min_replicas}/{max_replicas} (state: {d.status.state})") + return d + + +def stop(endpoint_id: str, deployment_id: str): + """Scale to zero: drains, then STOPPED, and billing stops. Config is preserved.""" + return scale(endpoint_id, deployment_id, 0, 0) + + +def delete(endpoint_id: str, deployment_id: str | None = None): + """Tear down: stop deployment, wait for STOPPED, clear split, delete deployment + endpoint.""" + if deployment_id: + stop(endpoint_id, deployment_id) + while True: + d = client.beta.endpoints.deployments.retrieve( + deployment_id, project_id=PROJECT_ID, endpoint_id=endpoint_id + ) + if d.status.state in ("DEPLOYMENT_STATE_STOPPED", "DEPLOYMENT_STATE_FAILED"): + break + print(f" waiting for STOPPED (now {d.status.state})") + time.sleep(10) + client.beta.endpoints.update(endpoint_id, project_id=PROJECT_ID, traffic_split=[]) + client.beta.endpoints.deployments.delete( + deployment_id, project_id=PROJECT_ID, endpoint_id=endpoint_id + ) + print(f"Deleted deployment {deployment_id}") + client.beta.endpoints.delete(endpoint_id, project_id=PROJECT_ID) + print(f"Deleted endpoint {endpoint_id}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("models", help="List deployable public models") + p.add_argument("--search", default=None) + + p = sub.add_parser("configs", help="List configs for a model") + p.add_argument("--model", required=True, help="Model ID (ml_...)") + + p = sub.add_parser("deploy", help="Create endpoint + deployment, route traffic, wait for READY") + p.add_argument("--model", required=True, help="Model ID (ml_...)") + p.add_argument("--config", required=True, help="Config revision ID (cr_...)") + p.add_argument("--config-project", default=PROJECT_ID, help="Config's owning project (from configs list)") + p.add_argument("--endpoint", required=True, help="New endpoint name") + p.add_argument("--min", type=int, default=1, dest="min_replicas") + p.add_argument("--max", type=int, default=1, dest="max_replicas") + p.add_argument("--inactive-timeout", type=int, default=None, help="Idle minutes before auto-stop") + + p = sub.add_parser("status", help="Show deployment status") + p.add_argument("--endpoint", required=True) + p.add_argument("--deployment", required=True) + + p = sub.add_parser("infer", help="Send a test chat completion") + p.add_argument("--name", required=True, help="Qualified endpoint name (slug/endpoint)") + p.add_argument("--prompt", default="What is 2+2?") + + p = sub.add_parser("scale", help="Update replica bounds") + p.add_argument("--endpoint", required=True) + p.add_argument("--deployment", required=True) + p.add_argument("--min", type=int, required=True, dest="min_replicas") + p.add_argument("--max", type=int, required=True, dest="max_replicas") + + p = sub.add_parser("stop", help="Scale to 0/0 (stops billing)") + p.add_argument("--endpoint", required=True) + p.add_argument("--deployment", required=True) + + p = sub.add_parser("rm", help="Delete deployment (if given) and endpoint") + p.add_argument("--endpoint", required=True) + p.add_argument("--deployment", default=None) + + args = parser.parse_args() + if args.command == "models": + list_supported_models(args.search) + elif args.command == "configs": + list_configs(args.model) + elif args.command == "deploy": + deploy( + args.model, args.config, args.config_project, args.endpoint, + args.min_replicas, args.max_replicas, args.inactive_timeout, + ) + elif args.command == "status": + show_status(args.endpoint, args.deployment) + elif args.command == "infer": + infer(args.name, args.prompt) + elif args.command == "scale": + scale(args.endpoint, args.deployment, args.min_replicas, args.max_replicas) + elif args.command == "stop": + stop(args.endpoint, args.deployment) + elif args.command == "rm": + delete(args.endpoint, args.deployment) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/together-dedicated-model-inference/scripts/rollout_new_version.py b/skills/together-dedicated-model-inference/scripts/rollout_new_version.py new file mode 100644 index 0000000..9f666b3 --- /dev/null +++ b/skills/together-dedicated-model-inference/scripts/rollout_new_version.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +""" +Together AI Dedicated Model Inference -- Roll Out a New Deployment + +Replace a live deployment with a new one (new model version, new config, or +new hardware) without downtime: create the target stopped, start a rollout +(blue-green or metric-gated canary), watch it, and control its lifecycle. + +Usage: + python rollout_new_version.py prepare --endpoint ep_abc123 --model ml_abc123 --config cr_abc123 --name v2 + python rollout_new_version.py start --endpoint ep_abc123 --source dep_src --target dep_tgt --replicas 4 + python rollout_new_version.py start --endpoint ep_abc123 --source dep_src --target dep_tgt \ + --replicas 4 --canary --steps 25,50,100 --gate-latency-regression 50 + python rollout_new_version.py watch --endpoint ep_abc123 --rollout rol_abc123 + python rollout_new_version.py pause|resume|complete|abort --endpoint ep_abc123 --rollout rol_abc123 + +Requires: + uv pip install --upgrade together # a release with the beta DMI surface (client.beta.*) + export TOGETHER_API_KEY=your_key + +Notes: + - The source must be in the endpoint's traffic split, or the rollout shifts nothing. + - Metric gates are canary-only and need live traffic on the endpoint to evaluate. + - After COMPLETED there's no rollback; revert by rolling out in reverse. +""" + +import argparse +import sys +import time + +from together import Together + +client = Together() +PROJECT_ID = client.whoami().project_id + +TERMINAL_STATES = {"ROLLOUT_STATE_COMPLETED", "ROLLOUT_STATE_ABORTED"} + + +def prepare_target(endpoint_id: str, model_id: str, config_id: str, config_project_id: str, name: str): + """Create the target deployment stopped (0/0) so the rollout scales it from zero.""" + deployment = client.beta.endpoints.deployments.create( + endpoint_id, + project_id=PROJECT_ID, + name=name, + model=f"projects/{PROJECT_ID}/models/{model_id}", + config=f"projects/{config_project_id}/configs/{config_id}", + autoscaling={"min_replicas": 0, "max_replicas": 0}, + ) + print(f"Created stopped target deployment: {deployment.id}") + print("Do NOT add it to the traffic split; the rollout moves traffic itself.") + return deployment + + +def start_rollout( + endpoint_id: str, + source_id: str, + target_id: str, + final_replicas: int, + canary: bool = False, + steps: list[int] | None = None, + step_interval: str = "600s", + gate_latency_regression: int | None = None, +): + """Create and start a rollout. Blue-green by default; --canary for a staged ladder. + + With a canary, an optional serving_latency regression gate pauses the rollout + (SYSTEM_PAUSED) if the target's p95 regresses more than the given percent vs the source. + """ + kwargs: dict = { + "project_id": PROJECT_ID, + "source_deployment_id": source_id, + "target_deployment_id": target_id, + "source_cleanup": "SOURCE_CLEANUP_POLICY_DRAIN", + "final_source_replicas": 0, + "final_target_replicas": final_replicas, + } + if canary: + ladder = steps or [10, 50, 100] + kwargs["canary"] = { + "steps": [ + # Scale target replicas proportionally to each traffic step. + {"traffic": pct, "replicas": max(1, round(final_replicas * pct / 100))} + for pct in ladder + ], + "step_interval": step_interval, + } + if gate_latency_regression is not None: + kwargs["metrics"] = [ + { + "name": "serving_latency", + "stat": "METRIC_STAT_TYPE_PERCENTILE", + "percentile": 95, + "regression_check": { + "max_regression_percent": gate_latency_regression, + "direction": "REGRESSION_DIRECTION_HIGHER_IS_WORSE", + }, + # 300s window for histogram percentiles; keep step_interval >= window + ~90s. + "window": "300s", + } + ] + else: + kwargs["blue_green"] = {} + + rollout = client.beta.endpoints.rollouts.create(endpoint_id, **kwargs) + client.beta.endpoints.rollouts.start(rollout.id, project_id=PROJECT_ID, endpoint_id=endpoint_id) + print(f"Rollout started: {rollout.id} ({'canary' if canary else 'blue-green'})") + return rollout + + +def watch(endpoint_id: str, rollout_id: str, poll: int = 30): + """Poll the rollout until it reaches a terminal state, printing progress.""" + while True: + r = client.beta.endpoints.rollouts.retrieve( + rollout_id, project_id=PROJECT_ID, endpoint_id=endpoint_id + ) + line = f" {r.state} step={getattr(r, 'current_step', None)} traffic={getattr(r, 'current_traffic_percent', None)}%" + condition = getattr(getattr(r, "status", None), "condition", None) + if condition is not None and getattr(condition, "category", None): + line += f" condition={condition.category}: {getattr(condition, 'message', '')}" + print(line) + if r.state in TERMINAL_STATES: + return r + if r.state == "ROLLOUT_STATE_SYSTEM_PAUSED": + print( + " Platform paused the rollout. Transient causes (METRICS_UNAVAILABLE," + " CAPACITY_EXHAUSTED): fix and `resume`. Regressions (METRIC_REGRESSION," + " HEALTH_REGRESSION): `abort` to revert to the source." + ) + time.sleep(poll) + + +def control(endpoint_id: str, rollout_id: str, action: str, reason: str | None = None): + """pause / resume / complete (fast-forward to target) / abort (revert to source).""" + rollouts = client.beta.endpoints.rollouts + common = {"project_id": PROJECT_ID, "endpoint_id": endpoint_id} + if action == "pause": + rollouts.pause(rollout_id, **common, reason=reason) + elif action == "resume": + rollouts.resume(rollout_id, **common) + elif action == "complete": + rollouts.promote(rollout_id, **common) + elif action == "abort": + rollouts.abort(rollout_id, **common, reason=reason) + print(f"{action}: {rollout_id}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("prepare", help="Create the stopped target deployment") + p.add_argument("--endpoint", required=True) + p.add_argument("--model", required=True, help="Model ID (ml_...)") + p.add_argument("--config", required=True, help="Config revision ID (cr_...)") + p.add_argument("--config-project", default=PROJECT_ID) + p.add_argument("--name", required=True, help="Target deployment name") + + p = sub.add_parser("start", help="Create and start the rollout") + p.add_argument("--endpoint", required=True) + p.add_argument("--source", required=True, help="Source deployment ID (currently serving)") + p.add_argument("--target", required=True, help="Target deployment ID (stopped)") + p.add_argument("--replicas", type=int, required=True, help="Final target replica count (>= 1)") + p.add_argument("--canary", action="store_true", help="Staged ladder instead of blue-green") + p.add_argument("--steps", default=None, help="Canary traffic ladder, e.g. 25,50,100") + p.add_argument("--interval", default="600s", help="Canary soak per step (default 600s)") + p.add_argument( + "--gate-latency-regression", type=int, default=None, + help="Canary gate: max p95 serving_latency regression percent vs source", + ) + + p = sub.add_parser("watch", help="Poll the rollout until terminal") + p.add_argument("--endpoint", required=True) + p.add_argument("--rollout", required=True) + + for action in ("pause", "resume", "complete", "abort"): + p = sub.add_parser(action, help=f"{action} a running rollout") + p.add_argument("--endpoint", required=True) + p.add_argument("--rollout", required=True) + p.add_argument("--reason", default=None) + + args = parser.parse_args() + if args.command == "prepare": + prepare_target(args.endpoint, args.model, args.config, args.config_project, args.name) + elif args.command == "start": + steps = [int(s) for s in args.steps.split(",")] if args.steps else None + start_rollout( + args.endpoint, args.source, args.target, args.replicas, + canary=args.canary, steps=steps, step_interval=args.interval, + gate_latency_regression=args.gate_latency_regression, + ) + elif args.command == "watch": + watch(args.endpoint, args.rollout) + else: + control(args.endpoint, args.rollout, args.command, getattr(args, "reason", None)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/together-dedicated-model-inference/scripts/upload_custom_model.py b/skills/together-dedicated-model-inference/scripts/upload_custom_model.py new file mode 100644 index 0000000..eb5a4b5 --- /dev/null +++ b/skills/together-dedicated-model-inference/scripts/upload_custom_model.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +""" +Together AI Dedicated Model Inference -- Upload and Deploy a Custom Model or LoRA Adapter + +Register a model record, import weights from Hugging Face or a presigned S3 +URL, poll the upload job, and deploy the result on a dedicated endpoint. + +Usage: + python upload_custom_model.py create --name gemma-4-31b-it --base-model ml_base123 + python upload_custom_model.py upload --model ml_abc123 --from-url https://huggingface.co/org/repo [--hf-token hf_...] [--adapter] + python upload_custom_model.py poll --job job_abc123 [--adapter] + python upload_custom_model.py deploy --model ml_abc123 --config cr_abc123 --endpoint my-custom-model + +Requires: + uv pip install --upgrade together # a release with the beta DMI surface (client.beta.*) + export TOGETHER_API_KEY=your_key + +Upload requirements: + - Weights must be a fine-tuned variant of a base model Together supports for + dedicated inference (uploads can't introduce new architectures), and must + fit on a single GPU. + - Hugging Face repo layout compatible with from_pretrained; adapters need + adapter_config.json + adapter_model.safetensors. + - S3 sources: one .zip/.tar.gz archive with files at the archive ROOT, and a + presigned URL valid for at least 100 minutes. + - Name records bare (gemma-4-31b-it), never org-prefixed (google/gemma-4-31b-it): + the platform prepends your project slug, and a doubled name can't be renamed. +""" + +import argparse +import sys +import time + +from together import Together + +client = Together() +PROJECT_ID = client.whoami().project_id + + +def create_record(name: str, base_model_id: str, description: str | None = None): + """Register the model/adapter record. Weights are attached by a later upload.""" + if "/" in name: + print(f"WARNING: '{name}' looks org-prefixed; this produces an unrenamable doubled slug.") + model = client.beta.models.create( + project_id=PROJECT_ID, + model={"name": name, "base_model_id": base_model_id, "description": description}, + ) + print(f"Created model record: {model.id} ({model.name})") + return model + + +def start_remote_upload(model_id: str, remote_url: str, hf_token: str | None, adapter: bool): + """Stream weights server-side from Hugging Face or a presigned S3 URL.""" + job = client.beta.models.remote_uploads.create( + project_id=PROJECT_ID, + type="adapter" if adapter else "model", + model_id=model_id, + remote_url=remote_url, + token=hf_token, + ) + print(f"Upload job: {job.id} (status: {job.status})") + return job + + +def poll_job(job_id: str, adapter: bool, timeout: int = 7200, poll: int = 20): + """Poll the upload job until REMOTE_UPLOAD_STATUS_SUCCEEDED.""" + upload_type = "adapters" if adapter else "models" + elapsed = 0 + while elapsed < timeout: + job = client.beta.models.remote_uploads.retrieve( + job_id, project_id=PROJECT_ID, type=upload_type + ) + print(f" {job.status} {job.status_message or ''}") + if job.status == "REMOTE_UPLOAD_STATUS_SUCCEEDED": + print("Upload complete. Verify files with: tg beta models ls-files ") + return job + if "FAILED" in (job.status or ""): + raise RuntimeError(f"Upload failed: {job.status_message}") + time.sleep(poll) + elapsed += poll + raise TimeoutError(f"Upload not finished after {timeout}s") + + +def deploy(model_id: str, config_id: str, config_project_id: str, endpoint_name: str): + """Deploy the uploaded model like any other: endpoint + deployment + traffic.""" + endpoint = client.beta.endpoints.create(project_id=PROJECT_ID, name=endpoint_name) + deployment = client.beta.endpoints.deployments.create( + endpoint.id, + project_id=PROJECT_ID, + name="prod", + model=f"projects/{PROJECT_ID}/models/{model_id}", + config=f"projects/{config_project_id}/configs/{config_id}", + autoscaling={"min_replicas": 1, "max_replicas": 1}, + ) + client.beta.endpoints.update( + endpoint.id, + project_id=PROJECT_ID, + traffic_split=[{"deployment_id": deployment.id, "weight": 1}], + ) + print(f"Deployed. Endpoint: {endpoint.id} Deployment: {deployment.id}") + print(f"Poll status, then send requests with model='{endpoint.name}'") + return endpoint, deployment + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("create", help="Register a model/adapter record") + p.add_argument("--name", required=True, help="Bare name, no org prefix") + p.add_argument("--base-model", required=True, help="Supported base model ID (ml_...)") + p.add_argument("--description", default=None) + + p = sub.add_parser("upload", help="Start a remote upload from HF or presigned S3 URL") + p.add_argument("--model", required=True, help="Model record ID (ml_...)") + p.add_argument("--from-url", required=True, help="HF repo URL or presigned S3 archive URL") + p.add_argument("--hf-token", default=None, help="Token for gated/private HF repos") + p.add_argument("--adapter", action="store_true", help="Upload is a LoRA adapter") + + p = sub.add_parser("poll", help="Poll an upload job until it succeeds") + p.add_argument("--job", required=True, help="Upload job ID (job_...)") + p.add_argument("--adapter", action="store_true") + + p = sub.add_parser("deploy", help="Deploy the uploaded model") + p.add_argument("--model", required=True, help="Model ID (ml_...)") + p.add_argument("--config", required=True, help="Config revision ID (cr_...)") + p.add_argument("--config-project", default=PROJECT_ID, help="Config's owning project") + p.add_argument("--endpoint", required=True, help="New endpoint name") + + args = parser.parse_args() + if args.command == "create": + create_record(args.name, args.base_model, args.description) + elif args.command == "upload": + start_remote_upload(args.model, args.from_url, args.hf_token, args.adapter) + elif args.command == "poll": + poll_job(args.job, args.adapter) + elif args.command == "deploy": + deploy(args.model, args.config, args.config_project, args.endpoint) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From e0e5b1845fd8379bcaa8493a0f46aba7daea24dd Mon Sep 17 00:00:00 2001 From: Zain Hasan Date: Mon, 13 Jul 2026 15:04:41 -0700 Subject: [PATCH 02/11] cold start fixes --- .../references/api-reference.md | 23 +++++++++++++++---- .../references/cli-reference.md | 7 ++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/skills/together-dedicated-model-inference/references/api-reference.md b/skills/together-dedicated-model-inference/references/api-reference.md index 6bc652f..d6ef521 100644 --- a/skills/together-dedicated-model-inference/references/api-reference.md +++ b/skills/together-dedicated-model-inference/references/api-reference.md @@ -332,13 +332,19 @@ Deletion is permanent, and a deployment must be stopped first. Order: 4. Delete the endpoint once it has no deployments. ```python -client.beta.endpoints.update("ep_abc123", project_id=project_id, traffic_split=[]) +# You can't zero the split with an empty list: the SDK/API omits an empty `traffic_split` +# from the PATCH body and returns "400 no fields to update". Send the split with this +# deployment's entry removed (keeping any others): +client.beta.endpoints.update("ep_abc123", project_id=project_id, + traffic_split=[{"deployment_id": "dep_other", "weight": 1}]) client.beta.endpoints.deployments.delete("dep_abc123", project_id=project_id, endpoint_id="ep_abc123") client.beta.endpoints.delete("ep_abc123", project_id=project_id) ``` -The CLI's `rm` smart-deletes by ID prefix and auto-detaches from traffic splits and -experiments; see [cli-reference.md](cli-reference.md). +When the deployment is the endpoint's **only** traffic entry there's nothing to route to, so +the empty-split limitation above blocks the pure-SDK path — use the CLI's `rm ... --force` +instead, which auto-detaches from the split and deletes the deployment and endpoint in one +step (it smart-deletes by ID prefix; see [cli-reference.md](cli-reference.md)). ## Monitoring @@ -365,7 +371,16 @@ events = client.beta.endpoints.list_events("ep_abc123", project_id=project_id, l Optional filters: `types` (event-type strings), `since` / `until` (time range), `subject_id` (for example `rol_abc123` for one rollout's audit trail), `deployment_ids` (scope to specific -deployments), `limit` / `after` (max 500, default 50). +deployments), `limit` / `after` (max 500, default 50). Events come back newest-first — reverse +them for a chronological timeline. + +The feed is the authoritative source for a **cold-start phase breakdown**. Alongside +`deployment.created` and `deployment.status_updated` (which carries each state transition plus +`first_ready_at` / `first_replica_ready_at`), `pod.startup_phase_changed` events trace the +replica boot — typically `PullingImage → Starting → LoadingWeights → StartupComplete` — each +with a `phase_budget`. Diffing consecutive event timestamps attributes wall-clock time to each +phase (image pull vs. engine start vs. weight load vs. warmup), which is finer-grained than +polling `status.state` alone. ## Send Inference Requests diff --git a/skills/together-dedicated-model-inference/references/cli-reference.md b/skills/together-dedicated-model-inference/references/cli-reference.md index 9c82e4c..77787cc 100644 --- a/skills/together-dedicated-model-inference/references/cli-reference.md +++ b/skills/together-dedicated-model-inference/references/cli-reference.md @@ -28,6 +28,13 @@ tg --version # check; upgrade with: uv tool upgrade togethe uses the project associated with your API key, overridable with the `TOGETHER_PROJECT_ID` env var or `--project` on any command. Every command accepts `--json` for machine-readable output. +**Scripting / non-interactive use (agents, CI, piped stdin).** Mutating commands such as +`deploy` print a confirmation preview and prompt before acting; with no TTY they can't read the +prompt and fail. Pass `--non-interactive` to skip the prompt. In that mode project inference +from the API key is *not* applied — `deploy` errors with `Project argument is required` unless +you also pass `--project ` (or set `TOGETHER_PROJECT_ID`). A reliable scripted deploy +is therefore `tg beta endpoints deploy ... --project --non-interactive --json`. + ## Command tree ```text From bdee768c4610ff1503b79ef150c4b584d6bf0b7c Mon Sep 17 00:00:00 2001 From: Zain Hasan Date: Mon, 13 Jul 2026 15:51:48 -0700 Subject: [PATCH 03/11] add routing to skills --- .../SKILL.md | 11 ++++++- .../references/api-reference.md | 12 ++++++-- .../references/cli-reference.md | 2 +- .../references/traffic-routing.md | 29 +++++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/skills/together-dedicated-model-inference/SKILL.md b/skills/together-dedicated-model-inference/SKILL.md index 291dc3e..9c2c9c8 100644 --- a/skills/together-dedicated-model-inference/SKILL.md +++ b/skills/together-dedicated-model-inference/SKILL.md @@ -100,7 +100,16 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; live in the dashboard (`https://api.together.ai/endpoints`); there's no metrics query API. - **Deletion order matters:** stop the deployment (wait for `STOPPED`), remove it from the traffic split, delete it, then delete the endpoint. The CLI's `rm` smart-deletes by ID - prefix and handles detaching; deleting an endpoint with deployments needs `--force`. + prefix and auto-detaches from the split; `--force` deletes an endpoint's deployments too. + But **neither `rm` nor `--force` stops a running deployment** — a `READY` deployment must + already be scaled to `0/0` and `STOPPED`, or `rm` fails with `deployment must be stopped or + failed before deletion (current state: ready)`. Scale down and wait for `STOPPED` first. +- **To see which deployment/replica served a request, read the inference response headers.** + The response *body*'s `model` field only echoes the endpoint's qualified name — identical for + every deployment. The routing headers distinguish them: `x-cluster` is the per-deployment + cluster ID and `worker_url` (inside the `x-i-router-log-event` header) is the replica pod. + This is the only way to verify a split, rollout, or A/B empirically. See + [traffic-routing.md](references/traffic-routing.md) (Observing routing). - **Rollouts are the safe way to replace a deployment on live traffic** — create the target stopped (`0/0`), then `tg beta endpoints rollout --from --canary`. Metric gates are canary-only and need live traffic to evaluate. diff --git a/skills/together-dedicated-model-inference/references/api-reference.md b/skills/together-dedicated-model-inference/references/api-reference.md index d6ef521..d44e904 100644 --- a/skills/together-dedicated-model-inference/references/api-reference.md +++ b/skills/together-dedicated-model-inference/references/api-reference.md @@ -343,8 +343,10 @@ client.beta.endpoints.delete("ep_abc123", project_id=project_id) When the deployment is the endpoint's **only** traffic entry there's nothing to route to, so the empty-split limitation above blocks the pure-SDK path — use the CLI's `rm ... --force` -instead, which auto-detaches from the split and deletes the deployment and endpoint in one -step (it smart-deletes by ID prefix; see [cli-reference.md](cli-reference.md)). +instead, which auto-detaches from the split and deletes the deployment and endpoint together +(it smart-deletes by ID prefix; see [cli-reference.md](cli-reference.md)). Note `--force` does +**not** stop a running deployment: scale it to `0/0` and wait for `STOPPED` first, or `rm` +fails with `deployment must be stopped or failed before deletion`. ## Monitoring @@ -399,6 +401,12 @@ response = client.chat.completions.create( ) ``` +To attribute a response to the deployment/replica that served it (e.g. verifying a split), +read the **response headers**, not the body: `x-cluster` is the per-deployment cluster ID and +`worker_url` (in `x-i-router-log-event`) is the replica pod. The body's `model` field only +echoes the endpoint name. Full method — including varying the sampling key so load spreads +across the split — is in [traffic-routing.md](traffic-routing.md) (Observing routing). + ## Troubleshooting - **`routing_error` / 503 on a `READY` deployment** — the deployment isn't in the traffic diff --git a/skills/together-dedicated-model-inference/references/cli-reference.md b/skills/together-dedicated-model-inference/references/cli-reference.md index 77787cc..bd0de1e 100644 --- a/skills/together-dedicated-model-inference/references/cli-reference.md +++ b/skills/together-dedicated-model-inference/references/cli-reference.md @@ -163,7 +163,7 @@ experiment (`abx_`), shadow experiment (`exp_`). ```bash tg beta endpoints rm dep_abc123 # delete a deployment (must be stopped) tg beta endpoints rm ep_abc123 # delete an endpoint with no deployments -tg beta endpoints rm ep_abc123 --force # tear down deployments too +tg beta endpoints rm ep_abc123 --force # tear down deployments too (must be STOPPED first) tg beta endpoints rm abx_abc123 # end an A/B test (traffic returns to control) tg beta endpoints rm exp_abc123 # stop a shadow experiment ``` diff --git a/skills/together-dedicated-model-inference/references/traffic-routing.md b/skills/together-dedicated-model-inference/references/traffic-routing.md index bb3fb52..2ac7966 100644 --- a/skills/together-dedicated-model-inference/references/traffic-routing.md +++ b/skills/together-dedicated-model-inference/references/traffic-routing.md @@ -7,6 +7,7 @@ traffic: weighted splits, rollouts, A/B tests, and shadow experiments. - [How routing works](#how-routing-works) - [Stickiness](#stickiness) +- [Observing routing](#observing-routing) - [Traffic splits (weights)](#traffic-splits-weights) - [Rollouts](#rollouts) - [Strategies](#strategies) @@ -48,6 +49,34 @@ Set `prompt_cache_key` on requests sharing a prompt prefix (e.g. every turn of o conversation) to control stickiness. Editing weights, adding/removing deployments, or a rollout shifting its percentage reassigns some keys. +## Observing routing + +To verify empirically how traffic actually lands — for a split, rollout, or A/B test — you +need two things the obvious approach misses: + +- **Attribution comes from the response headers, not the body.** The response body's `model` + field only echoes the endpoint's qualified name, so two deployments serving the same model + produce byte-identical bodies. The routing headers on the inference response distinguish + them: + - `x-cluster` — the per-deployment cluster ID (one stable value per deployment). + - `x-i-router-log-event` — a JSON array whose `worker_url` field is the replica's pod + (`http://:`); distinct pod IPs count distinct replicas within a deployment. + + These header names are operational, not part of the stable inference API contract — confirm + them against a live probe before relying on them. `x-cluster` values don't equal the `dep_` + management IDs; map them by briefly routing 100% to one deployment (see propagation note + below) and recording the `x-cluster` it returns. + +- **Vary the sampling key or all your load lands on one deployment.** Because routing is + sticky (above), sending N identical requests routes all N to the same deployment — you'll + measure 100/0 no matter what the weights say. Set a unique `prompt_cache_key` (or `user`) + per request so the sample spreads across the split. A few hundred varied requests gives a + stable share estimate. + +- **Split changes take tens of seconds to propagate.** After `endpoints.update(traffic_split=...)` + (or a scale change), the router keeps serving the old split briefly — a probe within ~10s + can still hit the previous target. Allow ~30s before measuring or mapping clusters. + ## Traffic splits (weights) Set by updating the endpoint (SDK/API only): From 732b344f3920c73e69c09a4990cce625893ab882 Mon Sep 17 00:00:00 2001 From: Zain Hasan Date: Mon, 13 Jul 2026 16:38:48 -0700 Subject: [PATCH 04/11] fix ab testing --- .../SKILL.md | 6 +- .../references/cli-reference.md | 10 +++- .../references/traffic-routing.md | 58 +++++++++++++++++-- 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/skills/together-dedicated-model-inference/SKILL.md b/skills/together-dedicated-model-inference/SKILL.md index 9c2c9c8..04e6d3f 100644 --- a/skills/together-dedicated-model-inference/SKILL.md +++ b/skills/together-dedicated-model-inference/SKILL.md @@ -67,8 +67,10 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; 1. Pick a model: `tg beta models public` (or upload custom weights first). 2. Optionally pick a config: `tg beta models configs ` (the CLI's `deploy` auto-picks when there's exactly one). -3. Deploy: `tg beta endpoints deploy --endpoint ` — creates the - endpoint, attaches a deployment, and routes 100% of traffic in one step. +3. Deploy: `tg beta endpoints deploy --endpoint ` — creates the + endpoint, attaches a deployment, and routes 100% of traffic in one step. For a public + catalog model, pass its **name** (`Qwen/Qwen2.5-7B-Instruct`); the catalog `ml_` ID is owned + by a platform project and won't resolve as a `deploy`/`ab` positional in your project. 4. Poll until `status.state` is `DEPLOYMENT_STATE_READY` (re-run `tg beta endpoints get `). 5. Send requests to `https://api-inference.together.ai/v1` with the qualified name as `model`. 6. Scale or reconfigure with `tg beta endpoints update `; split traffic, roll out new diff --git a/skills/together-dedicated-model-inference/references/cli-reference.md b/skills/together-dedicated-model-inference/references/cli-reference.md index bd0de1e..5d240c7 100644 --- a/skills/together-dedicated-model-inference/references/cli-reference.md +++ b/skills/together-dedicated-model-inference/references/cli-reference.md @@ -214,13 +214,16 @@ Creates a variant deployment for a model and starts an A/B experiment against a deployment that's already serving traffic. The CLI assigns the remainder to the control. ```bash -tg beta endpoints ab ml_CbJNwQC2ZqCU2iFT3mrCh \ +tg beta endpoints ab Qwen/Qwen2.5-7B-Instruct \ --control dep_control123 \ --percent 5 \ --name sampling-tweak-v1 ``` -The variant model is the positional argument (same forms as `deploy`). +The variant model is the positional argument (same forms as `deploy`). For a **public catalog +model, pass the name** (as above), not the `ml_` ID echoed by `deploy`/`models public` — that +ID is owned by a platform project and the positional resolves it in *your* project, failing +with `Model ml_… not found`. Use the `ml_` ID only for a model that lives in your own project. | Flag | Description | | --- | --- | @@ -231,7 +234,8 @@ The variant model is the positional argument (same forms as `deploy`). | `--enable-lora` | Enable the multi-LoRA kernel on the variant. | Ramping or editing an existing experiment is SDK/API only (`ab_experiments.update` replaces -the whole member set). End the test with `rm abx_...`. +the whole member set, and requires `update_mask="members"` + the current `etag` or it 400s — +see [traffic-routing.md](traffic-routing.md)). End the test with `rm abx_...`. ## endpoints shadow (shadow experiments) diff --git a/skills/together-dedicated-model-inference/references/traffic-routing.md b/skills/together-dedicated-model-inference/references/traffic-routing.md index 2ac7966..366d86d 100644 --- a/skills/together-dedicated-model-inference/references/traffic-routing.md +++ b/skills/together-dedicated-model-inference/references/traffic-routing.md @@ -77,6 +77,31 @@ need two things the obvious approach misses: (or a scale change), the router keeps serving the old split briefly — a probe within ~10s can still hit the previous target. Allow ~30s before measuring or mapping clusters. +Reading the header from the Python SDK needs `with_raw_response` (the normal `.create` returns +a parsed body with no headers), and `prompt_cache_key` is not a top-level argument — pass it in +`extra_body`. A minimal probe that tallies the split: + +```python +from collections import Counter +from together import Together + +client = Together(base_url="https://api-inference.together.ai/v1") +counts = Counter() +for i in range(200): # a few hundred varied requests → stable share estimate + raw = client.chat.completions.with_raw_response.create( + model="your-project-slug/my-endpoint", # qualified name, not ep_ ID + messages=[{"role": "user", "content": f"ping {i}"}], + max_tokens=1, + extra_body={"prompt_cache_key": f"probe-{i}"}, # unique key defeats sticky routing + ) + counts[raw.headers.get("x-cluster")] += 1 +print(counts) # {control_cluster: ~N*control%, variant_cluster: ~N*variant%} +``` + +Size N so the smallest arm gets a countable sample — a 5% arm needs a few hundred requests to +clear single digits. Expect binomial noise: a configured 95/5 typically measures ~92–97% on the +control at N=200, not exactly 95%. + ## Traffic splits (weights) Set by updating the endpoint (SDK/API only): @@ -280,11 +305,17 @@ Rules: split weight fails the test at start. A control with weight 0 (or absent) means the test receives no traffic. - Updating `members` replaces the whole set — resend every member each time. +- **The CLI `ab` creates the variant deployment, which then cold-starts.** Until it reaches + `READY` the experiment routes ~100% to the control (the variant can't serve yet), so a probe + fired immediately after create measures 0% variant. Poll the variant to `READY` before + measuring. -CLI (creates the variant deployment too): +CLI (creates the variant deployment too — pass the model **name** for a public model, not the +`ml_` ID; the catalog `ml_` ID is owned by a platform project and resolves as +`Model ml_… not found` here): ```bash -tg beta endpoints ab ml_CbJNwQC2ZqCU2iFT3mrCh --control dep_control123 --percent 5 --name candidate-v1 +tg beta endpoints ab Qwen/Qwen2.5-7B-Instruct --control dep_control123 --percent 5 --name candidate-v1 ``` SDK (create the variant deployment first): @@ -302,9 +333,28 @@ experiment = client.beta.endpoints.ab_experiments.create( ``` Ramp / add variants / remove variants: `ab_experiments.update` with the full new member set -(SDK/API only). Promote a winner with a rollout +(SDK/API only). **Pass `update_mask="members"` and the current `etag` (from a fresh +`retrieve`)** — calling `update(members=[...])` alone (the shape the SDK docstring implies is +enough) fails with a bare `400 Invalid Argument` that names no field; adding the mask + etag +fixes it (not isolated which of the two the server requires, so send both): + +```python +cur = client.beta.endpoints.ab_experiments.retrieve( + "abx_abc123", project_id=project_id, endpoint_id="ep_abc123") +client.beta.endpoints.ab_experiments.update( + "abx_abc123", project_id=project_id, endpoint_id="ep_abc123", + update_mask="members", etag=cur.etag, + members=[ + {"deployment_id": "dep_control123", "role": "AB_EXPERIMENT_MEMBER_ROLE_CONTROL", "percent": 80}, + {"deployment_id": "dep_variant456", "role": "AB_EXPERIMENT_MEMBER_ROLE_VARIANT", "percent": 20}, + ], +) +``` + +Promote a winner with a rollout (`rollout dep_variant456 --from dep_control123 --blue-green`), then delete the test -(`rm abx_abc123`) — deletion returns all traffic to the regular split immediately. +(`rm abx_abc123`) — deletion returns all traffic to the regular split immediately (verified: +after `rm`, a varied probe lands 100% on the control's cluster). ## Shadow experiments From c8a856d2156c34e769eaffc8a81b6f33057289cd Mon Sep 17 00:00:00 2001 From: Zain Hasan Date: Mon, 13 Jul 2026 17:18:23 -0700 Subject: [PATCH 05/11] shodaow improvements --- skills/together-dedicated-model-inference/SKILL.md | 5 ++++- .../references/traffic-routing.md | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/skills/together-dedicated-model-inference/SKILL.md b/skills/together-dedicated-model-inference/SKILL.md index 04e6d3f..c2191ab 100644 --- a/skills/together-dedicated-model-inference/SKILL.md +++ b/skills/together-dedicated-model-inference/SKILL.md @@ -71,7 +71,10 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; endpoint, attaches a deployment, and routes 100% of traffic in one step. For a public catalog model, pass its **name** (`Qwen/Qwen2.5-7B-Instruct`); the catalog `ml_` ID is owned by a platform project and won't resolve as a `deploy`/`ab` positional in your project. -4. Poll until `status.state` is `DEPLOYMENT_STATE_READY` (re-run `tg beta endpoints get `). +4. Poll until `status.state` is `DEPLOYMENT_STATE_READY`. For scripted polling use the SDK + `client.beta.endpoints.deployments.retrieve(dep_id, project_id=..., endpoint_id=...)`; the + CLI `get` takes an *endpoint* id (a `dep_` id is rejected) and its `--json` reports replica + counts but not per-deployment `status.state` — that renders only in the human table. 5. Send requests to `https://api-inference.together.ai/v1` with the qualified name as `model`. 6. Scale or reconfigure with `tg beta endpoints update `; split traffic, roll out new versions, or experiment as needed. diff --git a/skills/together-dedicated-model-inference/references/traffic-routing.md b/skills/together-dedicated-model-inference/references/traffic-routing.md index 366d86d..29d5ac3 100644 --- a/skills/together-dedicated-model-inference/references/traffic-routing.md +++ b/skills/together-dedicated-model-inference/references/traffic-routing.md @@ -395,6 +395,11 @@ experiment = client.beta.endpoints.shadow_experiments.create( Operating notes: - Changes (create/update/delete) take effect within ~30–60 seconds. +- **The create response can transiently show `targets: []` and `state: INACTIVE`** — target + attachment lands a moment after the experiment row. Confirm the mirror is wired up with a + follow-up `retrieve` (target present, `state: SHADOW_EXPERIMENT_STATE_ACTIVE`), not the create + response. (When reading targets, retrieve the experiment and read its `.targets` rather than + calling `targets.list(...)`.) - **Pause without deleting**: update `source` with `rate: 0`. - Updates to `source` are etag-guarded: retrieve first, pass `etag` back, handle `409 ABORTED` by re-reading. Use `update_mask` (`"source"`, `"description"`). From db667157a1e12bbb51b07f904cf6c06011992473 Mon Sep 17 00:00:00 2001 From: Zain Hasan Date: Mon, 13 Jul 2026 19:59:40 -0700 Subject: [PATCH 06/11] rollouts --- .../SKILL.md | 14 ++++-- .../references/models-and-configs.md | 50 +++++++++++++++++-- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/skills/together-dedicated-model-inference/SKILL.md b/skills/together-dedicated-model-inference/SKILL.md index c2191ab..564feac 100644 --- a/skills/together-dedicated-model-inference/SKILL.md +++ b/skills/together-dedicated-model-inference/SKILL.md @@ -65,12 +65,18 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; ## Workflow 1. Pick a model: `tg beta models public` (or upload custom weights first). -2. Optionally pick a config: `tg beta models configs ` (the CLI's `deploy` - auto-picks when there's exactly one). +2. Pick a config/precision: read the model's `deploymentProfiles` (they carry `quantization` + like `BF16`/`FP8`, `gpuCount`, and the exact `model`+`config` resource names). `tg beta + models configs ` is often empty for catalog architectures — see + [models-and-configs.md](references/models-and-configs.md). `deploy` auto-picks only when the + model has a single profile. 3. Deploy: `tg beta endpoints deploy --endpoint ` — creates the endpoint, attaches a deployment, and routes 100% of traffic in one step. For a public - catalog model, pass its **name** (`Qwen/Qwen2.5-7B-Instruct`); the catalog `ml_` ID is owned - by a platform project and won't resolve as a `deploy`/`ab` positional in your project. + catalog model with one profile, pass its **name** (`Qwen/Qwen2.5-7B-Instruct`); the catalog + `ml_`/`arch_` ID is owned by a platform project and won't resolve as a `deploy`/`ab` + positional in your project. If the model has **multiple profiles** (e.g. BF16 + FP8), a + bare-name deploy errors — pass the chosen profile's full resolved `model` resource path as + the positional plus its `--config` (see models-and-configs.md). 4. Poll until `status.state` is `DEPLOYMENT_STATE_READY`. For scripted polling use the SDK `client.beta.endpoints.deployments.retrieve(dep_id, project_id=..., endpoint_id=...)`; the CLI `get` takes an *endpoint* id (a `dep_` id is rejected) and its `--json` reports replica diff --git a/skills/together-dedicated-model-inference/references/models-and-configs.md b/skills/together-dedicated-model-inference/references/models-and-configs.md index 946f32c..ae2ddae 100644 --- a/skills/together-dedicated-model-inference/references/models-and-configs.md +++ b/skills/together-dedicated-model-inference/references/models-and-configs.md @@ -49,8 +49,8 @@ supported. Don't hardcode model names — list the catalog and use real IDs. Each supported-model entry carries `deploymentProfiles`: certified model-and-config pairs Together publishes. Each profile gives you `model` and `config` **resource names you can pass -straight into a deployment create**, plus `gpuType`, `gpuCount`, and `parallelism` (free-form: -`TP8`, `TP4`, `EP`, `PD` — not always a tensor-parallel degree). +straight into a deployment create**, plus `gpuType`, `gpuCount`, `parallelism` (free-form: +`TP8`, `TP4`, `EP`, `PD` — not always a tensor-parallel degree), and `quantization`. ```json { @@ -59,23 +59,65 @@ straight into a deployment create**, plus `gpuType`, `gpuCount`, and `parallelis "model": "projects/proj_weights/models/ml_weight/revisions/rv_snap", "parallelism": "TP8", "gpuType": "H100", - "gpuCount": 8 + "gpuCount": 8, + "quantization": "FP8" } ``` Note the owning projects in these resource names are platform projects — copy the strings verbatim rather than substituting your own project ID. +**A model usually has more than one profile** — the same weights published at different +`quantization` (e.g. `BF16` vs `FP8`), `gpuCount`, or `parallelism`. `deploymentProfiles` is +the authoritative list of what's deployable and the **only** place `quantization` is exposed; +`tg beta models configs ` frequently returns an empty list for catalog architectures, +so don't rely on it to enumerate a public model's options — read the profiles instead. Filter +the profiles to the precision/hardware you want, then take that profile's `model` and `config` +strings. + +### Deploying a specific profile (precision matters) + +When a model has exactly one profile, `tg beta endpoints deploy --endpoint ` +resolves it automatically. When it has several, a bare-name deploy **fails** rather than +guessing: + +```text +Multiple model revisions/configs found for Qwen/Qwen3.5-9B: ... +Error: Choose one and rerun with the additional flags: --model --config +``` + +Disambiguate by passing the chosen profile's **full resolved `model` resource path** as the +positional (not the bare `Qwen/...` name — that stays ambiguous even with `--config`) together +with its `--config`: + +```bash +# Deploy the BF16 profile specifically +tg beta endpoints deploy \ + "projects/proj_weights/models/ml_weight/revisions/rv_snap" \ + --endpoint my-endpoint \ + --config cr_certified +``` + +From the SDK, `deployments.create` already takes the profile's `model` and `config` resource +names directly (see [api-reference.md](api-reference.md)), so there's no ambiguity to resolve — +just pass the strings from the profile you picked. + ## Configs A config describes how a model runs: inference engine, hardware selectors, optimization profile. Together publishes configs per model; you pick one when creating a deployment. The -CLI's `deploy` auto-picks when the model has exactly one config. +CLI's `deploy` auto-picks when the model has exactly one config, and errors asking you to +disambiguate when there are several (see [Deploying a specific +profile](#deploying-a-specific-profile-precision-matters)). ```bash tg beta models configs ml_CbJNwQC2ZqCU2iFT3mrCh ``` +For a **public catalog architecture** this list is often empty — its deployable configs live in +the model's `deploymentProfiles` instead (above). `tg beta models configs` is most useful for +models in your own project (uploads, fine-tunes). + ```python configs = client.beta.models.configs.list( project_id=project_id, From 75e1029c050bdde7faad107ddb9ccaa17692b563 Mon Sep 17 00:00:00 2001 From: Zain Hasan Date: Mon, 13 Jul 2026 20:51:40 -0700 Subject: [PATCH 07/11] scaling --- .../references/api-reference.md | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/skills/together-dedicated-model-inference/references/api-reference.md b/skills/together-dedicated-model-inference/references/api-reference.md index d44e904..93ff134 100644 --- a/skills/together-dedicated-model-inference/references/api-reference.md +++ b/skills/together-dedicated-model-inference/references/api-reference.md @@ -196,14 +196,28 @@ client.beta.endpoints.deployments.update( CLI equivalent: `tg beta endpoints update dep_abc123 --min-replicas 1 --max-replicas 4`. -Optional windows (settable at deploy or update time via CLI flags): `scale_up_window` (metric must stay +Optional windows (settable at deploy or update time): `scale_up_window` (metric must stay above target this long before adding replicas), `scale_down_window` (cooldown between -scale-downs), `scale_to_zero_window` (idle time before scaling to zero). Scale-up reacts fast; -scale-down is deliberately slow so bursty traffic doesn't pay a cold start on every trough. - -New replicas **cold start**: placement → weight download → engine load → warmup. Minutes for -small models, longer for large ones. Keep `min_replicas >= 1` to avoid cold starts on the -first request; raise it ahead of known traffic spikes. +scale-downs, **default 5m**), `scale_to_zero_window` (idle time before scaling to zero). +Scale-up reacts fast; scale-down is deliberately slow so bursty traffic doesn't pay a cold +start on every trough. **All three are Go duration strings with a unit** (`"30s"`, `"5m"`) on +both the SDK (`autoscaling={..., "scale_up_window": "30s"}`) and the CLI +(`--scale-up-window 30s`). A bare integer is rejected by the API as `invalid request format` — +the CLI help text says "in seconds" but the unit is still required. + +New replicas **cold start**: placement → image pull → weight load → engine load → warmup. +Minutes even for small models (image pull on a cold node often dominates — up to several +minutes), longer for large ones. Two things to design around: + +- **`desired` reacts far faster than `ready` climbs.** The autoscaler moves `desired` within + roughly the metric window + `scale_up_window` (tens of seconds), but `readyReplicas` only + catches up after the full cold start. When measuring "scale-up speed," separate the + *decision* (`desired` / `status.scheduledReplicas` changing) from *fulfillment* + (`status.readyReplicas` changing); the `pod.startup_phase_changed` events (see + [Events Feed](#events-feed)) attribute the wall-clock to each boot phase. +- **Reactive scale-up cannot catch a spike shorter than the cold start.** Keep + `min_replicas >= 1` to avoid a cold start on the first request, and raise `min_replicas` + *ahead of* a known burst — a 90-second wave is over before a new replica is ready. ## Scaling Metrics @@ -240,6 +254,11 @@ CLI equivalent: `tg beta endpoints update dep_abc123 --min-replicas 1 --max-repl Caveats: +- **The default `inflight_requests` target (8) scales on concurrency, not latency.** A + capable replica can serve well above 8 concurrent requests with no latency degradation, so + the default fires scale-ups before the replica is actually latency-saturated. If you want + scaling tied to an SLO rather than raw concurrency, raise the target or switch to a latency + metric (`e2e_latency` / `ttft`, streaming only for `ttft`). - `throughput_per_replica`, `ttft`, and `decoding_speed` are recorded **only for streaming responses** — scaling on them without `"stream": true` traffic won't work. - Latency metrics default to p95 over the measurement window; set `"percentile"` to `"p50"`, From e381a0961cc2120690bdb2441be11d9a30808fcf Mon Sep 17 00:00:00 2001 From: Zain Hasan Date: Mon, 13 Jul 2026 22:47:50 -0700 Subject: [PATCH 08/11] reflect scale time fix --- .../references/api-reference.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/skills/together-dedicated-model-inference/references/api-reference.md b/skills/together-dedicated-model-inference/references/api-reference.md index 93ff134..d80b79a 100644 --- a/skills/together-dedicated-model-inference/references/api-reference.md +++ b/skills/together-dedicated-model-inference/references/api-reference.md @@ -196,14 +196,10 @@ client.beta.endpoints.deployments.update( CLI equivalent: `tg beta endpoints update dep_abc123 --min-replicas 1 --max-replicas 4`. -Optional windows (settable at deploy or update time): `scale_up_window` (metric must stay +Optional windows (settable at deploy or update time via CLI flags): `scale_up_window` (metric must stay above target this long before adding replicas), `scale_down_window` (cooldown between -scale-downs, **default 5m**), `scale_to_zero_window` (idle time before scaling to zero). -Scale-up reacts fast; scale-down is deliberately slow so bursty traffic doesn't pay a cold -start on every trough. **All three are Go duration strings with a unit** (`"30s"`, `"5m"`) on -both the SDK (`autoscaling={..., "scale_up_window": "30s"}`) and the CLI -(`--scale-up-window 30s`). A bare integer is rejected by the API as `invalid request format` — -the CLI help text says "in seconds" but the unit is still required. +scale-downs), `scale_to_zero_window` (idle time before scaling to zero). Scale-up reacts fast; +scale-down is deliberately slow so bursty traffic doesn't pay a cold start on every trough. New replicas **cold start**: placement → image pull → weight load → engine load → warmup. Minutes even for small models (image pull on a cold node often dominates — up to several From 3ef7ed3f6155b4b77f5c93b5cfa473322ca1e771 Mon Sep 17 00:00:00 2001 From: Zain Hasan Date: Tue, 14 Jul 2026 12:02:57 -0700 Subject: [PATCH 09/11] bring in changes from docs + SDK --- .../SKILL.md | 59 +++++++------ .../references/api-reference.md | 74 ++++++++++++----- .../references/cli-reference.md | 83 ++++++++++++------- .../references/models-and-configs.md | 50 ++++++++++- .../references/traffic-routing.md | 21 ++++- .../scripts/deploy_model.py | 9 +- .../scripts/rollout_new_version.py | 32 +++++-- .../scripts/upload_custom_model.py | 5 +- 8 files changed, 233 insertions(+), 100 deletions(-) diff --git a/skills/together-dedicated-model-inference/SKILL.md b/skills/together-dedicated-model-inference/SKILL.md index 564feac..f216385 100644 --- a/skills/together-dedicated-model-inference/SKILL.md +++ b/skills/together-dedicated-model-inference/SKILL.md @@ -1,6 +1,6 @@ --- name: together-dedicated-model-inference -description: "Deploy and operate models on dedicated GPUs with Together AI's Dedicated Model Inference (DMI, the v2 dedicated endpoints API): beta endpoints, deployments, hardware configs, autoscaling with auto-shutdown, traffic splitting, canary/blue-green/rolling rollouts, A/B tests, shadow experiments, and custom model or LoRA adapter uploads. Reach for it whenever the user mentions together beta endpoints or tg beta commands, client.beta.endpoints, DMI resources like ep_/dep_/cr_/ml_ IDs, or wants production model serving with traffic management on Together AI. Use together-dedicated-endpoints only for the legacy v1 endpoints API." +description: "Deploy and operate models on dedicated GPUs with Together AI's Dedicated Model Inference (DMI, the v2 dedicated endpoints API): beta endpoints, deployments, deployment profiles and hardware configs, autoscaling, traffic splitting, canary/blue-green/rolling rollouts, A/B tests, shadow experiments, Prometheus metrics, and custom model or LoRA adapter uploads. Reach for it whenever the user mentions together beta endpoints or tg beta commands, client.beta.endpoints, DMI resources like ep_/dep_/cr_/ml_ IDs, or wants production model serving with traffic management on Together AI. Use together-dedicated-endpoints only for the legacy v1 endpoints API." --- # Together Dedicated Model Inference @@ -20,8 +20,9 @@ The resource model has six parts: **Project → Model → Config → Endpoint and count, optimization profile). The model determines the available configs. - **Replica** — one model instance on its own dedicated hardware. -Inference goes to `https://api-inference.together.ai/v1` with the endpoint's **qualified name** -(`/`) as the `model` parameter. Management goes through the +Inference goes to `https://api-inference.together.ai/v1` with the **endpoint string** (the +project-qualified name `/` — the docs' current term for what was +earlier called the qualified name) as the `model` parameter. Management goes through the Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; `tg` and `together` are interchangeable), the `client.beta.*` SDK namespaces, or the `/v2` REST API at `https://api.together.ai`. @@ -32,7 +33,7 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; - Anything involving `tg beta endpoints` / `tg beta models` (a.k.a. `together beta ...`) CLI commands - Anything involving `client.beta.endpoints` / `client.beta.models` SDK namespaces - Traffic management: splits, rollouts (canary/blue-green/rolling), A/B tests, shadow experiments -- Autoscaling, scale-to-zero/auto-shutdown, deployment lifecycle, monitoring and events +- Autoscaling, scale-to-zero, deployment lifecycle, monitoring (dashboards, events, Prometheus scrape) - Uploading custom model weights or LoRA adapters for dedicated serving ## Hand Off To Another Skill @@ -77,24 +78,30 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; positional in your project. If the model has **multiple profiles** (e.g. BF16 + FP8), a bare-name deploy errors — pass the chosen profile's full resolved `model` resource path as the positional plus its `--config` (see models-and-configs.md). -4. Poll until `status.state` is `DEPLOYMENT_STATE_READY`. For scripted polling use the SDK - `client.beta.endpoints.deployments.retrieve(dep_id, project_id=..., endpoint_id=...)`; the - CLI `get` takes an *endpoint* id (a `dep_` id is rejected) and its `--json` reports replica - counts but not per-deployment `status.state` — that renders only in the human table. +4. Poll until `status.state` is `DEPLOYMENT_STATE_READY`. The CLI `get` now accepts an + endpoint **or** deployment ID: `tg beta endpoints get dep_... --json | jq -r '.status.state'` + is the scripted polling loop; the SDK equivalent is + `client.beta.endpoints.deployments.retrieve(dep_id, project_id=..., endpoint_id=...)`. + First-time provisioning commonly takes up to ~20 minutes. 5. Send requests to `https://api-inference.together.ai/v1` with the qualified name as `model`. -6. Scale or reconfigure with `tg beta endpoints update `; split traffic, roll out new - versions, or experiment as needed. +6. Scale, reconfigure, or set traffic weights with `tg beta endpoints update ` + (`--min/--max-replicas`, `--scaling-metrics`, `--traffic-weight`); roll out new versions or + experiment as needed. 7. Clean up: `tg beta endpoints update --min-replicas 0 --max-replicas 0` to stop - billing, or `tg beta endpoints rm ` to delete. + billing, or `tg beta endpoints rm --force` to tear everything down (it scales + deployments to zero itself). ## High-Signal Rules -- **Billing runs while replicas run.** Per minute, per replica, by hardware. Always scale to - zero (`min_replicas: 0, max_replicas: 0`) or delete when the user is done, and suggest - `--inactive-timeout` (minutes; auto-stop) for dev workloads. +- **Billing runs while replicas run, and there is no automatic idle shutdown.** Per minute, + per replica, by hardware. The old `inactive_timeout` auto-stop was removed — always scale to + zero (`min_replicas: 0, max_replicas: 0`) or delete when the user is done; a forgotten + deployment bills until someone stops it. - **A `READY` deployment serves nothing until it's in the endpoint's traffic split.** The CLI's - `deploy` routes automatically; SDK flows must call `endpoints.update(traffic_split=[...])`. - A `routing_error`/503 on a READY deployment almost always means a missing/zero weight. + `deploy` routes automatically; otherwise set a weight with `tg beta endpoints update + --traffic-weight N` (upserts one entry) or replace the split via SDK + `endpoints.update(traffic_split=[...])`. A `routing_error`/503 on a READY deployment almost + always means a missing/zero weight. - **Management IDs vs inference name.** Management calls take IDs (`ep_`, `dep_`, `cr_`, `ml_`, `rol_`, `abx_`, `exp_`); inference takes the qualified name `/`. - **The SDK requires `project_id` on every method** — derive it with `client.whoami().project_id`. @@ -108,13 +115,13 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; - **Metric names come in two disjoint catalogs.** Autoscaling uses `gpu_utilization`, `inflight_requests`, `ttft`, etc.; rollout gates use `serving_latency`, `router_error_rate`, etc. The sets are not interchangeable — a name from the wrong catalog is rejected. Charts - live in the dashboard (`https://api.together.ai/endpoints`); there's no metrics query API. -- **Deletion order matters:** stop the deployment (wait for `STOPPED`), remove it from the - traffic split, delete it, then delete the endpoint. The CLI's `rm` smart-deletes by ID - prefix and auto-detaches from the split; `--force` deletes an endpoint's deployments too. - But **neither `rm` nor `--force` stops a running deployment** — a `READY` deployment must - already be scaled to `0/0` and `STOPPED`, or `rm` fails with `deployment must be stopped or - failed before deletion (current state: ready)`. Scale down and wait for `STOPPED` first. + live in the dashboard (`https://api.together.ai/endpoints`); raw series can be scraped from + the org-scoped Prometheus-compatible metrics endpoint (beta — see api-reference.md). +- **Deletion order matters:** stop the deployment (wait for `STOPPED`), delete it, then + delete the endpoint. The CLI's `rm` smart-deletes by ID prefix and auto-detaches from the + split. Deleting still requires a stopped deployment, but `rm` now scales down for you: + `rm dep_...` on a running deployment sets `0/0` and asks you to retry once `STOPPED`, and + `rm ep_... --force` scales the endpoint's deployments to zero itself as part of teardown. - **To see which deployment/replica served a request, read the inference response headers.** The response *body*'s `model` field only echoes the endpoint's qualified name — identical for every deployment. The routing headers distinguish them: `x-cluster` is the per-deployment @@ -122,8 +129,10 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; This is the only way to verify a split, rollout, or A/B empirically. See [traffic-routing.md](references/traffic-routing.md) (Observing routing). - **Rollouts are the safe way to replace a deployment on live traffic** — create the target - stopped (`0/0`), then `tg beta endpoints rollout --from --canary`. - Metric gates are canary-only and need live traffic to evaluate. + with **at least one replica, wait for `READY`**, then + `tg beta endpoints rollout --from --canary`. Do NOT start a rollout with a + `0/0` target: traffic can shift before a replica is ready, causing `deployment_stopped` + errors. Metric gates are canary-only and need live traffic to evaluate. - The `client.beta.*` SDK surface and `together beta` CLI are **beta**: pin a current SDK release (`uv pip install --upgrade together`) and expect the surface to evolve. diff --git a/skills/together-dedicated-model-inference/references/api-reference.md b/skills/together-dedicated-model-inference/references/api-reference.md index d80b79a..707211b 100644 --- a/skills/together-dedicated-model-inference/references/api-reference.md +++ b/skills/together-dedicated-model-inference/references/api-reference.md @@ -10,7 +10,7 @@ - [Deployment States](#deployment-states) - [Autoscaling](#autoscaling) - [Scaling Metrics](#scaling-metrics) -- [Stop / Restart / Auto-Shutdown](#stop--restart--auto-shutdown) +- [Stop / Restart](#stop--restart) - [Route Traffic](#route-traffic) - [List, Paginate, Filter](#list-paginate-filter) - [Delete Resources](#delete-resources) @@ -128,7 +128,6 @@ curl -X POST "https://api.together.ai/v2/projects/$PROJECT_ID/endpoints/ep_abc12 | `config` | Yes | Config-revision resource name. Hardware and engine are fixed for the deployment's life. | | `autoscaling` | Yes | Replica bounds + optional scaling metrics. See [Autoscaling](#autoscaling). | | `enable_lora` | No | Enable LoRA adapter loading. Toggling later requires a redeploy. | -| `inactive_timeout` | No | Minutes of inactivity before auto-stop; `0` disables. | The speculator (draft model for speculative decoding) comes from the config and is pinned at creation — it cannot be set on the deployment. @@ -138,9 +137,11 @@ creation — it cannot be set on the deployment. ## Poll Deployment Status -The CLI's `get` prints each deployment's state and ready/desired replica counts alongside the -endpoint, so re-running `tg beta endpoints get ep_abc123` works for quick polling. The full -status fields below are SDK/API. +The CLI's `get` accepts an endpoint **or** deployment ID. `tg beta endpoints get ep_abc123` +prints the endpoint with each deployment's state and replica counts; `tg beta endpoints get +dep_abc123` resolves the parent endpoint and prints that deployment's detail — +`tg beta endpoints get dep_abc123 --json | jq -r '.status.state'` gives a machine-readable +state for polling loops. The same fields via the SDK: ```python d = client.beta.endpoints.deployments.retrieve( @@ -262,7 +263,10 @@ Caveats: - These names are the **autoscaling catalog**, distinct from the rollout-gate catalog (`serving_latency`, `router_error_rate`, ...) — the two are not interchangeable. -## Stop / Restart / Auto-Shutdown +## Stop / Restart + +There is **no automatic idle shutdown** — a deployment runs and bills until you stop it +(the earlier `inactive_timeout` auto-stop was removed from the API). ```python # Stop (drains, then STOPPED; billing stops) @@ -276,24 +280,26 @@ client.beta.endpoints.deployments.update( "dep_abc123", project_id=project_id, endpoint_id="ep_abc123", autoscaling={"min_replicas": 1, "max_replicas": 2}, ) - -# Auto-shutdown after 30 idle minutes -client.beta.endpoints.deployments.update( - "dep_abc123", project_id=project_id, endpoint_id="ep_abc123", - inactive_timeout=30, -) ``` CLI equivalents: `tg beta endpoints update dep_abc123 --min-replicas 0 --max-replicas 0` -(stop), `--min-replicas 1 --max-replicas 2` (restart), `--inactive-timeout 30` (auto-shutdown). +(stop), `--min-replicas 1 --max-replicas 2` (restart). A stopped deployment keeps its configuration and traffic-split weight (the weight reapplies when it scales back up), but never restarts on its own. ## Route Traffic -Traffic is routed by updating the endpoint's `traffic_split` (SDK/API only; the CLI's `deploy` -sets it automatically for the first deployment): +Traffic is routed by the endpoint's `traffic_split`. The CLI's `deploy` sets it automatically +for the first deployment. To change one deployment's weight, use the CLI's upsert (it resolves +the parent endpoint and preserves the other deployments' weights): + +```bash +tg beta endpoints update dep_abc123 --traffic-weight 30 +tg beta endpoints update dep_abc123 --traffic-weight 0 # out of rotation, no scale-down +``` + +To replace the whole split at once, update the endpoint from the SDK/API: ```python client.beta.endpoints.update( @@ -359,9 +365,11 @@ client.beta.endpoints.delete("ep_abc123", project_id=project_id) When the deployment is the endpoint's **only** traffic entry there's nothing to route to, so the empty-split limitation above blocks the pure-SDK path — use the CLI's `rm ... --force` instead, which auto-detaches from the split and deletes the deployment and endpoint together -(it smart-deletes by ID prefix; see [cli-reference.md](cli-reference.md)). Note `--force` does -**not** stop a running deployment: scale it to `0/0` and wait for `STOPPED` first, or `rm` -fails with `deployment must be stopped or failed before deletion`. +(it smart-deletes by ID prefix; see [cli-reference.md](cli-reference.md)). The CLI now handles +running deployments: `rm ep_... --force` scales the endpoint's deployments to `0/0` itself as +part of teardown, and a bare `rm dep_...` on a running deployment scales it to `0/0` and asks +you to retry once it reaches `STOPPED` (the delete itself still requires a stopped +deployment). ## Monitoring @@ -371,11 +379,33 @@ and deployment — the same series that drive autoscaling and rollout gates. Two - **Analytics dashboard** — `https://api.together.ai/endpoints` shows per-endpoint charts. Use it to monitor at a glance and to compare deployments during a rollout or A/B test. - **Events feed** — the audit trail of lifecycle changes (below). +- **Prometheus-compatible metrics endpoint** (beta) — org-scoped scrape target for your own + Prometheus/Grafana/Datadog stack (below). + +Note the autoscaling metric names (`gpu_utilization`, `inflight_requests`, ...) and rollout +gate names (`serving_latency`, `router_error_rate`, ...) are two disjoint write-side catalogs +— not interchangeable with each other or with the raw Prometheus series below. + +### Prometheus-compatible metrics endpoint (beta) + +Org-scoped, bearer-authenticated scrape target (beta — host/path may change, and access may +need enabling for your org): + +```bash +curl -H "Authorization: Bearer $TOGETHER_API_KEY" \ + "https://o11y-de2-metrics.cloud.together.ai/organizations/$ORG_ID/metrics" +``` -There is no user-facing metrics query API. Metric *names* appear only in the two write-side -catalogs — autoscaling metrics (`gpu_utilization`, `inflight_requests`, ...) and rollout gate -metrics (`serving_latency`, `router_error_rate`, ...) — and the two sets are not -interchangeable. +Works with any Prometheus-compatible scraper (set `metrics_path: +/organizations//metrics`, target `o11y-de2-metrics.cloud.together.ai`, bearer +credentials). Series are grouped by request-path stage — edge (`edge_inference_requests_total`, +`edge_inference_request_duration_ms`, `edge_inference_ttft_ms`, `edge_inference_inflight_requests`), +router (`router_inference_request_duration_seconds`, `router_inference_ttft_seconds`, +`router_pre_worker_duration_seconds`, `router_token_count`, ...), and worker +(`worker_ttft_seconds`, `worker_generation_duration_seconds`, `worker_tpot_seconds`, +`worker_token_total`, ...). Labels include `endpoint_id`/`endpoint_name`, +`deployment_id`/`deployment_name`, `replica_id`, `model`, `status_code`, `is_streaming`, and +`token_type`. ## Events Feed diff --git a/skills/together-dedicated-model-inference/references/cli-reference.md b/skills/together-dedicated-model-inference/references/cli-reference.md index 5d240c7..00819f4 100644 --- a/skills/together-dedicated-model-inference/references/cli-reference.md +++ b/skills/together-dedicated-model-inference/references/cli-reference.md @@ -42,7 +42,7 @@ tg beta endpoints deploy Deploy a model: create endpoint + deployment + route traffic in one step ls List endpoints in the project get Get endpoint details (also the default command: tg beta endpoints ep_abc123) - update Update a deployment: replica bounds, scaling metrics, auto-shutdown + update Update a deployment: replica bounds, scaling metrics, traffic weight rm Smart-delete any endpoint/deployment/experiment by ID prefix rollout Start or control a rollout (canary / blue-green / rolling) ab Start an A/B test (creates the variant deployment) @@ -57,7 +57,8 @@ tg beta models upload Upload local weights to a model record ls-files List files in a model or adapter ls-revisions List revisions for a model - update / delete + download Download model/adapter files to a local directory + update / rm Update or delete a model record (rm was previously `delete`) remote-uploads create | retrieve | list (import from Hugging Face or presigned URL) ``` @@ -82,41 +83,46 @@ name/ID (adds a deployment to it). | `--scale-up-window` | — | Seconds the metric must stay above target before adding replicas. | | `--scale-down-window` | — | Cooldown seconds between scale-downs. | | `--scale-to-zero-window` | — | Idle time before scaling to zero replicas. | -| `--inactive-timeout` | — | Minutes of inactivity before auto-stop; `0` disables. | | `--model-revision` | latest | Model revision ID (`rv_...`) to pin. | | `--placement` | — | Placement profile to use. | | `--placement.regions` | — | Comma-separated inline placement regions (alternative to a profile). | | `--placement.constraint` | — | `required` or `preferred` — how strictly to enforce inline placement. | -| `--placement.hipaa` | off | Require HIPAA-eligible placement. | | `--enable-lora` | off | Run the multi-LoRA kernel so adapters hot-load. Toggling later needs a redeploy. | +There is no `--inactive-timeout` / auto-shutdown — deployments run and bill until you stop +them. + ```bash # Deploy a public model, single replica tg beta endpoints deploy ml_CbJNwQC2ZqCU2iFT3mrCh --endpoint my-endpoint -# Explicit config, autoscaling 1-10, auto-stop after 30 idle minutes +# Explicit config, autoscaling 1-10 tg beta endpoints deploy ml_CbJNwQC2ZqCU2iFT3mrCh \ --endpoint my-endpoint \ --config cr_CbzGdmn14t3HYrXXitmKa \ - --min-replicas 1 --max-replicas 10 \ - --inactive-timeout 30 + --min-replicas 1 --max-replicas 10 ``` -Output includes the endpoint ID (`ep_...`), deployment ID (`dep_...`), and the **qualified -name** (`your-project-slug/my-endpoint`) — the value for the inference `model` parameter. +Output includes the endpoint ID (`ep_...`), deployment ID (`dep_...`), and the **endpoint +string** (`your-project-slug/my-endpoint`, shown as "Inference Name") — the value for the +inference `model` parameter. First-time provisioning commonly takes up to ~20 minutes while +weights download and hardware is allocated. ## endpoints ls / get ```bash tg beta endpoints ls [--limit N] [--after CURSOR] [--org] [--public] -tg beta endpoints get ep_abc123 +tg beta endpoints get ep_abc123 # endpoint detail: split + each deployment's state/replicas +tg beta endpoints get dep_abc123 # deployment detail (parent endpoint resolved automatically) + +# Machine-readable deployment state for polling loops +tg beta endpoints get dep_abc123 --json | jq -r '.status.state' ``` -`get` shows endpoint detail including the traffic split and, for each deployment, its state -and ready/desired replica counts — re-running it is a quick way to poll a deployment as it -comes up. Passing an endpoint ID with no subcommand (`tg beta endpoints ep_abc123`) runs `get`. -For the full status fields (scheduled replicas, status message) or deployment lists with -`filter`/`order_by`, use the SDK/API — see [api-reference.md](api-reference.md). +`get` accepts an endpoint **or** deployment ID. Re-running it is a quick way to poll a +deployment as it comes up. Passing an ID with no subcommand (`tg beta endpoints ep_abc123`) +runs `get`. For deployment lists with `filter`/`order_by`, use the SDK/API — see +[api-reference.md](api-reference.md). ## endpoints update @@ -131,12 +137,15 @@ tg beta endpoints update dep_abc123 --min-replicas 2 --max-replicas 4 tg beta endpoints update dep_abc123 --min-replicas 0 --max-replicas 0 tg beta endpoints update dep_abc123 --min-replicas 1 --max-replicas 2 -# Auto-shutdown after 30 idle minutes -tg beta endpoints update dep_abc123 --inactive-timeout 30 - # Scale on a specific metric tg beta endpoints update dep_abc123 --min-replicas 1 --max-replicas 4 \ --scaling-metrics '[{"name":"gpu_utilization","type":"METRIC_TARGET_TYPE_UTILIZATION","target":70}]' + +# Set this deployment's traffic weight (preserves the other deployments' weights) +tg beta endpoints update dep_abc123 --traffic-weight 30 + +# Take it out of rotation without scaling it down +tg beta endpoints update dep_abc123 --traffic-weight 0 ``` | Flag | Description | @@ -145,15 +154,17 @@ tg beta endpoints update dep_abc123 --min-replicas 1 --max-replicas 4 \ | `--min-replicas` / `--max-replicas` | Updated replica bounds. | | `--scale-up-window` / `--scale-down-window` / `--scale-to-zero-window` | Autoscaling stabilization windows. | | `--scaling-metrics` | Autoscaling metrics as a JSON array (see api-reference.md, Scaling Metrics). | -| `--inactive-timeout` | Minutes of inactivity before auto-stop; `0` disables. | +| `--traffic-weight` | Capacity weight in the endpoint's traffic split. Upserts just this deployment's entry; `0` stops routing to it. | | `--etag` | ETag for optimistic concurrency. | Notes: -- `update` targets deployments, not endpoints — there is still no CLI command to edit an - endpoint's traffic split. +- `--traffic-weight` edits one deployment's split entry. Replacing the whole split at once is + still an SDK/API operation (`endpoints.update(traffic_split=[...])`). - LoRA loading can't be changed after a deployment is created — redeploy with `deploy --enable-lora` to turn it on or off. +- There is no `--inactive-timeout` — auto-shutdown was removed; stop idle deployments with + `--min-replicas 0 --max-replicas 0`. ## endpoints rm (smart delete) @@ -161,23 +172,26 @@ Resolves the resource by ID prefix and deletes it: endpoint (`ep_`), deployment experiment (`abx_`), shadow experiment (`exp_`). ```bash -tg beta endpoints rm dep_abc123 # delete a deployment (must be stopped) +tg beta endpoints rm dep_abc123 # delete a deployment (must be STOPPED) tg beta endpoints rm ep_abc123 # delete an endpoint with no deployments -tg beta endpoints rm ep_abc123 --force # tear down deployments too (must be STOPPED first) +tg beta endpoints rm ep_abc123 --force # scale deployments to 0/0 and tear everything down tg beta endpoints rm abx_abc123 # end an A/B test (traffic returns to control) tg beta endpoints rm exp_abc123 # stop a shadow experiment ``` -Deleting a deployment auto-detaches it from the traffic split and any experiments. A running -deployment must be stopped first (scale to `0/0`, wait for `STOPPED`). `rm` does not accept -rollout IDs (`rol_`) — delete rollouts from the SDK/API. +Deleting a deployment auto-detaches it from the traffic split and any experiments. Running +deployments: `rm dep_...` on a running deployment scales it to `0/0` for you and asks you to +retry once it reaches `STOPPED` (the delete itself still requires a stopped deployment); +`rm ep_... --force` scales the endpoint's deployments down itself as part of teardown. `rm` +does not accept rollout IDs (`rol_`) — delete rollouts from the SDK/API. ## endpoints rollout One command both starts and controls rollouts. (This command was previously named `promote`; older examples showing `tg beta endpoints promote` map one-to-one onto `rollout`.) To start: -pass the **target** deployment ID -(create it stopped, `0/0` bounds) and a strategy flag. +pass the **target** deployment ID and a strategy flag. The target must have at least one +replica and be `READY` first — starting with a `0/0` target risks `deployment_stopped` errors +(see [traffic-routing.md](traffic-routing.md), Rollouts). ```bash # Canary (default ladder 10,50,100 over 10m intervals) @@ -295,6 +309,12 @@ tg beta models remote-uploads list tg beta models list tg beta models ls-files ml_abc123 tg beta models ls-revisions ml_abc123 + +# Download files back to your machine (--format hf = HuggingFace snapshot layout) +tg beta models download ml_abc123 ./local-dir [--revision rv_...] [--format hf] + +# Delete a model record (metadata only; renamed from `delete`) +tg beta models rm ml_abc123 ``` `--type` values are asymmetric: **write** commands (`upload`, `remote-uploads create`) take @@ -308,9 +328,10 @@ Full upload flows (requirements, S3 archive format, polling, deploying the resul These are SDK/API-only operations; don't hunt for CLI flags: -- Edit an endpoint's traffic split (`endpoints update` targets deployments only). -- List deployments or use `filter` / `order_by` (though `get` shows each deployment's state - and replica counts for quick polling). +- Replace an endpoint's traffic split in one call (`update --traffic-weight` upserts one + deployment at a time; the full-split write is `endpoints.update(traffic_split=[...])`). +- List deployments or use `filter` / `order_by` (though `get` reads a single deployment and + shows every deployment's state on the endpoint view). - Create a metric-gated canary rollout, or delete a rollout. - Ramp/edit an existing A/B experiment, or update a shadow experiment's sampling. - Read the events feed. diff --git a/skills/together-dedicated-model-inference/references/models-and-configs.md b/skills/together-dedicated-model-inference/references/models-and-configs.md index ae2ddae..f4e7870 100644 --- a/skills/together-dedicated-model-inference/references/models-and-configs.md +++ b/skills/together-dedicated-model-inference/references/models-and-configs.md @@ -8,6 +8,7 @@ - [Pricing and hardware](#pricing-and-hardware) - [Upload a custom model](#upload-a-custom-model) - [Upload a LoRA adapter](#upload-a-lora-adapter) +- [Download and delete models](#download-and-delete-models) - [Upload troubleshooting](#upload-troubleshooting) ## Choose a model @@ -165,7 +166,10 @@ Single-GPU per-hour rates (multi-GPU configs cost proportionally more; check | GPU | Hardware ID | Cost/hour | | --- | --- | --- | | H100 80GB | `1xnvidia-h100-80gb` | $5.49 | +| H200 141GB | `1xnvidia-h200-141gb` | $6.65 | | B200 180GB | `1xnvidia-b200-180gb` | $8.99 | +| GB300 280GB | `1xnvidia-gb300-280gb` | $15.00 | +| B300 280GB | `1xnvidia-b300-280gb` | $15.00 | As a scale reference: one H100 replica running continuously is about $132/day, or roughly $3,950 over a 30-day month. @@ -173,10 +177,28 @@ $3,950 over a 30-day month. Cost levers: - `min_replicas` sets the cost floor (always running); `max_replicas` sets the ceiling. -- `inactive_timeout` (auto-shutdown) stops idle deployments; the next request pays a cold start. +- **Stop when idle** — scale to `0/0` (or delete). There is **no automatic idle shutdown**; + a deployment runs and bills until you stop it, and the first request after a restart pays + a cold start. - **On-demand** (per-minute, no commitment) vs **reserved** (committed term, lower effective rate, guaranteed hardware — contact Together sales). +### Instance types and capacity + +A config's `accelerator_type` + `accelerator_count` selectors map to a deployable **instance +type** (e.g. `1xnvidia-h100-80gb`) — the unit of hardware you pay for while replicas run. +Check per-hour price and per-region capacity before deploying: + +```bash +curl -s -H "Authorization: Bearer $TOGETHER_API_KEY" \ + https://api.together.ai/v2/public/inference-instance-types +``` + +Each instance type lists its `regions`, and each region reports `headroom` — a best-effort +hint of how many more replicas currently fit. A headroom of N with `RELATION_GTE` means at +least N units are free (the true number may be higher). Use it to pick a region with capacity +before you deploy. + DMI vs serverless rule of thumb: DMI wins when a replica stays busy most of the day (fixed cost spread over high throughput, plus reserved capacity and predictable latency); serverless wins for low or bursty traffic where a dedicated replica would idle. @@ -185,17 +207,22 @@ wins for low or bursty traffic where a dedicated replica would idle. Serve your own fine-tuned weights. Requirements: -- **Source**: Hugging Face Hub or an S3 presigned URL (or local files via CLI upload). +- **Source**: your local machine (CLI upload), Hugging Face Hub, or an S3 presigned URL. - **Architecture**: a fine-tuned variant of a base model Together supports for dedicated inference — uploads cannot introduce new architectures. -- **Type**: text generation or embedding; **must fit on a single GPU** (multi-GPU uploads not - supported). +- **Type**: text generation. - **Format**: standard Hugging Face repo layout compatible with `from_pretrained` (`config.json`, `*.safetensors`, `tokenizer.json`, ...). - **S3 archives**: a single `.zip` / `.tar.gz` with the files at the **archive root** (no nested top-level directory); presigned URL valid for at least 100 minutes. From inside the model dir: `tar -czvf ../model.tar.gz .` +Meeting these is necessary but not sufficient: an unsupported base model, layer type, or +adapter rank is rejected with an error identifying the problem at create or upload time. + +Uploaded models are **Private** by default (visible only to your project); Internal makes a +model visible to your whole organization, Public to anyone. + ### Step 1 — Register the model record Every custom model references a supported base model via `base_model_id`. **Name it bare** @@ -291,6 +318,21 @@ Deploy the adapter's `ml_...` ID like a base model, using a config for its base hot-load adapters onto a running deployment, the deployment must have been created with `--enable-lora` (toggling later requires a redeploy). +## Download and delete models + +Download a model's or adapter's files back to your machine, or delete the record: + +```bash +# Download files (--format hf lays them out as a HuggingFace snapshot) +tg beta models download ml_abc123 ./local-dir [--revision rv_...] [--format hf] + +# Delete the model record (does not delete uploaded files) +tg beta models rm ml_abc123 +``` + +Note the delete command is `rm` (the earlier `delete` name is gone), and `update` can no +longer change a record's base model. + ## Upload troubleshooting - **"Model not found" during upload** — create the record first (`models create`) and pass diff --git a/skills/together-dedicated-model-inference/references/traffic-routing.md b/skills/together-dedicated-model-inference/references/traffic-routing.md index 29d5ac3..0dff382 100644 --- a/skills/together-dedicated-model-inference/references/traffic-routing.md +++ b/skills/together-dedicated-model-inference/references/traffic-routing.md @@ -104,7 +104,18 @@ control at N=200, not exactly 95%. ## Traffic splits (weights) -Set by updating the endpoint (SDK/API only): +Set one deployment's weight from the CLI — it resolves the parent endpoint and preserves the +other deployments' weights, so run it once per deployment: + +```bash +tg beta endpoints update dep_abc123 --traffic-weight 70 +tg beta endpoints update dep_def456 --traffic-weight 30 + +# Take a deployment out of rotation without scaling it down +tg beta endpoints update dep_abc123 --traffic-weight 0 +``` + +Or replace the whole split at once from the SDK/API: ```python client.beta.endpoints.update( @@ -137,10 +148,14 @@ the split when replacing a deployment on live traffic (new model version, new co Requirements: -- Source and target under the same endpoint. +- Source and target under the same endpoint. The source can be a live weighted deployment or + an A/B / shadow experiment member (the CLI detaches it). - The source must be referenced in the traffic split (a rollout only shifts traffic already routed to the source; otherwise it runs but moves nothing). -- Create the target **stopped** (`0/0` bounds) so the rollout scales it up from zero. +- **The target must have at least one replica and be `READY` before you start.** Do NOT start + a rollout with the target at `0/0`: traffic can shift before the target has a ready replica, + causing `deployment_stopped` errors. The rollout adjusts the target's replica count after it + starts. (Earlier guidance said to create the target stopped — that's obsolete.) - One active rollout per endpoint (`409` on create otherwise). ### Strategies diff --git a/skills/together-dedicated-model-inference/scripts/deploy_model.py b/skills/together-dedicated-model-inference/scripts/deploy_model.py index f8609d6..98cdfcb 100644 --- a/skills/together-dedicated-model-inference/scripts/deploy_model.py +++ b/skills/together-dedicated-model-inference/scripts/deploy_model.py @@ -20,8 +20,8 @@ uv pip install --upgrade together # a release with the beta DMI surface (client.beta.*) export TOGETHER_API_KEY=your_key -Billing note: replicas bill per minute while running. `stop` (scale to 0/0) -halts billing without deleting anything. +Billing note: replicas bill per minute while running and there is NO automatic +idle shutdown. `stop` (scale to 0/0) halts billing without deleting anything. """ import argparse @@ -68,7 +68,6 @@ def deploy( endpoint_name: str, min_replicas: int = 1, max_replicas: int = 1, - inactive_timeout: int | None = None, ): """Create an endpoint, attach a deployment, and route 100% of traffic to it. @@ -84,7 +83,6 @@ def deploy( model=f"projects/{PROJECT_ID}/models/{model_id}", config=f"projects/{config_project_id}/configs/{config_id}", autoscaling={"min_replicas": min_replicas, "max_replicas": max_replicas}, - inactive_timeout=inactive_timeout, ) print(f"Created deployment: {deployment.id}") @@ -206,7 +204,6 @@ def main() -> int: p.add_argument("--endpoint", required=True, help="New endpoint name") p.add_argument("--min", type=int, default=1, dest="min_replicas") p.add_argument("--max", type=int, default=1, dest="max_replicas") - p.add_argument("--inactive-timeout", type=int, default=None, help="Idle minutes before auto-stop") p = sub.add_parser("status", help="Show deployment status") p.add_argument("--endpoint", required=True) @@ -238,7 +235,7 @@ def main() -> int: elif args.command == "deploy": deploy( args.model, args.config, args.config_project, args.endpoint, - args.min_replicas, args.max_replicas, args.inactive_timeout, + args.min_replicas, args.max_replicas, ) elif args.command == "status": show_status(args.endpoint, args.deployment) diff --git a/skills/together-dedicated-model-inference/scripts/rollout_new_version.py b/skills/together-dedicated-model-inference/scripts/rollout_new_version.py index 9f666b3..1dd3508 100644 --- a/skills/together-dedicated-model-inference/scripts/rollout_new_version.py +++ b/skills/together-dedicated-model-inference/scripts/rollout_new_version.py @@ -3,7 +3,7 @@ Together AI Dedicated Model Inference -- Roll Out a New Deployment Replace a live deployment with a new one (new model version, new config, or -new hardware) without downtime: create the target stopped, start a rollout +new hardware) without downtime: create the target and wait for READY, start a rollout (blue-green or metric-gated canary), watch it, and control its lifecycle. Usage: @@ -19,6 +19,9 @@ export TOGETHER_API_KEY=your_key Notes: + - The target must have at least one replica and be READY before the rollout + starts (a 0/0 target risks deployment_stopped errors mid-shift). `prepare` + creates it with one replica and waits; keep it OUT of the traffic split. - The source must be in the endpoint's traffic split, or the rollout shifts nothing. - Metric gates are canary-only and need live traffic on the endpoint to evaluate. - After COMPLETED there's no rollback; revert by rolling out in reverse. @@ -37,17 +40,32 @@ def prepare_target(endpoint_id: str, model_id: str, config_id: str, config_project_id: str, name: str): - """Create the target deployment stopped (0/0) so the rollout scales it from zero.""" + """Create the target deployment with one replica and wait for READY. + + The rollout adjusts the target's replica count after it starts, but it must + already be READY when the rollout begins — traffic can shift before a 0/0 + target has a ready replica, causing deployment_stopped errors. + """ deployment = client.beta.endpoints.deployments.create( endpoint_id, project_id=PROJECT_ID, name=name, model=f"projects/{PROJECT_ID}/models/{model_id}", config=f"projects/{config_project_id}/configs/{config_id}", - autoscaling={"min_replicas": 0, "max_replicas": 0}, + autoscaling={"min_replicas": 1, "max_replicas": 1}, ) - print(f"Created stopped target deployment: {deployment.id}") - print("Do NOT add it to the traffic split; the rollout moves traffic itself.") + print(f"Created target deployment: {deployment.id} — waiting for READY...") + while True: + d = client.beta.endpoints.deployments.retrieve( + deployment.id, project_id=PROJECT_ID, endpoint_id=endpoint_id + ) + print(f" {d.status.state}") + if d.status.state == "DEPLOYMENT_STATE_READY": + break + if d.status.state == "DEPLOYMENT_STATE_FAILED": + raise RuntimeError(f"Target deployment failed: {d.status.message}") + time.sleep(15) + print("Target READY. Do NOT add it to the traffic split; the rollout moves traffic itself.") return deployment @@ -148,7 +166,7 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="command", required=True) - p = sub.add_parser("prepare", help="Create the stopped target deployment") + p = sub.add_parser("prepare", help="Create the target deployment and wait for READY") p.add_argument("--endpoint", required=True) p.add_argument("--model", required=True, help="Model ID (ml_...)") p.add_argument("--config", required=True, help="Config revision ID (cr_...)") @@ -158,7 +176,7 @@ def main() -> int: p = sub.add_parser("start", help="Create and start the rollout") p.add_argument("--endpoint", required=True) p.add_argument("--source", required=True, help="Source deployment ID (currently serving)") - p.add_argument("--target", required=True, help="Target deployment ID (stopped)") + p.add_argument("--target", required=True, help="Target deployment ID (READY, not in the split)") p.add_argument("--replicas", type=int, required=True, help="Final target replica count (>= 1)") p.add_argument("--canary", action="store_true", help="Staged ladder instead of blue-green") p.add_argument("--steps", default=None, help="Canary traffic ladder, e.g. 25,50,100") diff --git a/skills/together-dedicated-model-inference/scripts/upload_custom_model.py b/skills/together-dedicated-model-inference/scripts/upload_custom_model.py index eb5a4b5..841460a 100644 --- a/skills/together-dedicated-model-inference/scripts/upload_custom_model.py +++ b/skills/together-dedicated-model-inference/scripts/upload_custom_model.py @@ -17,8 +17,9 @@ Upload requirements: - Weights must be a fine-tuned variant of a base model Together supports for - dedicated inference (uploads can't introduce new architectures), and must - fit on a single GPU. + dedicated inference (uploads can't introduce new architectures). Meeting + the requirements is necessary but not sufficient: unsupported base models, + layer types, or adapter ranks are rejected at create/upload time. - Hugging Face repo layout compatible with from_pretrained; adapters need adapter_config.json + adapter_model.safetensors. - S3 sources: one .zip/.tar.gz archive with files at the archive ROOT, and a From c6752b9d3270d72f26ac591d8841ea861f7e3ae4 Mon Sep 17 00:00:00 2001 From: Zain Hasan Date: Tue, 14 Jul 2026 12:08:13 -0700 Subject: [PATCH 10/11] remove pricing details --- .../references/models-and-configs.md | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/skills/together-dedicated-model-inference/references/models-and-configs.md b/skills/together-dedicated-model-inference/references/models-and-configs.md index f4e7870..355b1cc 100644 --- a/skills/together-dedicated-model-inference/references/models-and-configs.md +++ b/skills/together-dedicated-model-inference/references/models-and-configs.md @@ -160,19 +160,12 @@ DMI bills **per minute, per running replica, by hardware** — model and token v affect cost. A replica stops billing as soon as it scales down; a stopped deployment costs nothing. -Single-GPU per-hour rates (multi-GPU configs cost proportionally more; check -[the pricing page](https://docs.together.ai/docs/dedicated-endpoints/pricing) for current rates): - -| GPU | Hardware ID | Cost/hour | -| --- | --- | --- | -| H100 80GB | `1xnvidia-h100-80gb` | $5.49 | -| H200 141GB | `1xnvidia-h200-141gb` | $6.65 | -| B200 180GB | `1xnvidia-b200-180gb` | $8.99 | -| GB300 280GB | `1xnvidia-gb300-280gb` | $15.00 | -| B300 280GB | `1xnvidia-b300-280gb` | $15.00 | - -As a scale reference: one H100 replica running continuously is about $132/day, or roughly -$3,950 over a 30-day month. +Pricing is per hour per single GPU, and multi-GPU configs cost proportionally more (a +four-GPU config costs four times the single-GPU rate). For current rates per hardware type +(H100, H200, B200, ...), check the +[pricing page](https://docs.together.ai/docs/dedicated-endpoints/pricing) or query +`/v2/public/inference-instance-types` (see [Instance types and +capacity](#instance-types-and-capacity)) — don't quote rates from memory. Cost levers: From e97d8fadc2ef30d5e644cf01581bac35414884d3 Mon Sep 17 00:00:00 2001 From: Zain Hasan Date: Tue, 14 Jul 2026 12:54:44 -0700 Subject: [PATCH 11/11] remove rollouts --- .../SKILL.md | 32 ++- .../agents/openai.yaml | 2 +- .../references/api-reference.md | 24 +- .../references/cli-reference.md | 42 +--- .../references/models-and-configs.md | 3 +- .../references/traffic-routing.md | 219 +++--------------- .../scripts/rollout_new_version.py | 217 ----------------- 7 files changed, 62 insertions(+), 477 deletions(-) delete mode 100644 skills/together-dedicated-model-inference/scripts/rollout_new_version.py diff --git a/skills/together-dedicated-model-inference/SKILL.md b/skills/together-dedicated-model-inference/SKILL.md index f216385..90a0bee 100644 --- a/skills/together-dedicated-model-inference/SKILL.md +++ b/skills/together-dedicated-model-inference/SKILL.md @@ -1,6 +1,6 @@ --- name: together-dedicated-model-inference -description: "Deploy and operate models on dedicated GPUs with Together AI's Dedicated Model Inference (DMI, the v2 dedicated endpoints API): beta endpoints, deployments, deployment profiles and hardware configs, autoscaling, traffic splitting, canary/blue-green/rolling rollouts, A/B tests, shadow experiments, Prometheus metrics, and custom model or LoRA adapter uploads. Reach for it whenever the user mentions together beta endpoints or tg beta commands, client.beta.endpoints, DMI resources like ep_/dep_/cr_/ml_ IDs, or wants production model serving with traffic management on Together AI. Use together-dedicated-endpoints only for the legacy v1 endpoints API." +description: "Deploy and operate models on dedicated GPUs with Together AI's Dedicated Model Inference (DMI, the v2 dedicated endpoints API): beta endpoints, deployments, deployment profiles and hardware configs, autoscaling, traffic splitting, A/B tests, shadow experiments, Prometheus metrics, and custom model or LoRA adapter uploads. Reach for it whenever the user mentions together beta endpoints or tg beta commands, client.beta.endpoints, DMI resources like ep_/dep_/cr_/ml_ IDs, or wants production model serving with traffic management on Together AI. Use together-dedicated-endpoints only for the legacy v1 endpoints API." --- # Together Dedicated Model Inference @@ -32,7 +32,7 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; - Deploying a model (public, fine-tuned, or uploaded) on dedicated GPUs via the v2 API - Anything involving `tg beta endpoints` / `tg beta models` (a.k.a. `together beta ...`) CLI commands - Anything involving `client.beta.endpoints` / `client.beta.models` SDK namespaces -- Traffic management: splits, rollouts (canary/blue-green/rolling), A/B tests, shadow experiments +- Traffic management: splits, A/B tests, shadow experiments - Autoscaling, scale-to-zero, deployment lifecycle, monitoring (dashboards, events, Prometheus scrape) - Uploading custom model weights or LoRA adapters for dedicated serving @@ -54,9 +54,8 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; - Read [references/cli-reference.md](references/cli-reference.md) for the one-command CLI path - **Endpoint/deployment lifecycle from the SDK or API** (create, poll, scale, stop, delete, list filters) - Read [references/api-reference.md](references/api-reference.md) -- **Split traffic, rollouts, A/B tests, shadow experiments** +- **Split traffic, A/B tests, shadow experiments** - Read [references/traffic-routing.md](references/traffic-routing.md) - - Start with [scripts/rollout_new_version.py](scripts/rollout_new_version.py) for rollouts - **Choose a model or config, pricing, upload custom weights or LoRA adapters** - Read [references/models-and-configs.md](references/models-and-configs.md) - Start with [scripts/upload_custom_model.py](scripts/upload_custom_model.py) @@ -85,7 +84,7 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; First-time provisioning commonly takes up to ~20 minutes. 5. Send requests to `https://api-inference.together.ai/v1` with the qualified name as `model`. 6. Scale, reconfigure, or set traffic weights with `tg beta endpoints update ` - (`--min/--max-replicas`, `--scaling-metrics`, `--traffic-weight`); roll out new versions or + (`--min/--max-replicas`, `--scaling-metrics`, `--traffic-weight`); split traffic or experiment as needed. 7. Clean up: `tg beta endpoints update --min-replicas 0 --max-replicas 0` to stop billing, or `tg beta endpoints rm --force` to tear everything down (it scales @@ -103,7 +102,7 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; `endpoints.update(traffic_split=[...])`. A `routing_error`/503 on a READY deployment almost always means a missing/zero weight. - **Management IDs vs inference name.** Management calls take IDs (`ep_`, `dep_`, `cr_`, `ml_`, - `rol_`, `abx_`, `exp_`); inference takes the qualified name `/`. + `abx_`, `exp_`); inference takes the qualified name `/`. - **The SDK requires `project_id` on every method** — derive it with `client.whoami().project_id`. The SDK also takes models/configs as resource names (`projects/{p}/models/{ml}`, `projects/{p}/configs/{cr}`), while the CLI takes bare IDs. Use the config's own `projectId` @@ -112,11 +111,10 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; Prefer shifting traffic by changing replica counts; keep weights stable. - **Scale-to-zero is all-or-nothing:** `min_replicas: 0` with a positive `max_replicas` is rejected. `0/0` stops; on a running deployment the floor is otherwise `1`. -- **Metric names come in two disjoint catalogs.** Autoscaling uses `gpu_utilization`, - `inflight_requests`, `ttft`, etc.; rollout gates use `serving_latency`, `router_error_rate`, - etc. The sets are not interchangeable — a name from the wrong catalog is rejected. Charts - live in the dashboard (`https://api.together.ai/endpoints`); raw series can be scraped from - the org-scoped Prometheus-compatible metrics endpoint (beta — see api-reference.md). +- **Autoscaling metric names are their own catalog** (`gpu_utilization`, `inflight_requests`, + `ttft`, ...) — raw Prometheus series names are rejected in scaling policies. Charts live in + the dashboard (`https://api.together.ai/endpoints`); raw series can be scraped from the + org-scoped Prometheus-compatible metrics endpoint (beta — see api-reference.md). - **Deletion order matters:** stop the deployment (wait for `STOPPED`), delete it, then delete the endpoint. The CLI's `rm` smart-deletes by ID prefix and auto-detaches from the split. Deleting still requires a stopped deployment, but `rm` now scales down for you: @@ -126,13 +124,12 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; The response *body*'s `model` field only echoes the endpoint's qualified name — identical for every deployment. The routing headers distinguish them: `x-cluster` is the per-deployment cluster ID and `worker_url` (inside the `x-i-router-log-event` header) is the replica pod. - This is the only way to verify a split, rollout, or A/B empirically. See + This is the only way to verify a split or A/B empirically. See [traffic-routing.md](references/traffic-routing.md) (Observing routing). -- **Rollouts are the safe way to replace a deployment on live traffic** — create the target - with **at least one replica, wait for `READY`**, then - `tg beta endpoints rollout --from --canary`. Do NOT start a rollout with a - `0/0` target: traffic can shift before a replica is ready, causing `deployment_stopped` - errors. Metric gates are canary-only and need live traffic to evaluate. +- **To replace a deployment on live traffic**, create the new deployment on the same + endpoint, wait for `READY`, then shift traffic gradually with `--traffic-weight` (and + replica counts), watching the dashboard between steps. Take the old deployment out with + `--traffic-weight 0`, then scale it down and delete it. - The `client.beta.*` SDK surface and `together beta` CLI are **beta**: pin a current SDK release (`uv pip install --upgrade together`) and expect the surface to evolve. @@ -143,7 +140,6 @@ Together CLI (`tg beta ...` — install with `uv tool install "together[cli]"`; - **Traffic routing guide**: [references/traffic-routing.md](references/traffic-routing.md) - **Models, configs, pricing, uploads**: [references/models-and-configs.md](references/models-and-configs.md) - **Deploy workflow script**: [scripts/deploy_model.py](scripts/deploy_model.py) -- **Rollout workflow script**: [scripts/rollout_new_version.py](scripts/rollout_new_version.py) - **Custom model upload script**: [scripts/upload_custom_model.py](scripts/upload_custom_model.py) ## Official Docs diff --git a/skills/together-dedicated-model-inference/agents/openai.yaml b/skills/together-dedicated-model-inference/agents/openai.yaml index 3ea3ee8..30f1a9c 100644 --- a/skills/together-dedicated-model-inference/agents/openai.yaml +++ b/skills/together-dedicated-model-inference/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Together Dedicated Model Inference" short_description: "Together AI dedicated model inference (v2 endpoints)" - default_prompt: "Use $together-dedicated-model-inference to deploy a model on dedicated Together AI GPUs (beta endpoints/deployments), manage autoscaling, split traffic, run rollouts, A/B tests, or shadow experiments, or upload a custom model or LoRA adapter." + default_prompt: "Use $together-dedicated-model-inference to deploy a model on dedicated Together AI GPUs (beta endpoints/deployments), manage autoscaling, split traffic, run A/B tests or shadow experiments, or upload a custom model or LoRA adapter." diff --git a/skills/together-dedicated-model-inference/references/api-reference.md b/skills/together-dedicated-model-inference/references/api-reference.md index 707211b..ada4d31 100644 --- a/skills/together-dedicated-model-inference/references/api-reference.md +++ b/skills/together-dedicated-model-inference/references/api-reference.md @@ -25,7 +25,7 @@ `/v2/projects/{project_id}/endpoints/{endpoint_id}/deployments`). Auth is `Authorization: Bearer $TOGETHER_API_KEY`. - **Inference API**: `https://api-inference.together.ai/v1` — same request shapes as serverless. -- **SDK namespaces**: `client.beta.endpoints` (with `.deployments`, `.rollouts`, +- **SDK namespaces**: `client.beta.endpoints` (with `.deployments`, `.ab_experiments`, `.shadow_experiments`) and `client.beta.models` (with `.configs`, `.remote_uploads`). TypeScript uses the camelCase equivalents (`client.beta.endpoints`, `abExperiments`, `shadowExperiments`, `remoteUploads`). @@ -50,7 +50,6 @@ project_id = client.whoami().project_id | Config | `cr_...` | A config revision. Configs are immutable; each revision has a new ID. | | Endpoint | `ep_...` | Inference uses the qualified name `/`. | | Deployment | `dep_...` | Qualified name `//`. | -| Rollout | `rol_...` | See [traffic-routing.md](traffic-routing.md). | | A/B experiment | `abx_...` | See [traffic-routing.md](traffic-routing.md). | | Shadow experiment / target | `exp_...` / `shet_...` | See [traffic-routing.md](traffic-routing.md). | | Upload job | `job_...` | See [models-and-configs.md](models-and-configs.md). | @@ -260,8 +259,8 @@ Caveats: responses** — scaling on them without `"stream": true` traffic won't work. - Latency metrics default to p95 over the measurement window; set `"percentile"` to `"p50"`, `"p90"`, `"p95"`, or `"p99"` to change it. -- These names are the **autoscaling catalog**, distinct from the rollout-gate catalog - (`serving_latency`, `router_error_rate`, ...) — the two are not interchangeable. +- These names are the **autoscaling catalog** — raw Prometheus series names are not valid + here. ## Stop / Restart @@ -309,8 +308,8 @@ client.beta.endpoints.update( ) ``` -Weights are relative capacity: share = weight × ready replicas. Full semantics, plus rollouts, -A/B tests, and shadow experiments, are in [traffic-routing.md](traffic-routing.md). +Weights are relative capacity: share = weight × ready replicas. Full semantics, plus gradual +cutovers, A/B tests, and shadow experiments, are in [traffic-routing.md](traffic-routing.md). ## List, Paginate, Filter @@ -374,17 +373,16 @@ deployment). ## Monitoring DMI records latency, throughput, replica-count, and utilization metrics for every endpoint -and deployment — the same series that drive autoscaling and rollout gates. Two surfaces: +and deployment — the same series that drive autoscaling. Three surfaces: - **Analytics dashboard** — `https://api.together.ai/endpoints` shows per-endpoint charts. - Use it to monitor at a glance and to compare deployments during a rollout or A/B test. + Use it to monitor at a glance and to compare deployments during a cutover or A/B test. - **Events feed** — the audit trail of lifecycle changes (below). - **Prometheus-compatible metrics endpoint** (beta) — org-scoped scrape target for your own Prometheus/Grafana/Datadog stack (below). -Note the autoscaling metric names (`gpu_utilization`, `inflight_requests`, ...) and rollout -gate names (`serving_latency`, `router_error_rate`, ...) are two disjoint write-side catalogs -— not interchangeable with each other or with the raw Prometheus series below. +Note the autoscaling metric names (`gpu_utilization`, `inflight_requests`, ...) are their +own catalog — not interchangeable with the raw Prometheus series below. ### Prometheus-compatible metrics endpoint (beta) @@ -410,14 +408,14 @@ router (`router_inference_request_duration_seconds`, `router_inference_ttft_seco ## Events Feed Each endpoint has an audit feed (newest first) merging endpoint- and deployment-scoped events: -scale-ups, traffic shifts, readiness changes, rollout pauses. +scale-ups, traffic shifts, readiness changes. ```python events = client.beta.endpoints.list_events("ep_abc123", project_id=project_id, limit=50) ``` Optional filters: `types` (event-type strings), `since` / `until` (time range), `subject_id` -(for example `rol_abc123` for one rollout's audit trail), `deployment_ids` (scope to specific +(scope to one subject's audit trail), `deployment_ids` (scope to specific deployments), `limit` / `after` (max 500, default 50). Events come back newest-first — reverse them for a chronological timeline. diff --git a/skills/together-dedicated-model-inference/references/cli-reference.md b/skills/together-dedicated-model-inference/references/cli-reference.md index 00819f4..71868be 100644 --- a/skills/together-dedicated-model-inference/references/cli-reference.md +++ b/skills/together-dedicated-model-inference/references/cli-reference.md @@ -11,7 +11,6 @@ and wires up the underlying resources. - [endpoints ls / get](#endpoints-ls--get) - [endpoints update](#endpoints-update) - [endpoints rm (smart delete)](#endpoints-rm-smart-delete) -- [endpoints rollout](#endpoints-rollout) - [endpoints ab (A/B tests)](#endpoints-ab-ab-tests) - [endpoints shadow (shadow experiments)](#endpoints-shadow-shadow-experiments) - [models commands](#models-commands) @@ -44,7 +43,6 @@ tg beta endpoints get Get endpoint details (also the default command: tg beta endpoints ep_abc123) update Update a deployment: replica bounds, scaling metrics, traffic weight rm Smart-delete any endpoint/deployment/experiment by ID prefix - rollout Start or control a rollout (canary / blue-green / rolling) ab Start an A/B test (creates the variant deployment) shadow Start a shadow experiment (creates the shadow deployment) @@ -183,44 +181,7 @@ Deleting a deployment auto-detaches it from the traffic split and any experiment deployments: `rm dep_...` on a running deployment scales it to `0/0` for you and asks you to retry once it reaches `STOPPED` (the delete itself still requires a stopped deployment); `rm ep_... --force` scales the endpoint's deployments down itself as part of teardown. `rm` -does not accept rollout IDs (`rol_`) — delete rollouts from the SDK/API. - -## endpoints rollout - -One command both starts and controls rollouts. (This command was previously named `promote`; -older examples showing `tg beta endpoints promote` map one-to-one onto `rollout`.) To start: -pass the **target** deployment ID and a strategy flag. The target must have at least one -replica and be `READY` first — starting with a `0/0` target risks `deployment_stopped` errors -(see [traffic-routing.md](traffic-routing.md), Rollouts). - -```bash -# Canary (default ladder 10,50,100 over 10m intervals) -tg beta endpoints rollout dep_target456 --from dep_source123 --canary - -# Custom canary ladder -tg beta endpoints rollout dep_target456 --from dep_source123 \ - --canary --steps 25,50,75,100 --interval 10m - -# Blue-green (single cutover; the default strategy) / rolling (capacity-preserving swap) -tg beta endpoints rollout dep_target456 --from dep_source123 --blue-green -tg beta endpoints rollout dep_target456 --from dep_source123 --rolling - -# Keep the source scaled up after completion instead of draining it -tg beta endpoints rollout dep_target456 --from dep_source123 --canary --source-cleanup keep -``` - -`--from` is inferred from the traffic split when omitted. To control a running rollout, pass -the **rollout** ID (`rol_...`) and a control flag: - -```bash -tg beta endpoints rollout rol_abc123 --pause --reason "holding for review" -tg beta endpoints rollout rol_abc123 --continue # resume -tg beta endpoints rollout rol_abc123 --complete # fast-forward, all traffic to target -tg beta endpoints rollout rol_abc123 --abort --reason "tail latency regression" -``` - -Metric-gated canaries are SDK/API only (`rollouts.create` with a `metrics` block) — see -[traffic-routing.md](traffic-routing.md). +smart-deletes endpoints, deployments, and experiments only. ## endpoints ab (A/B tests) @@ -332,7 +293,6 @@ These are SDK/API-only operations; don't hunt for CLI flags: deployment at a time; the full-split write is `endpoints.update(traffic_split=[...])`). - List deployments or use `filter` / `order_by` (though `get` reads a single deployment and shows every deployment's state on the endpoint view). -- Create a metric-gated canary rollout, or delete a rollout. - Ramp/edit an existing A/B experiment, or update a shadow experiment's sampling. - Read the events feed. - Inference itself — point the SDK or curl at `https://api-inference.together.ai/v1`. diff --git a/skills/together-dedicated-model-inference/references/models-and-configs.md b/skills/together-dedicated-model-inference/references/models-and-configs.md index 355b1cc..abcc2b8 100644 --- a/skills/together-dedicated-model-inference/references/models-and-configs.md +++ b/skills/together-dedicated-model-inference/references/models-and-configs.md @@ -148,7 +148,8 @@ Key properties: - **Configs are immutable.** New revisions get new `cr_...` IDs; a deployment pins the revision you selected, so hardware and engine never change underneath it. - **Hardware is fixed for a deployment's life.** To change hardware, create a new deployment - with a different config and shift traffic over (ideally with a rollout). + with a different config and shift traffic over with weights and replica counts (see + [traffic-routing.md](traffic-routing.md)). - **Speculative decoding** is declared by the config (`draftModel` in the response when enabled); the deployment's speculator is derived and pinned at create — you can't set it yourself. Speculative decoding raises average throughput but can add occasional tail-latency diff --git a/skills/together-dedicated-model-inference/references/traffic-routing.md b/skills/together-dedicated-model-inference/references/traffic-routing.md index 0dff382..5932eed 100644 --- a/skills/together-dedicated-model-inference/references/traffic-routing.md +++ b/skills/together-dedicated-model-inference/references/traffic-routing.md @@ -1,7 +1,7 @@ # Dedicated Model Inference — Traffic Routing -How an endpoint decides which deployment serves each request, and the four tools for moving -traffic: weighted splits, rollouts, A/B tests, and shadow experiments. +How an endpoint decides which deployment serves each request, and the tools for moving +traffic: weighted splits, A/B tests, and shadow experiments. ## Contents @@ -9,13 +9,7 @@ traffic: weighted splits, rollouts, A/B tests, and shadow experiments. - [Stickiness](#stickiness) - [Observing routing](#observing-routing) - [Traffic splits (weights)](#traffic-splits-weights) -- [Rollouts](#rollouts) - - [Strategies](#strategies) - - [Create and start](#create-and-start) - - [Metric gates (canary only)](#metric-gates-canary-only) - - [Lifecycle controls](#lifecycle-controls) - - [Rollout states](#rollout-states) - - [Failure categories](#failure-categories) +- [Replace a deployment (gradual cutover)](#replace-a-deployment-gradual-cutover) - [A/B tests](#ab-tests) - [Shadow experiments](#shadow-experiments) - [Choosing a strategy](#choosing-a-strategy) @@ -28,12 +22,9 @@ each request through sampling stages: 1. **Traffic split** — sample a candidate in proportion to capacity (weight × ready replicas). 2. **A/B test** — if the candidate is an A/B control, re-sample among control + variants by the test percentages. -3. **Rollout** — if the candidate is an active rollout's source, re-sample between source and - target by the current rollout percentage. -4. **Route** — send to the final deployment. +3. **Route** — send to the final deployment. -The A/B stage runs before the rollout stage, so a deployment can be both an A/B control and a -rollout source. Shadow experiments sit outside this path entirely (copies, not routing). +Shadow experiments sit outside this path entirely (copies, not routing). ## Stickiness @@ -46,12 +37,12 @@ caches warm across a conversation. Key precedence: 3. Otherwise a key derived from request content. Set `prompt_cache_key` on requests sharing a prompt prefix (e.g. every turn of one -conversation) to control stickiness. Editing weights, adding/removing deployments, or a -rollout shifting its percentage reassigns some keys. +conversation) to control stickiness. Editing weights or adding/removing deployments +reassigns some keys. ## Observing routing -To verify empirically how traffic actually lands — for a split, rollout, or A/B test — you +To verify empirically how traffic actually lands — for a split or an A/B test — you need two things the obvious approach misses: - **Attribution comes from the response headers, not the body.** The response body's `model` @@ -140,178 +131,33 @@ Semantics that trip people up: reapplies on scale-up. - An endpoint with no routable deployment returns `routing_error`. -## Rollouts +## Replace a deployment (gradual cutover) -A rollout migrates traffic from a **source** deployment to a **target** deployment under the -same endpoint, scaling the target up and draining the source. Use it instead of hand-editing -the split when replacing a deployment on live traffic (new model version, new config/hardware). +To migrate an endpoint to a new model version, config, or hardware on live traffic, run both +deployments side by side and shift traffic with weights and replica counts: -Requirements: +1. Create the new deployment on the **same endpoint**, sized for its eventual share, and wait + for `READY`. It receives no traffic yet (weight unset). +2. Give it a small share: `tg beta endpoints update dep_new --traffic-weight 10` (with the old + deployment holding e.g. weight 90 — remember share is weight x ready replicas, so match + replica counts or account for them). +3. Watch the dashboard (and the [Observing routing](#observing-routing) probe) between steps; + raise the new deployment's weight stepwise (25 -> 50 -> 100-equivalent) as confidence grows. +4. Cut over: `tg beta endpoints update dep_old --traffic-weight 0` takes the old deployment + out of rotation without scaling it down — instant rollback is just restoring its weight. +5. Once stable, scale the old deployment to `0/0`, wait for `STOPPED`, and `rm` it. -- Source and target under the same endpoint. The source can be a live weighted deployment or - an A/B / shadow experiment member (the CLI detaches it). -- The source must be referenced in the traffic split (a rollout only shifts traffic already - routed to the source; otherwise it runs but moves nothing). -- **The target must have at least one replica and be `READY` before you start.** Do NOT start - a rollout with the target at `0/0`: traffic can shift before the target has a ready replica, - causing `deployment_stopped` errors. The rollout adjusts the target's replica count after it - starts. (Earlier guidance said to create the target stopped — that's obsolete.) -- One active rollout per endpoint (`409` on create otherwise). +Weight changes propagate in tens of seconds (see the propagation note above) and reassign +some sticky keys. To evaluate the candidate *before* giving it live traffic, use a +[shadow experiment](#shadow-experiments); to compare on a fixed slice of live traffic, use an +[A/B test](#ab-tests). -### Strategies - -| Strategy | Traffic movement | Use when | -| --- | --- | --- | -| **Canary** | Staged ladder (e.g. 25→50→75→100%), source drains stepwise. | Gradual, metric-gated exposure. Safest for production. | -| **Blue-green** | Single 0→100 cutover once the target is ready. Default. | Atomic flip/rollback; you have capacity to run both at full size briefly. | -| **Rolling** | Replica-by-replica swap (target +1 / source −1 per batch). | Preserve total GPU capacity; no ramp or gate needed. | - -Metric gates are **canary-only** — blue-green and rolling have no soak window, and a -`metrics` block is rejected at create time. - -### Create and start - -CLI (`rollout` creates and starts in one step): - -```bash -tg beta endpoints rollout dep_target456 --from dep_source123 --canary --steps 25,50,75,100 --interval 10m -``` - -SDK (two steps — `create` returns `PENDING`, `start` begins shifting): - -```python -rollout = client.beta.endpoints.rollouts.create( - "ep_abc123", - project_id=project_id, - source_deployment_id="dep_source123", - target_deployment_id="dep_target456", - blue_green={}, # or canary={...} / rolling={...} - source_cleanup="SOURCE_CLEANUP_POLICY_DRAIN", # or keep the source scaled up - final_source_replicas=0, - final_target_replicas=4, # must be >= 1 -) -client.beta.endpoints.rollouts.start(rollout.id, project_id=project_id, endpoint_id="ep_abc123") -``` - -### Metric gates (canary only) - -SDK/API only. After each step's soak (`step_interval`), the gate evaluates the rules and -either advances or pauses (`SYSTEM_PAUSED`) for review. Rules are ANDed. - -```python -rollout = client.beta.endpoints.rollouts.create( - "ep_abc123", - project_id=project_id, - source_deployment_id="dep_source123", - target_deployment_id="dep_target456", - canary={ - "steps": [ - {"traffic": 25, "replicas": 1}, - {"traffic": 50, "replicas": 2}, - {"traffic": 100, "replicas": 4}, - ], - "step_interval": "390s", - }, - metrics=[ - { - "name": "serving_latency", - "stat": "METRIC_STAT_TYPE_PERCENTILE", - "percentile": 95, - "regression_check": { - "max_regression_percent": 50, - "direction": "REGRESSION_DIRECTION_HIGHER_IS_WORSE", - }, - "window": "300s", - } - ], - source_cleanup="SOURCE_CLEANUP_POLICY_DRAIN", - final_source_replicas=0, - final_target_replicas=4, -) -``` - -Gate catalog (`name` must be one of these — anything else, including autoscaling metric -names like `gpu_utilization`, is rejected with `400`): - -| `name` | Type | Measures | `stat` | -| --- | --- | --- | --- | -| `serving_latency` | Histogram | Engine request duration — the preferred latency gate. | `AVG` or `PERCENTILE` | -| `router_latency` | Histogram | Router per-attempt latency. Bimodal — gate on `PERCENTILE` ≥ 95, not the mean. | `AVG` or `PERCENTILE` | -| `router_error_rate` | Ratio | Router 5xx / all responses. | `AVG` | -| `inflight_requests` | Gauge | In-flight requests across the target's pods. | `AVG` | - -Each rule uses exactly one criterion: - -- **`regression_check`** — target vs source; fails when the target is worse by more than - `max_regression_percent` in the given direction. Relative safety ("no more than 50% slower"). -- **`threshold_check`** — target vs an absolute `value` with `THRESHOLD_OPERATOR_GT|GTE|LT|LTE`. - Latency values are **milliseconds** (`30000` = 30 s). Fixed SLOs ("p95 under 30 s"). - -Practical rules: - -- **Gates need live traffic.** With no requests on the target, the gate reports - `METRICS_UNAVAILABLE` and the rollout pauses instead of completing. Keep the endpoint under - load for the whole rollout. -- Use `window: "300s"` for histogram percentiles — shorter windows sit inside ingestion lag - and hold too few samples, causing spurious pauses. -- Keep `step_interval >= window + ~90s` so metrics settle and new replicas warm past their - cold start before the gate samples. - -### Lifecycle controls - -| CLI | SDK | Effect | -| --- | --- | --- | -| `rollout --pause` | `rollouts.pause` | Hold at the current step (honored at the next step boundary). | -| `rollout --continue` | `rollouts.resume` | Resume from the current step (re-evaluates a paused step). | -| `rollout --complete` | `rollouts.promote` | Fast-forward: all traffic to the target. | -| `rollout --abort` | `rollouts.abort` | Roll traffic back to the source. Terminal; only while active. | - -Pass `--reason` / `reason=` on pause and abort to record why. Monitor with -`rollouts.retrieve` (`state`, `current_step`, `current_traffic_percent`, per-step metric -verdicts) or the dashboard's Rollouts tab. - -**After `COMPLETED` there is no rollback** — the source is drained. Revert by running a new -rollout in the reverse direction. Delete pending/terminal rollouts via `rollouts.delete` -(SDK/API; `rm` doesn't take `rol_` IDs). - -### Rollout states - -| State | Meaning | -| --- | --- | -| `PENDING` | Created, not started. | -| `RUNNING` | Actively shifting traffic. | -| `STABILIZING` | Soaking at a step's traffic level. | -| `PAUSED` | Operator-paused; split holds. | -| `SYSTEM_PAUSED` | Platform-paused for review — read `status.condition.category` before acting. | -| `ABORTING` / `ABORTED` | Reverting / reverted to the source (terminal). | -| `COMPLETED` | All traffic on the target (terminal). | - -### Failure categories - -On `SYSTEM_PAUSED` or terminal failure, `status.condition.category` is typed — trust it over -the free-text message: - -| Category | Meaning | Action | -| --- | --- | --- | -| `METRIC_REGRESSION` | Gate found the target worse than the source. | Abort. | -| `HEALTH_REGRESSION` | Health check failed during soak. | Abort. | -| `METRICS_UNAVAILABLE` | Too few samples (usually no live traffic). | Restore load, resume. | -| `TARGET_NOT_READY` | Target didn't become ready in time. | Investigate target, resume or abort. | -| `SOURCE_NOT_DRAINED` | Source replicas didn't drain. | Wait/investigate, resume. | -| `CAPACITY_EXHAUSTED` | Not enough fleet capacity (checked in-rollout, not at create). | Wait; often auto-resumes. | -| `ROUTING_ERROR` | Invalid routing configuration mid-shift. | Fix routing, resume or abort. | -| `DEPENDENCY_OUTAGE` | Platform dependency unreachable. | Wait for recovery, resume. | -| `POLICY_INFEASIBLE` | An autoscaling ceiling dropped below the step's replica floor. | Raise the ceiling or lower the step target, resume. | -| `UNDER_SERVED` | Ready capacity fell below what the split requires. | Restore capacity or reduce the split, resume. | -| `ENTITLEMENT_LAPSED` | Billing/org entitlement lapsed mid-run. | Restore it, resume or abort. | -| `ABORTED_BY_OPERATOR` | Someone aborted. | Terminal; run a new rollout. | -| `INTERNAL` | Unexpected platform error. | Contact support with the rollout ID. | ## A/B tests An A/B experiment holds a **fixed** split between a control deployment and 1–19 variants, subdividing only the control's share of traffic. Use it to measure a candidate before -promoting; use a rollout to actually migrate. +committing; migrate with a [gradual cutover](#replace-a-deployment-gradual-cutover). Rules: @@ -366,10 +212,11 @@ client.beta.endpoints.ab_experiments.update( ) ``` -Promote a winner with a rollout -(`rollout dep_variant456 --from dep_control123 --blue-green`), then delete the test -(`rm abx_abc123`) — deletion returns all traffic to the regular split immediately (verified: -after `rm`, a varied probe lands 100% on the control's cluster). +To ship a winning variant, delete the test (`rm abx_abc123`) and move traffic with a +[gradual cutover](#replace-a-deployment-gradual-cutover): give the variant a traffic weight +and take the old control to `--traffic-weight 0`. Deleting the test returns all traffic to +the regular split immediately (verified: after `rm`, a varied probe lands 100% on the +control's cluster) — so set the variant's weight promptly after deleting. ## Shadow experiments @@ -436,7 +283,7 @@ lifetimes. | Goal | Tool | | --- | --- | | Serve two deployments for redundancy | Traffic split with weights. | -| Replace a deployment on live traffic | Rollout (canary if you want metric gates). | +| Replace a deployment on live traffic | Gradual cutover with `--traffic-weight` steps. | | Measure a candidate on a fixed slice of user-visible traffic | A/B test. | | Test a candidate with zero user impact / warm it up | Shadow experiment. | -| Ship the A/B or shadow winner | Rollout from the current deployment to the winner. | +| Ship the A/B or shadow winner | Delete the experiment, then shift weights to the winner. | diff --git a/skills/together-dedicated-model-inference/scripts/rollout_new_version.py b/skills/together-dedicated-model-inference/scripts/rollout_new_version.py deleted file mode 100644 index 1dd3508..0000000 --- a/skills/together-dedicated-model-inference/scripts/rollout_new_version.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -""" -Together AI Dedicated Model Inference -- Roll Out a New Deployment - -Replace a live deployment with a new one (new model version, new config, or -new hardware) without downtime: create the target and wait for READY, start a rollout -(blue-green or metric-gated canary), watch it, and control its lifecycle. - -Usage: - python rollout_new_version.py prepare --endpoint ep_abc123 --model ml_abc123 --config cr_abc123 --name v2 - python rollout_new_version.py start --endpoint ep_abc123 --source dep_src --target dep_tgt --replicas 4 - python rollout_new_version.py start --endpoint ep_abc123 --source dep_src --target dep_tgt \ - --replicas 4 --canary --steps 25,50,100 --gate-latency-regression 50 - python rollout_new_version.py watch --endpoint ep_abc123 --rollout rol_abc123 - python rollout_new_version.py pause|resume|complete|abort --endpoint ep_abc123 --rollout rol_abc123 - -Requires: - uv pip install --upgrade together # a release with the beta DMI surface (client.beta.*) - export TOGETHER_API_KEY=your_key - -Notes: - - The target must have at least one replica and be READY before the rollout - starts (a 0/0 target risks deployment_stopped errors mid-shift). `prepare` - creates it with one replica and waits; keep it OUT of the traffic split. - - The source must be in the endpoint's traffic split, or the rollout shifts nothing. - - Metric gates are canary-only and need live traffic on the endpoint to evaluate. - - After COMPLETED there's no rollback; revert by rolling out in reverse. -""" - -import argparse -import sys -import time - -from together import Together - -client = Together() -PROJECT_ID = client.whoami().project_id - -TERMINAL_STATES = {"ROLLOUT_STATE_COMPLETED", "ROLLOUT_STATE_ABORTED"} - - -def prepare_target(endpoint_id: str, model_id: str, config_id: str, config_project_id: str, name: str): - """Create the target deployment with one replica and wait for READY. - - The rollout adjusts the target's replica count after it starts, but it must - already be READY when the rollout begins — traffic can shift before a 0/0 - target has a ready replica, causing deployment_stopped errors. - """ - deployment = client.beta.endpoints.deployments.create( - endpoint_id, - project_id=PROJECT_ID, - name=name, - model=f"projects/{PROJECT_ID}/models/{model_id}", - config=f"projects/{config_project_id}/configs/{config_id}", - autoscaling={"min_replicas": 1, "max_replicas": 1}, - ) - print(f"Created target deployment: {deployment.id} — waiting for READY...") - while True: - d = client.beta.endpoints.deployments.retrieve( - deployment.id, project_id=PROJECT_ID, endpoint_id=endpoint_id - ) - print(f" {d.status.state}") - if d.status.state == "DEPLOYMENT_STATE_READY": - break - if d.status.state == "DEPLOYMENT_STATE_FAILED": - raise RuntimeError(f"Target deployment failed: {d.status.message}") - time.sleep(15) - print("Target READY. Do NOT add it to the traffic split; the rollout moves traffic itself.") - return deployment - - -def start_rollout( - endpoint_id: str, - source_id: str, - target_id: str, - final_replicas: int, - canary: bool = False, - steps: list[int] | None = None, - step_interval: str = "600s", - gate_latency_regression: int | None = None, -): - """Create and start a rollout. Blue-green by default; --canary for a staged ladder. - - With a canary, an optional serving_latency regression gate pauses the rollout - (SYSTEM_PAUSED) if the target's p95 regresses more than the given percent vs the source. - """ - kwargs: dict = { - "project_id": PROJECT_ID, - "source_deployment_id": source_id, - "target_deployment_id": target_id, - "source_cleanup": "SOURCE_CLEANUP_POLICY_DRAIN", - "final_source_replicas": 0, - "final_target_replicas": final_replicas, - } - if canary: - ladder = steps or [10, 50, 100] - kwargs["canary"] = { - "steps": [ - # Scale target replicas proportionally to each traffic step. - {"traffic": pct, "replicas": max(1, round(final_replicas * pct / 100))} - for pct in ladder - ], - "step_interval": step_interval, - } - if gate_latency_regression is not None: - kwargs["metrics"] = [ - { - "name": "serving_latency", - "stat": "METRIC_STAT_TYPE_PERCENTILE", - "percentile": 95, - "regression_check": { - "max_regression_percent": gate_latency_regression, - "direction": "REGRESSION_DIRECTION_HIGHER_IS_WORSE", - }, - # 300s window for histogram percentiles; keep step_interval >= window + ~90s. - "window": "300s", - } - ] - else: - kwargs["blue_green"] = {} - - rollout = client.beta.endpoints.rollouts.create(endpoint_id, **kwargs) - client.beta.endpoints.rollouts.start(rollout.id, project_id=PROJECT_ID, endpoint_id=endpoint_id) - print(f"Rollout started: {rollout.id} ({'canary' if canary else 'blue-green'})") - return rollout - - -def watch(endpoint_id: str, rollout_id: str, poll: int = 30): - """Poll the rollout until it reaches a terminal state, printing progress.""" - while True: - r = client.beta.endpoints.rollouts.retrieve( - rollout_id, project_id=PROJECT_ID, endpoint_id=endpoint_id - ) - line = f" {r.state} step={getattr(r, 'current_step', None)} traffic={getattr(r, 'current_traffic_percent', None)}%" - condition = getattr(getattr(r, "status", None), "condition", None) - if condition is not None and getattr(condition, "category", None): - line += f" condition={condition.category}: {getattr(condition, 'message', '')}" - print(line) - if r.state in TERMINAL_STATES: - return r - if r.state == "ROLLOUT_STATE_SYSTEM_PAUSED": - print( - " Platform paused the rollout. Transient causes (METRICS_UNAVAILABLE," - " CAPACITY_EXHAUSTED): fix and `resume`. Regressions (METRIC_REGRESSION," - " HEALTH_REGRESSION): `abort` to revert to the source." - ) - time.sleep(poll) - - -def control(endpoint_id: str, rollout_id: str, action: str, reason: str | None = None): - """pause / resume / complete (fast-forward to target) / abort (revert to source).""" - rollouts = client.beta.endpoints.rollouts - common = {"project_id": PROJECT_ID, "endpoint_id": endpoint_id} - if action == "pause": - rollouts.pause(rollout_id, **common, reason=reason) - elif action == "resume": - rollouts.resume(rollout_id, **common) - elif action == "complete": - rollouts.promote(rollout_id, **common) - elif action == "abort": - rollouts.abort(rollout_id, **common, reason=reason) - print(f"{action}: {rollout_id}") - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - sub = parser.add_subparsers(dest="command", required=True) - - p = sub.add_parser("prepare", help="Create the target deployment and wait for READY") - p.add_argument("--endpoint", required=True) - p.add_argument("--model", required=True, help="Model ID (ml_...)") - p.add_argument("--config", required=True, help="Config revision ID (cr_...)") - p.add_argument("--config-project", default=PROJECT_ID) - p.add_argument("--name", required=True, help="Target deployment name") - - p = sub.add_parser("start", help="Create and start the rollout") - p.add_argument("--endpoint", required=True) - p.add_argument("--source", required=True, help="Source deployment ID (currently serving)") - p.add_argument("--target", required=True, help="Target deployment ID (READY, not in the split)") - p.add_argument("--replicas", type=int, required=True, help="Final target replica count (>= 1)") - p.add_argument("--canary", action="store_true", help="Staged ladder instead of blue-green") - p.add_argument("--steps", default=None, help="Canary traffic ladder, e.g. 25,50,100") - p.add_argument("--interval", default="600s", help="Canary soak per step (default 600s)") - p.add_argument( - "--gate-latency-regression", type=int, default=None, - help="Canary gate: max p95 serving_latency regression percent vs source", - ) - - p = sub.add_parser("watch", help="Poll the rollout until terminal") - p.add_argument("--endpoint", required=True) - p.add_argument("--rollout", required=True) - - for action in ("pause", "resume", "complete", "abort"): - p = sub.add_parser(action, help=f"{action} a running rollout") - p.add_argument("--endpoint", required=True) - p.add_argument("--rollout", required=True) - p.add_argument("--reason", default=None) - - args = parser.parse_args() - if args.command == "prepare": - prepare_target(args.endpoint, args.model, args.config, args.config_project, args.name) - elif args.command == "start": - steps = [int(s) for s in args.steps.split(",")] if args.steps else None - start_rollout( - args.endpoint, args.source, args.target, args.replicas, - canary=args.canary, steps=steps, step_interval=args.interval, - gate_latency_regression=args.gate_latency_regression, - ) - elif args.command == "watch": - watch(args.endpoint, args.rollout) - else: - control(args.endpoint, args.rollout, args.command, getattr(args, "reason", None)) - return 0 - - -if __name__ == "__main__": - sys.exit(main())