From cdbcf14b7642e37a22bbaf0d91b3b32ac83be352 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:16:32 -0700 Subject: [PATCH 1/9] Marking TypeScript SDK as pre-release --- AGENTS.md | 4 +++- CONFIG.md | 10 ++++++---- CONTRIBUTING.md | 4 +--- README.md | 13 ++++++------- SECURITY.md | 1 - cli/README.md | 3 +-- typescript/AGENTS.md | 2 +- typescript/README.md | 2 +- 8 files changed, 19 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f18d712..56ab2cc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,7 @@ Common subdirectories include: - `cli`: Rust `qt` CLI, local server, and local run/metrics storage. - `python`: The `quantiles` Python SDK. -- `typescript`: The `@quantiles` TypeScript SDK. +- `typescript`: The `@quantiles` TypeScript SDK. (not currently released) Note there is a separate repository, [github.com/quantiles-evals/skill](https://github.com/quantiles-evals/skill), that contains the agent skill. @@ -85,6 +85,8 @@ When editing the Python SDK subdirectory, preserve `async` behavior, stable JSON ### `typescript/` directory +>Note: the Typescript SDK has not been released. The code should be considered in-develpoment. + The `typescript/` directory contains the Quantiles TypeScript SDK, which allows users to author custom local AI evals using the local Quantiles server and Typescript. It exposes workflow primitives such as `QuantilesClient`, `QuantilesRun`, stable JSON utilities, and shared SDK types. When editing the TypeScript SDK subdirectory, preserve strict typing, JSON-serializable public surfaces, ESM behavior, and documented package exports. diff --git a/CONFIG.md b/CONFIG.md index b90b72c..34d3bf4 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -34,7 +34,7 @@ model = "openai:gpt-5.4-nano" # see below for the full reference ``` -Alternatively, if you want to build your own custom evaluation, using the Quantiles Python or TypeScript SDK, define that custom eval as follows: +Alternatively, if you want to build your own custom evaluation using the Quantiles Python define that custom eval as follows: ```toml # This is configuration for a custom code benchmark called my-eval. @@ -45,8 +45,10 @@ Alternatively, if you want to build your own custom evaluation, using the Quanti # This `type` parameter defaults to "builtin". for custom evals, be sure to override # the default with "custom_code" here. type = "custom_code" -# This is the command that the CLI will execute to run your custom eval code. -command = ["python", "my_eval.py"] +# This is the command that the CLI will execute to run your custom eval code. We recommend +# using the `uv` tool to configure and manage your Python evals, but this field can be any +# command, so you're free to use any tools you want. +command = ["uv", "run", "my_eval.py"] # see below for the full reference ``` @@ -109,7 +111,7 @@ Note that custom code evaluations can customize the model in code. See the [PubM #### The `input` table -For `custom_code` benchmarks, `input` is an arbitrary TOML table that becomes a JSON object in the custom eval (e.g. a `dict` in Python and a `Map` in TypeScript): +For `custom_code` benchmarks, `input` is an arbitrary TOML table that becomes a Python `dict` in your custom eval: ```toml [benchmarks.my-eval] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 46c1f9c..28b04df 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,9 +31,7 @@ git clone https://github.com/quantiles-evals/quantiles.git cd quantiles ``` - +For the [`./cli`](./cli) directory, ensure you have Rust installed and use the `mise.toml` file in that directory to run common build/lint/check/test commands. For the [`./python`](./python) directory, use the [`uv`](https://docs.astral.sh/uv/) tool for similar commands. The [`typescript/`](./typescript) directory holds the yet-unreleased and in-development Typescript SDK. ## Security diff --git a/README.md b/README.md index 9b469cc..f257914 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ It includes a CLI (called `qt`), SDKs, built-in benchmarks, local run history, a Evaluation workflows quickly outgrow one-off scripts once teams need caching, retries, dataset handling, metrics capture, and run comparison. Quantiles gives teams those primitives so they don't have to built them from scratch: -- Write standard Python or TypeScript, with familiar developer patterns +- Write standard Python, with familiar developer patterns - Run workflows locally from the CLI - Automatically record runs, steps, metrics, events, inputs, and final outputs - Store execution history locally in open data formats @@ -107,7 +107,7 @@ Quantiles is built as a local-first, offline system that keeps benchmark executi The Quantiles toolchain, including the `qt` CLI, SDKs, on-disk data formats, and REST API, is optimized to use your local computing power by default instead of relying on cloud or other non-local resources. -The CLI and SDKs (TypeScript or Python) support offline benchmark workflows, including the following local execution and analysis features: +Both the CLI and Python SDK support offline benchmark workflows, including the following local execution and analysis features: - Benchmark code runs locally on your machine - Measurements are computed locally, except for remote model calls, hosted judges, external tools, and LLM-as-judge evaluations that may call remote providers (e.g. OpenAI, Anthropic, cloud providers, etc.) @@ -128,7 +128,7 @@ Quantiles also provides a [benchmark hub](https://quantiles.io/benchmark-hub) fo ## Custom Evaluations -A custom evaluation is a [Python](https://quantiles.io/documentation/reference/python-sdk) or [TypeScript](https://quantiles.io/documentation/reference/typescript-sdk) program that is run by the `qt` CLI and uses the [Quantiles API](https://quantiles.io/documentation/reference/rest-api) to execute an eval. Your code owns the evaluation logic like loading data, calling a model or agent, scoring outputs, computing metrics, and returning a summary. Quantiles manages [durable steps, step caching, and step resume](https://quantiles.io/documentation/workflows-and-steps), metrics, inputs, outputs, and comparisons. +A custom evaluation is a [Python](https://quantiles.io/documentation/reference/python-sdk) program that is run by the `qt` CLI and uses the [Quantiles API](https://quantiles.io/documentation/reference/rest-api) to execute an eval. Your code owns the evaluation logic like loading data, calling a model or agent, scoring outputs, computing metrics, and returning a summary. Quantiles manages [durable steps, step caching, and step resume](https://quantiles.io/documentation/workflows-and-steps), metrics, inputs, outputs, and comparisons. Custom evaluations are configured in `quantiles.toml` with `type = "custom_code"`: @@ -147,11 +147,11 @@ Use custom evaluations when you need to measure behavior that is specific to you Read more about how to build and run custom evaluations at [quantiles.io/documentation/custom-evaluations](https://quantiles.io/documentation/custom-evaluations). -### SDKs +### Python SDK -Use the official Quantiles SDKs to build your custom evaluations with higher-level workflow primitives like durable steps, structured inputs/outputs, and metrics emission, using patterns and practices native to Python and TypeScript. The SDKs integrate tightly with the `qt` CLI’s local API for running, recording and analyzing benchmarks. +Use the official Quantiles Python SDK to build your custom evaluations with primitives like durable steps, structured inputs/outputs, and metrics emission, using patterns and practices native to Python. The SDK integrates tightly with the `qt` CLI’s local API for running, recording and analyzing benchmarks. -The Python SDK is located in this repository at [`python/`](./python). Read more about it at [quantiles.io/documentation/reference/python-sdk](https://quantiles.io/documentation/reference/python-sdk). The TypeScript SDK is located in this repository at [`typescript/`](./typescript). Read more about it at [quantiles.io/documentation/reference/typescript-sdk](https://quantiles.io/documentation/reference/typescript-sdk). +The code for the Python SDK is located in this repository at [`./python/`](./python). Read more about it at [quantiles.io/documentation/reference/python-sdk](https://quantiles.io/documentation/reference/python-sdk) ## Coding Agents @@ -184,7 +184,6 @@ Start here: - [Quickstart](https://quantiles.io/documentation/quickstart) - [Agent Overview](https://quantiles.io/documentation/evals-with-agents) - [Python SDK](https://quantiles.io/documentation/reference/python-sdk) -- [TypeScript SDK](https://quantiles.io/documentation/reference/typescript-sdk) ## Contributing diff --git a/SECURITY.md b/SECURITY.md index 0ef0a77..20d01c4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,7 +8,6 @@ Quantiles provides security fixes for the latest released versions of actively m | --------------------------------------------- | --------------------------------------------- | | Latest `qt` CLI release | Yes | | Latest Python SDK release | Yes | -| Latest TypeScript SDK release | Yes | | Older releases | No, unless explicitly stated in release notes | ## Reporting a Vulnerability diff --git a/cli/README.md b/cli/README.md index faba89e..eb556f5 100644 --- a/cli/README.md +++ b/cli/README.md @@ -105,8 +105,7 @@ The Quantiles CLI, `qt`, keeps execution simple: your code runs locally, while ` ``` +--------------------------------------+ -| Your Script | -| (TypeScript / Python / Shell) | +| Benchmark / Custom Eval (Python) | +-------------------+------------------+ │ │ HTTP / JSON diff --git a/typescript/AGENTS.md b/typescript/AGENTS.md index fa20370..d684c2a 100644 --- a/typescript/AGENTS.md +++ b/typescript/AGENTS.md @@ -2,7 +2,7 @@ ## Scope -These instructions apply to the Quantiles TypeScript SDK. For changes outside the SDK, follow the nearest applicable `AGENTS.md` file. +These instructions apply to the Quantiles TypeScript SDK. For changes outside the SDK, follow the nearest applicable `AGENTS.md` file. Note that this SDK is currently unreleased and should be treated as in development. No guarantees are made about the stability of this API or anything else. ## Project Overview diff --git a/typescript/README.md b/typescript/README.md index fef57ac..f505f17 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -1,6 +1,6 @@ # `@quantiles/sdk` -TypeScript client for the local Quantiles eval runner. +TypeScript client for the local Quantiles eval runner. Note that this SDK is currently unreleased and should be treated as in development. No guarantees are made about the stability of this API or anything else. ## Installation From 75514ac8d7a59d8b8f203022cee3609b4636dcc6 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:18:32 -0700 Subject: [PATCH 2/9] small tweak --- CONFIG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONFIG.md b/CONFIG.md index 34d3bf4..ecfe64f 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -99,7 +99,7 @@ Note that models require specific configuration based on the provider. For detai ### `custom_code` -Custom evaluations are external programs built with one of the Quantiles SDKs. Their config sections contain the following fields: +Custom evaluations are external programs built with the Quantiles Python SDK. Their config sections contain the following fields: | Field | Type | Required | Description | |--|--|--|--| From 596f97fe6f6ff4502a85f96038c2f213dc1002d1 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:23:45 -0700 Subject: [PATCH 3/9] progress --- AGENTS.md | 6 +++--- CONTRIBUTING.md | 2 +- typescript/AGENTS.md | 2 +- typescript/README.md | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 56ab2cc..64a5d12 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,7 @@ Common subdirectories include: - `cli`: Rust `qt` CLI, local server, and local run/metrics storage. - `python`: The `quantiles` Python SDK. -- `typescript`: The `@quantiles` TypeScript SDK. (not currently released) +- `typescript`: The `@quantiles` TypeScript SDK. (in development and unsupported) Note there is a separate repository, [github.com/quantiles-evals/skill](https://github.com/quantiles-evals/skill), that contains the agent skill. @@ -85,9 +85,9 @@ When editing the Python SDK subdirectory, preserve `async` behavior, stable JSON ### `typescript/` directory ->Note: the Typescript SDK has not been released. The code should be considered in-develpoment. +>**Not released**. The TypeScript SDK is currently in development and is unsupported and unreleased. -The `typescript/` directory contains the Quantiles TypeScript SDK, which allows users to author custom local AI evals using the local Quantiles server and Typescript. It exposes workflow primitives such as `QuantilesClient`, `QuantilesRun`, stable JSON utilities, and shared SDK types. +The `typescript/` directory contains the Quantiles TypeScript SDK, which allows users to author custom local AI evals with the Quantiles stack using TypeScript. It exposes workflow primitives such as `QuantilesClient`, `QuantilesRun`, stable JSON utilities, and shared SDK types. When editing the TypeScript SDK subdirectory, preserve strict typing, JSON-serializable public surfaces, ESM behavior, and documented package exports. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 28b04df..f591ec5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,7 @@ git clone https://github.com/quantiles-evals/quantiles.git cd quantiles ``` -For the [`./cli`](./cli) directory, ensure you have Rust installed and use the `mise.toml` file in that directory to run common build/lint/check/test commands. For the [`./python`](./python) directory, use the [`uv`](https://docs.astral.sh/uv/) tool for similar commands. The [`typescript/`](./typescript) directory holds the yet-unreleased and in-development Typescript SDK. +For the [`./cli`](./cli) directory, ensure you have Rust installed and use the `mise.toml` file in that directory to run common build/lint/check/test commands. For the [`./python`](./python) directory, use the [`uv`](https://docs.astral.sh/uv/) tool for similar commands. The [`typescript/`](./typescript) directory holds the yet-unreleased, unsupported and in-development TypeScript SDK. ## Security diff --git a/typescript/AGENTS.md b/typescript/AGENTS.md index d684c2a..eb5b820 100644 --- a/typescript/AGENTS.md +++ b/typescript/AGENTS.md @@ -2,7 +2,7 @@ ## Scope -These instructions apply to the Quantiles TypeScript SDK. For changes outside the SDK, follow the nearest applicable `AGENTS.md` file. Note that this SDK is currently unreleased and should be treated as in development. No guarantees are made about the stability of this API or anything else. +These instructions apply to the Quantiles TypeScript SDK. For changes outside the SDK, follow the nearest applicable `AGENTS.md` file. Note that this SDK is currently unsupported, unreleased and should be treated as in development. ## Project Overview diff --git a/typescript/README.md b/typescript/README.md index f505f17..a4ecd16 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -1,6 +1,6 @@ # `@quantiles/sdk` -TypeScript client for the local Quantiles eval runner. Note that this SDK is currently unreleased and should be treated as in development. No guarantees are made about the stability of this API or anything else. +TypeScript client for the local Quantiles eval runner. Note that this SDK is currently in development and is unsupported and unreleased. ## Installation From 85a45c0ff111b69d20b35703d9471502792f3733 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:25:12 -0700 Subject: [PATCH 4/9] progress --- AGENTS.md | 2 +- typescript/AGENTS.md | 2 +- typescript/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 64a5d12..91ac0a9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,7 +85,7 @@ When editing the Python SDK subdirectory, preserve `async` behavior, stable JSON ### `typescript/` directory ->**Not released**. The TypeScript SDK is currently in development and is unsupported and unreleased. +>**Not released**. The TypeScript SDK is currently unsupported and unreleased. The `typescript/` directory contains the Quantiles TypeScript SDK, which allows users to author custom local AI evals with the Quantiles stack using TypeScript. It exposes workflow primitives such as `QuantilesClient`, `QuantilesRun`, stable JSON utilities, and shared SDK types. diff --git a/typescript/AGENTS.md b/typescript/AGENTS.md index eb5b820..7242051 100644 --- a/typescript/AGENTS.md +++ b/typescript/AGENTS.md @@ -2,7 +2,7 @@ ## Scope -These instructions apply to the Quantiles TypeScript SDK. For changes outside the SDK, follow the nearest applicable `AGENTS.md` file. Note that this SDK is currently unsupported, unreleased and should be treated as in development. +These instructions apply to the Quantiles TypeScript SDK. For changes outside the SDK, follow the nearest applicable `AGENTS.md` file. Note that this SDK is currently unreleased and unsupported. ## Project Overview diff --git a/typescript/README.md b/typescript/README.md index a4ecd16..7cbd7ed 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -1,6 +1,6 @@ # `@quantiles/sdk` -TypeScript client for the local Quantiles eval runner. Note that this SDK is currently in development and is unsupported and unreleased. +TypeScript client for the local Quantiles eval runner. Note that this SDK is currently unsupported and unreleased. ## Installation From 5e4b55f82988b3e49ac08c5e059cb70b89899d75 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:26:01 -0700 Subject: [PATCH 5/9] progress --- CONFIG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONFIG.md b/CONFIG.md index ecfe64f..ec62c4e 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -34,7 +34,7 @@ model = "openai:gpt-5.4-nano" # see below for the full reference ``` -Alternatively, if you want to build your own custom evaluation using the Quantiles Python define that custom eval as follows: +Alternatively, if you want to build your own custom evaluation using the Quantiles Python SDK, define that custom eval as follows: ```toml # This is configuration for a custom code benchmark called my-eval. From 0d3c1593723613d7bccd4e534ead50ece6dce006 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:26:54 -0700 Subject: [PATCH 6/9] progress --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f591ec5..84e8d22 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,7 @@ git clone https://github.com/quantiles-evals/quantiles.git cd quantiles ``` -For the [`./cli`](./cli) directory, ensure you have Rust installed and use the `mise.toml` file in that directory to run common build/lint/check/test commands. For the [`./python`](./python) directory, use the [`uv`](https://docs.astral.sh/uv/) tool for similar commands. The [`typescript/`](./typescript) directory holds the yet-unreleased, unsupported and in-development TypeScript SDK. +For the [`./cli`](./cli) directory, ensure you have Rust installed and use the `mise.toml` file in that directory to run common build/lint/check/test commands. For the [`./python`](./python) directory, use the [`uv`](https://docs.astral.sh/uv/) tool for similar commands. The [`typescript/`](./typescript) directory holds the yet-unreleased, unsupported TypeScript SDK. ## Security From e139ecd35aef32a7a0d75a13458df81a46324eb1 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:29:49 -0700 Subject: [PATCH 7/9] progress --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f257914..0ae8886 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ Quantiles also provides a [benchmark hub](https://quantiles.io/benchmark-hub) fo ## Custom Evaluations -A custom evaluation is a [Python](https://quantiles.io/documentation/reference/python-sdk) program that is run by the `qt` CLI and uses the [Quantiles API](https://quantiles.io/documentation/reference/rest-api) to execute an eval. Your code owns the evaluation logic like loading data, calling a model or agent, scoring outputs, computing metrics, and returning a summary. Quantiles manages [durable steps, step caching, and step resume](https://quantiles.io/documentation/workflows-and-steps), metrics, inputs, outputs, and comparisons. +A custom evaluation is a [Python](https://quantiles.io/documentation/reference/python-sdk) program that is run by the `qt` CLI and uses its [local storage](http://quantiles.io/documentation/local-first-offline) and [durable workflow engine](https://quantiles.io/documentation/workflows-and-steps) to run efficiently and reliably. Your code owns the evaluation logic like loading data, calling a model or agent, scoring outputs, computing metrics, and returning a summary. Quantiles manages [durable steps, step caching, and step resume](https://quantiles.io/documentation/workflows-and-steps), metrics, inputs, outputs, and comparisons. Custom evaluations are configured in `quantiles.toml` with `type = "custom_code"`: From a6f927c88cd01d7a22074d5376d98fc932b84e2d Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Mon, 29 Jun 2026 14:29:13 -0700 Subject: [PATCH 8/9] typecheck and fmt --- python/examples/prompt_eval.py | 169 +++++++++++++++++---------------- 1 file changed, 87 insertions(+), 82 deletions(-) diff --git a/python/examples/prompt_eval.py b/python/examples/prompt_eval.py index 2474c64..df7c396 100644 --- a/python/examples/prompt_eval.py +++ b/python/examples/prompt_eval.py @@ -1,11 +1,12 @@ from typing import Literal, TypedDict, cast from quantiles import ( - Statistics, - emit, - entrypoint, - step, - workflow, + JsonValue, + Statistics, + emit, + entrypoint, + step, + workflow, ) from quantiles.workflow_context import WorkflowContext @@ -13,101 +14,105 @@ class EvalInput(TypedDict): - prompt_version: str + prompt_version: str class EvalCase(TypedDict): - id: str - ticket: str - expected: Label + id: str + ticket: str + expected: Label class CaseResult(TypedDict): - id: str - expected: Label - prediction: Label - correct: bool - tokens_used: int + id: str + expected: Label + prediction: Label + correct: bool + tokens_used: int CASES: list[EvalCase] = [ - { - "id": "double-charge", - "ticket": "I was charged twice for my subscription renewal.", - "expected": "billing", - }, - { - "id": "csv-crash", - "ticket": "The app crashes every time I upload a CSV file.", - "expected": "bug", - }, - { - "id": "invite-teammate", - "ticket": "Can you show me how to invite another teammate?", - "expected": "how_to", - }, + { + "id": "double-charge", + "ticket": "I was charged twice for my subscription renewal.", + "expected": "billing", + }, + { + "id": "csv-crash", + "ticket": "The app crashes every time I upload a CSV file.", + "expected": "bug", + }, + { + "id": "invite-teammate", + "ticket": "Can you show me how to invite another teammate?", + "expected": "how_to", + }, ] async def evaluate_case( - ctx: WorkflowContext, - item: EvalCase, - prompt_version: str, + ctx: WorkflowContext, + item: EvalCase, + prompt_version: str, ) -> CaseResult: - prediction = cast( - Label, - await step( - ctx, - step_key=f"case:{item['id']}", - input_value={ - "sample_id": item["id"], - "ticket": item["ticket"], - "expected": item["expected"], - "prompt_version": prompt_version, - }, - execute=lambda: call_model(item["ticket"]), - ), - ) - - return { - "id": item["id"], + prediction = cast( + Label, + await step( + ctx, + step_key=f"case:{item['id']}", + input_value={ + "sample_id": item["id"], + "ticket": item["ticket"], "expected": item["expected"], - "prediction": prediction, - "correct": prediction == item["expected"], - "tokens_used": len(item["ticket"].split()), - } - - -async def handler(input_value: EvalInput | None, ctx: WorkflowContext): - prompt_version = (input_value or {}).get("prompt_version", "A") - results: list[CaseResult] = [] - for item in CASES: - results.append(await evaluate_case(ctx, item, prompt_version)) - - correct = sum(1 for result in results if result["correct"]) - accuracy = Statistics.accuracy(correct, len(results)) - tokens_used = sum(result["tokens_used"] for result in results) - - await emit(ctx, "accuracy", accuracy) - await emit(ctx, "correct_count", float(correct)) - await emit(ctx, "total_count", float(len(results))) - await emit(ctx, "tokens_used", float(tokens_used), "tokens") - - return { - "accuracy": accuracy, - "correct_count": correct, - "total_count": len(results), - "results": results, - } + "prompt_version": prompt_version, + }, + execute=lambda: call_model(item["ticket"]), + ), + ) + + return { + "id": item["id"], + "expected": item["expected"], + "prediction": prediction, + "correct": prediction == item["expected"], + "tokens_used": len(item["ticket"].split()), + } + + +async def handler(input_value: EvalInput | None, ctx: WorkflowContext) -> JsonValue: + prompt_version = (input_value or {}).get("prompt_version", "A") + results: list[CaseResult] = [] + for item in CASES: + results.append(await evaluate_case(ctx, item, prompt_version)) + + correct = sum(1 for result in results if result["correct"]) + accuracy = Statistics.accuracy(correct, len(results)) + tokens_used = sum(result["tokens_used"] for result in results) + + await emit(ctx, "accuracy", accuracy) + await emit(ctx, "correct_count", float(correct)) + await emit(ctx, "total_count", float(len(results))) + await emit(ctx, "tokens_used", float(tokens_used), "tokens") + + return cast( + JsonValue, + # the below value is a valid JsonValue + { + "accuracy": accuracy, + "correct_count": correct, + "total_count": len(results), + "results": results, + }, + ) async def call_model(ticket: str) -> Label: - text = ticket.lower() - if "crash" in text: - return "bug" - if "invite" in text or "show me" in text: - return "how_to" - return "billing" + text = ticket.lower() + if "crash" in text: + return "bug" + if "invite" in text or "show me" in text: + return "how_to" + return "billing" support_triage = workflow("support-triage", handler) From b601c905d85d7b5f45e05cb8908ae5113d055ad5 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Mon, 29 Jun 2026 14:42:32 -0700 Subject: [PATCH 9/9] moving advisory note to top of typescript readme files --- typescript/AGENTS.md | 4 +++- typescript/README.md | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/typescript/AGENTS.md b/typescript/AGENTS.md index 7242051..6580205 100644 --- a/typescript/AGENTS.md +++ b/typescript/AGENTS.md @@ -1,8 +1,10 @@ # AGENTS.md +>**Note**: This SDK is currently unreleased and unsupported. + ## Scope -These instructions apply to the Quantiles TypeScript SDK. For changes outside the SDK, follow the nearest applicable `AGENTS.md` file. Note that this SDK is currently unreleased and unsupported. +These instructions apply to the Quantiles TypeScript SDK. For changes outside the SDK, follow the nearest applicable `AGENTS.md` file. ## Project Overview diff --git a/typescript/README.md b/typescript/README.md index 7cbd7ed..6315ecf 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -1,6 +1,8 @@ # `@quantiles/sdk` -TypeScript client for the local Quantiles eval runner. Note that this SDK is currently unsupported and unreleased. +>**Note**: this SDK is currently unsupported and unreleased. + +TypeScript client for the local Quantiles eval runner. ## Installation