From 4978c1a18fddd235e438e8ffd063083617395adb Mon Sep 17 00:00:00 2001 From: William Watson Date: Mon, 3 Aug 2026 19:18:44 -0400 Subject: [PATCH 1/4] fix: enforce OpenAI tool choice contract --- docs/docs/instructions/server/tool_calling.md | 38 ++- src/include/openai_tool_policy.hpp | 300 ++++++++++++++++++ src/server/rest_handler.cpp | 53 +++- src/server/server.cpp | 8 +- src/test/openai_tool_policy/CMakeLists.txt | 10 + src/test/openai_tool_policy/test.cpp | 160 ++++++++++ 6 files changed, 561 insertions(+), 8 deletions(-) create mode 100644 src/include/openai_tool_policy.hpp create mode 100644 src/test/openai_tool_policy/CMakeLists.txt create mode 100644 src/test/openai_tool_policy/test.cpp diff --git a/docs/docs/instructions/server/tool_calling.md b/docs/docs/instructions/server/tool_calling.md index c61ee114..b74bc843 100644 --- a/docs/docs/instructions/server/tool_calling.md +++ b/docs/docs/instructions/server/tool_calling.md @@ -69,6 +69,42 @@ Tool calling has been supported since `v0.9.26`. Please refer to the [model card](https://fastflowlm.com/docs/models/) to see whether tool calling is supported for each model. +## OpenAI tool-selection behavior + +The `/v1/chat/completions` endpoint accepts the OpenAI `tool_choice` values +`none`, `auto`, and `required`. It also accepts a named function choice: + +```json +{ + "tool_choice": { + "type": "function", + "function": { "name": "get_temperature" } + } +} +``` + +`parallel_tool_calls` is a boolean. When it is `false`, a response containing +more than one function call is rejected. A named choice must refer to a function +in the request's `tools` array. `required` and named choices must produce at +least one function call; FastFlowLM returns a model error instead of silently +returning a successful text response when the model violates that contract. +The Chat Completions `allowed_tools` form is also supported for limiting the +request to a named subset of its function tools in `auto` or `required` mode. +Hosted and custom tool types are not supported by FastFlowLM model templates. + +Tool-call responses are checked before they are returned. Each call must name an +available function, and `function.arguments` must be a string containing a JSON +object. Requests with tools are therefore buffered before an OpenAI-compatible +stream is emitted. This keeps streaming response shapes compatible but makes +time to first token approximately equal to the complete tool-call generation +time. + +Function definitions with `strict: true` currently return an explicit request +error. FastFlowLM does not yet have constrained JSON-schema decoding, so +claiming strict schema adherence would be incorrect. Applications must continue +to validate argument values against their function schema before executing a +tool. + ## 🐍 How to use tool calling with FLM via Python script This section walks through an end‑to‑end tool‑calling flow in both streaming and non‑streaming modes. @@ -341,4 +377,4 @@ Here we recommend some useful tools and walk you through their step‑by‑step #### 🔍 Tool: Google PSE Search -#### 🤔 Other tools \ No newline at end of file +#### 🤔 Other tools diff --git a/src/include/openai_tool_policy.hpp b/src/include/openai_tool_policy.hpp new file mode 100644 index 00000000..b67d664f --- /dev/null +++ b/src/include/openai_tool_policy.hpp @@ -0,0 +1,300 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace openai_tools { + +using json = nlohmann::ordered_json; + +enum class ToolChoiceMode { + none, + auto_select, + required, + named, +}; + +class ToolPolicyError : public std::runtime_error { +public: + ToolPolicyError(std::string message, std::string type, std::string param, int status_code) + : std::runtime_error(std::move(message)), + type(std::move(type)), + param(std::move(param)), + status_code(status_code) {} + + std::string type; + std::string param; + int status_code; +}; + +struct ToolPolicy { + ToolChoiceMode mode = ToolChoiceMode::none; + std::string required_function; + bool parallel_tool_calls = true; + bool has_tool_contract = false; + json tools = json::array(); + + bool requires_buffered_validation() const { + return has_tool_contract; + } +}; + +inline ToolPolicyError invalid_request(const std::string& message, const std::string& param) { + return ToolPolicyError(message, "invalid_request_error", param, 400); +} + +inline ToolPolicyError model_contract_error(const std::string& message) { + return ToolPolicyError(message, "model_error", "tool_choice", 500); +} + +inline bool valid_function_name(const std::string& name) { + return !name.empty() && name.size() <= 64 && std::all_of(name.begin(), name.end(), [](unsigned char value) { + return (value >= 'a' && value <= 'z') || (value >= 'A' && value <= 'Z') || + (value >= '0' && value <= '9') || value == '_' || value == '-'; + }); +} + +inline std::string function_name(const json& tool, std::size_t index) { + const std::string param = "tools[" + std::to_string(index) + "]"; + if (!tool.is_object() || !tool.contains("type") || !tool["type"].is_string() || + tool["type"] != "function") { + throw invalid_request(param + " must be a function tool", param); + } + if (!tool.contains("function") || !tool["function"].is_object() || + !tool["function"].contains("name") || !tool["function"]["name"].is_string() || + !valid_function_name(tool["function"]["name"].get())) { + throw invalid_request( + param + ".function.name must contain at most 64 letters, digits, underscores, or hyphens", + param + ".function.name"); + } + if (tool["function"].contains("strict") && !tool["function"]["strict"].is_boolean()) { + throw invalid_request(param + ".function.strict must be a boolean", param + ".function.strict"); + } + if (tool["function"].value("strict", false)) { + throw invalid_request( + param + ".function.strict is not supported because constrained JSON-schema decoding is unavailable", + param + ".function.strict"); + } + return tool["function"]["name"].get(); +} + +inline ToolPolicy parse_tool_policy(const json& request) { + ToolPolicy policy; + if (request.contains("tools") && !request["tools"].is_array()) { + throw invalid_request("tools must be an array", "tools"); + } + policy.tools = request.contains("tools") ? request["tools"] : json::array(); + policy.has_tool_contract = !policy.tools.empty() || request.contains("tool_choice") || + request.contains("parallel_tool_calls"); + + std::unordered_set available_names; + for (std::size_t i = 0; i < policy.tools.size(); ++i) { + const auto name = function_name(policy.tools[i], i); + if (!available_names.insert(name).second) { + throw invalid_request("tool function names must be unique: " + name, "tools"); + } + } + + if (request.contains("parallel_tool_calls")) { + if (!request["parallel_tool_calls"].is_boolean()) { + throw invalid_request("parallel_tool_calls must be a boolean", "parallel_tool_calls"); + } + policy.parallel_tool_calls = request["parallel_tool_calls"].get(); + } + + if (!request.contains("tool_choice")) { + policy.mode = policy.tools.empty() ? ToolChoiceMode::none : ToolChoiceMode::auto_select; + return policy; + } + + const auto& choice = request["tool_choice"]; + if (choice.is_string()) { + const auto value = choice.get(); + if (value == "none") { + policy.mode = ToolChoiceMode::none; + policy.tools = json::array(); + } else if (value == "auto") { + policy.mode = ToolChoiceMode::auto_select; + } else if (value == "required") { + policy.mode = ToolChoiceMode::required; + } else { + throw invalid_request("tool_choice must be 'none', 'auto', 'required', or a named function", "tool_choice"); + } + } else if (choice.is_object() && choice.contains("type") && choice["type"] == "function") { + if (!choice.contains("function") || + !choice["function"].is_object() || !choice["function"].contains("name") || + !choice["function"]["name"].is_string() || + !valid_function_name(choice["function"]["name"].get())) { + throw invalid_request( + "named tool_choice must have the shape {type:'function', function:{name:'...'}}", + "tool_choice"); + } + policy.mode = ToolChoiceMode::named; + policy.required_function = choice["function"]["name"].get(); + if (!available_names.contains(policy.required_function)) { + throw invalid_request( + "tool_choice names a function that is not present in tools: " + policy.required_function, + "tool_choice.function.name"); + } + json selected = json::array(); + for (const auto& tool : policy.tools) { + if (tool["function"]["name"] == policy.required_function) { + selected.push_back(tool); + } + } + policy.tools = std::move(selected); + } else if (choice.is_object() && choice.contains("type") && choice["type"] == "allowed_tools") { + if (!choice.contains("allowed_tools") || !choice["allowed_tools"].is_object()) { + throw invalid_request("allowed_tools tool_choice must contain an allowed_tools object", "tool_choice.allowed_tools"); + } + const auto& allowed = choice["allowed_tools"]; + if (!allowed.contains("mode") || !allowed["mode"].is_string() || + (allowed["mode"] != "auto" && allowed["mode"] != "required")) { + throw invalid_request("allowed_tools.mode must be 'auto' or 'required'", "tool_choice.allowed_tools.mode"); + } + if (!allowed.contains("tools") || !allowed["tools"].is_array() || allowed["tools"].empty()) { + throw invalid_request("allowed_tools.tools must be a non-empty array", "tool_choice.allowed_tools.tools"); + } + + std::unordered_set allowed_names; + for (std::size_t i = 0; i < allowed["tools"].size(); ++i) { + const auto& selected = allowed["tools"][i]; + const auto param = "tool_choice.allowed_tools.tools[" + std::to_string(i) + "]"; + if (!selected.is_object() || !selected.contains("type") || selected["type"] != "function" || + !selected.contains("function") || !selected["function"].is_object() || + !selected["function"].contains("name") || !selected["function"]["name"].is_string()) { + throw invalid_request(param + " must name a function", param); + } + const auto name = selected["function"]["name"].get(); + if (!available_names.contains(name)) { + throw invalid_request("allowed_tools names a function that is not present in tools: " + name, param); + } + allowed_names.insert(name); + } + + policy.mode = allowed["mode"] == "required" ? ToolChoiceMode::required : ToolChoiceMode::auto_select; + json selected_tools = json::array(); + for (const auto& tool : policy.tools) { + if (allowed_names.contains(tool["function"]["name"].get())) { + selected_tools.push_back(tool); + } + } + policy.tools = std::move(selected_tools); + } else { + throw invalid_request("tool_choice must be a supported string or function-tool object", "tool_choice"); + } + + if ((policy.mode == ToolChoiceMode::required || policy.mode == ToolChoiceMode::named) && policy.tools.empty()) { + throw invalid_request("tool_choice requires at least one function in tools", "tool_choice"); + } + return policy; +} + +inline json apply_policy_prompt(json messages, const ToolPolicy& policy) { + if (!messages.is_array()) { + throw invalid_request("messages must be an array", "messages"); + } + + std::string directive; + if (policy.mode == ToolChoiceMode::required) { + directive = "Tool selection policy: call one or more of the provided functions before replying."; + } else if (policy.mode == ToolChoiceMode::named) { + directive = "Tool selection policy: call the function named '" + policy.required_function + "' before replying."; + } + if (!policy.parallel_tool_calls && policy.mode != ToolChoiceMode::none) { + if (!directive.empty()) directive += " "; + directive += "Emit no more than one function call in this response."; + } + + if (!directive.empty()) { + const json policy_message = {{"role", "system"}, {"content", directive}}; + messages.insert(messages.begin(), policy_message); + } + return messages; +} + +inline void validate_tool_calls(const json& choices, const ToolPolicy& policy) { + if (!choices.is_array() || choices.empty() || !choices[0].is_object() || + !choices[0].contains("message") || !choices[0]["message"].is_object()) { + throw model_contract_error("model response is missing choices[0].message"); + } + + const auto& message = choices[0]["message"]; + if (message.contains("tool_calls") && !message["tool_calls"].is_null() && + !message["tool_calls"].is_array()) { + throw model_contract_error("choices[0].message.tool_calls must be an array"); + } + const bool has_calls = message.contains("tool_calls") && message["tool_calls"].is_array() && + !message["tool_calls"].empty(); + const std::size_t call_count = has_calls ? message["tool_calls"].size() : 0; + + if (!choices[0].contains("finish_reason") || !choices[0]["finish_reason"].is_string()) { + throw model_contract_error("model response is missing choices[0].finish_reason"); + } + const auto finish_reason = choices[0]["finish_reason"].get(); + if ((has_calls && finish_reason != "tool_calls") || (!has_calls && finish_reason == "tool_calls")) { + throw model_contract_error("finish_reason does not match the model's tool calls"); + } + + if (policy.mode == ToolChoiceMode::none && has_calls) { + throw model_contract_error("model emitted a tool call while tool_choice was 'none'"); + } + if ((policy.mode == ToolChoiceMode::required || policy.mode == ToolChoiceMode::named) && !has_calls) { + throw model_contract_error("model did not emit a tool call required by tool_choice"); + } + if (!policy.parallel_tool_calls && call_count > 1) { + throw model_contract_error("model emitted parallel tool calls while parallel_tool_calls was false"); + } + + std::unordered_set available_names; + for (const auto& tool : policy.tools) { + available_names.insert(tool["function"]["name"].get()); + } + + if (!has_calls) return; + for (std::size_t i = 0; i < call_count; ++i) { + const auto& call = message["tool_calls"][i]; + const auto prefix = "choices[0].message.tool_calls[" + std::to_string(i) + "]"; + if (!call.is_object() || !call.contains("type") || !call["type"].is_string() || + call["type"] != "function" || + !call.contains("id") || !call["id"].is_string() || call["id"].get().empty() || + !call.contains("function") || !call["function"].is_object() || + !call["function"].contains("name") || !call["function"]["name"].is_string()) { + throw model_contract_error(prefix + " is not a valid function tool call"); + } + const auto name = call["function"]["name"].get(); + if (!available_names.contains(name)) { + throw model_contract_error("model called an unavailable function: " + name); + } + if (policy.mode == ToolChoiceMode::named && name != policy.required_function) { + throw model_contract_error("model called '" + name + "' instead of required function '" + policy.required_function + "'"); + } + if (!call["function"].contains("arguments") || !call["function"]["arguments"].is_string()) { + throw model_contract_error(prefix + ".function.arguments must be a JSON-encoded string"); + } + try { + const auto arguments = json::parse(call["function"]["arguments"].get()); + if (!arguments.is_object()) { + throw model_contract_error(prefix + ".function.arguments must encode a JSON object"); + } + } catch (const nlohmann::json::exception&) { + throw model_contract_error(prefix + ".function.arguments is not valid JSON"); + } + } +} + +inline json error_response(const ToolPolicyError& error) { + return {{"error", { + {"message", error.what()}, + {"type", error.type}, + {"param", error.param}, + {"code", error.status_code}, + }}}; +} + +} // namespace openai_tools diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 26540966..898c7a63 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -7,6 +7,7 @@ * \version 0.9.24 */ #include "rest_handler.hpp" +#include "openai_tool_policy.hpp" #include "wstream_buf.hpp" #include "streaming_ostream.hpp" #include "streaming_ostream_openai.hpp" @@ -567,6 +568,42 @@ json RestHandler::build_nstream_response(std::string response_text) { }); } +namespace { + +void send_buffered_chat_completion_stream(const json& response, + const StreamResponseCallback& send_streaming_response) { + const auto& choice = response["choices"][0]; + json content_chunk = { + {"id", response["id"]}, + {"object", "chat.completion.chunk"}, + {"created", response["created"]}, + {"model", response["model"]}, + {"choices", json::array({{ + {"index", 0}, + {"delta", choice["message"]}, + {"finish_reason", nullptr}, + }})}, + }; + send_streaming_response(json("data: " + content_chunk.dump() + "\n\n"), false); + + json final_chunk = { + {"id", response["id"]}, + {"object", "chat.completion.chunk"}, + {"created", response["created"]}, + {"model", response["model"]}, + {"choices", json::array({{ + {"index", 0}, + {"delta", json::object()}, + {"finish_reason", choice["finish_reason"]}, + }})}, + {"usage", response["usage"]}, + }; + send_streaming_response(json("data: " + final_chunk.dump() + "\n\n"), false); + send_streaming_response(json("data: [DONE]\n\n"), true); +} + +} // namespace + ///@brief Handle the show request ///@param request the request ///@param send_response the send response @@ -1070,7 +1107,8 @@ void RestHandler::handle_openai_chat_completion(const json& request, std::string model = request.value("model", current_model_tag); bool stream = request.value("stream", false); int length_limit = request.value("max_tokens", request.value("max_completion_tokens", 4096)); - json tools = request.value("tools", json::array()); + const auto tool_policy = openai_tools::parse_tool_policy(request); + json tools = tool_policy.tools; json options = request.value("options", json::object()); auto load_start_time = time_utils::now(); @@ -1083,6 +1121,7 @@ void RestHandler::handle_openai_chat_completion(const json& request, configure_chat_engine_parameters(options, request); + current_messages = openai_tools::apply_policy_prompt(current_messages, tool_policy); current_messages = normalize_messages(current_messages); current_messages = normalize_template(current_messages); @@ -1123,7 +1162,7 @@ void RestHandler::handle_openai_chat_completion(const json& request, uniformed_input.tools = tools; meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first; meta_info.max_prefill_len = this->prefill_chunk_len; - if (stream){ + if (stream && !tool_policy.requires_buffered_validation()){ // Create a wrapper callback that passes the pre-formatted SSE string directly cancellation_token->reset(); auto_chat_engine->reset_parser(); @@ -1229,6 +1268,7 @@ void RestHandler::handle_openai_chat_completion(const json& request, } // check response_text json choices = build_nstream_response(response_text); + openai_tools::validate_tool_calls(choices, tool_policy); response = { {"id", "fastflowlm-chat-completion"}, {"object", "chat.completion"}, @@ -1252,9 +1292,16 @@ void RestHandler::handle_openai_chat_completion(const json& request, header_print("❌ ", "Generation Cancelled!"); this->prompt_cache.reset(); } - send_response(response); + if (stream) { + send_buffered_chat_completion_stream(response, send_streaming_response); + } + else { + send_response(response); + } } + } catch (const openai_tools::ToolPolicyError& e) { + send_response(openai_tools::error_response(e)); } catch (const std::exception& e) { json error_response = { {"error", { diff --git a/src/server/server.cpp b/src/server/server.cpp index b7bf6683..67876494 100644 --- a/src/server/server.cpp +++ b/src/server/server.cpp @@ -738,9 +738,9 @@ bool WebServer::handle_request(http::request& req, if (code == 400) { status = http::status::bad_request; } - //else if () { - - //} + else if (code >= 500) { + status = http::status::internal_server_error; + } } response_ref.result(status); @@ -1087,4 +1087,4 @@ std::unique_ptr create_lm_server(model_list& models, ModelDownloader& }); return server; -} \ No newline at end of file +} diff --git a/src/test/openai_tool_policy/CMakeLists.txt b/src/test/openai_tool_policy/CMakeLists.txt new file mode 100644 index 00000000..d8f4d51c --- /dev/null +++ b/src/test/openai_tool_policy/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.22) +project(openai_tool_policy_test LANGUAGES CXX) + +enable_testing() + +add_executable(openai_tool_policy_test test.cpp) +target_compile_features(openai_tool_policy_test PRIVATE cxx_std_20) +target_include_directories(openai_tool_policy_test PRIVATE ../../include) + +add_test(NAME openai_tool_policy_contract COMMAND openai_tool_policy_test) diff --git a/src/test/openai_tool_policy/test.cpp b/src/test/openai_tool_policy/test.cpp new file mode 100644 index 00000000..35286437 --- /dev/null +++ b/src/test/openai_tool_policy/test.cpp @@ -0,0 +1,160 @@ +#include "openai_tool_policy.hpp" + +#include +#include +#include + +using openai_tools::ToolChoiceMode; +using openai_tools::ToolPolicyError; +using openai_tools::json; + +namespace { + +json tools() { + return json::array({ + {{"type", "function"}, {"function", { + {"name", "read_file"}, + {"description", "Read a file"}, + {"parameters", {{"type", "object"}}}, + }}}, + {{"type", "function"}, {"function", { + {"name", "write_file"}, + {"description", "Write a file"}, + {"parameters", {{"type", "object"}}}, + }}}, + }); +} + +json choices_with_calls(std::initializer_list names) { + json calls = json::array(); + for (const auto& name : names) { + calls.push_back({ + {"id", "call_1"}, + {"type", "function"}, + {"function", {{"name", name}, {"arguments", R"({"path":"README.md"})"}}}, + }); + } + return json::array({{ + {"message", {{"role", "assistant"}, {"tool_calls", calls}}}, + {"finish_reason", "tool_calls"}, + }}); +} + +json choices_with_text() { + return json::array({{ + {"message", {{"role", "assistant"}, {"content", "done"}}}, + {"finish_reason", "stop"}, + }}); +} + +void require(bool condition) { + if (!condition) { + throw std::runtime_error("contract assertion failed"); + } +} + +void expect_error(const std::function& action, int status_code) { + try { + action(); + } catch (const ToolPolicyError& error) { + require(error.status_code == status_code); + return; + } + throw std::runtime_error("expected ToolPolicyError"); +} + +} // namespace + +int main() { + const auto automatic = openai_tools::parse_tool_policy({{"tools", tools()}}); + require(automatic.mode == ToolChoiceMode::auto_select); + require(automatic.tools.size() == 2); + require(automatic.requires_buffered_validation()); + + const auto plain_chat = openai_tools::parse_tool_policy(json::object()); + require(plain_chat.mode == ToolChoiceMode::none); + require(!plain_chat.requires_buffered_validation()); + + const auto none = openai_tools::parse_tool_policy({{"tools", tools()}, {"tool_choice", "none"}}); + require(none.mode == ToolChoiceMode::none); + require(none.tools.empty()); + openai_tools::validate_tool_calls(choices_with_text(), none); + expect_error([&] { openai_tools::validate_tool_calls(choices_with_calls({"read_file"}), none); }, 500); + + const auto required = openai_tools::parse_tool_policy({{"tools", tools()}, {"tool_choice", "required"}}); + openai_tools::validate_tool_calls(choices_with_calls({"read_file"}), required); + expect_error([&] { openai_tools::validate_tool_calls(choices_with_text(), required); }, 500); + + const json named_choice = {{"type", "function"}, {"function", {{"name", "write_file"}}}}; + const auto named = openai_tools::parse_tool_policy({{"tools", tools()}, {"tool_choice", named_choice}}); + require(named.mode == ToolChoiceMode::named); + require(named.tools.size() == 1); + require(named.tools[0]["function"]["name"] == "write_file"); + openai_tools::validate_tool_calls(choices_with_calls({"write_file"}), named); + expect_error([&] { openai_tools::validate_tool_calls(choices_with_calls({"read_file"}), named); }, 500); + + const json allowed_choice = { + {"type", "allowed_tools"}, + {"allowed_tools", { + {"mode", "required"}, + {"tools", json::array({ + {{"type", "function"}, {"function", {{"name", "read_file"}}}}, + })}, + }}, + }; + const auto allowed = openai_tools::parse_tool_policy({{"tools", tools()}, {"tool_choice", allowed_choice}}); + require(allowed.mode == ToolChoiceMode::required); + require(allowed.tools.size() == 1); + require(allowed.tools[0]["function"]["name"] == "read_file"); + openai_tools::validate_tool_calls(choices_with_calls({"read_file"}), allowed); + + const auto serial = openai_tools::parse_tool_policy({ + {"tools", tools()}, {"tool_choice", "required"}, {"parallel_tool_calls", false}, + }); + expect_error([&] { + openai_tools::validate_tool_calls(choices_with_calls({"read_file", "write_file"}), serial); + }, 500); + + expect_error([&] { openai_tools::parse_tool_policy({{"tool_choice", "required"}}); }, 400); + expect_error([&] { + openai_tools::parse_tool_policy({{"tools", tools()}, {"tool_choice", "sometimes"}}); + }, 400); + expect_error([&] { + openai_tools::parse_tool_policy({{"tools", tools()}, {"parallel_tool_calls", "false"}}); + }, 400); + expect_error([&] { openai_tools::parse_tool_policy({{"tools", nullptr}}); }, 400); + expect_error([&] { + openai_tools::parse_tool_policy({{"tools", tools()}, {"parallel_tool_calls", nullptr}}); + }, 400); + expect_error([&] { + openai_tools::parse_tool_policy({ + {"tools", tools()}, + {"tool_choice", {{"type", "function"}, {"function", {{"name", "missing"}}}}}, + }); + }, 400); + expect_error([&] { + auto unavailable = allowed_choice; + unavailable["allowed_tools"]["tools"][0]["function"]["name"] = "missing"; + openai_tools::parse_tool_policy({{"tools", tools()}, {"tool_choice", unavailable}}); + }, 400); + + auto malformed = choices_with_calls({"read_file"}); + malformed[0]["message"]["tool_calls"][0]["function"]["arguments"] = "{{bad"; + expect_error([&] { openai_tools::validate_tool_calls(malformed, required); }, 500); + + auto wrong_finish_reason = choices_with_calls({"read_file"}); + wrong_finish_reason[0]["finish_reason"] = "stop"; + expect_error([&] { openai_tools::validate_tool_calls(wrong_finish_reason, required); }, 500); + + auto strict_tools = tools(); + strict_tools[0]["function"]["strict"] = true; + expect_error([&] { openai_tools::parse_tool_policy({{"tools", strict_tools}}); }, 400); + + const auto prompted = openai_tools::apply_policy_prompt( + json::array({{{"role", "user"}, {"content", "write it"}}}), named); + require(prompted.size() == 2); + require(prompted[0]["role"] == "system"); + require(prompted[0]["content"].get().find("write_file") != std::string::npos); + + std::cout << "openai_tool_policy contract tests passed\n"; +} From a3cdde1d18a8242346c3e6ca1b530f3d7884dec9 Mon Sep 17 00:00:00 2001 From: William Watson Date: Mon, 3 Aug 2026 20:40:21 -0400 Subject: [PATCH 2/4] fix: complete OpenAI tool contract enforcement --- debian/rules | 3 + src/CMakeLists.txt | 6 ++ src/include/openai_tool_policy.hpp | 88 +++++++++++++++++++++- src/server/rest_handler.cpp | 65 ++++++---------- src/test/openai_tool_policy/CMakeLists.txt | 5 -- src/test/openai_tool_policy/test.cpp | 81 ++++++++++++++++++++ 6 files changed, 202 insertions(+), 46 deletions(-) diff --git a/debian/rules b/debian/rules index 60d66f44..c27f2373 100755 --- a/debian/rules +++ b/debian/rules @@ -14,6 +14,9 @@ override_dh_auto_configure: override_dh_auto_build: cmake --build build +override_dh_auto_test: + ctest --test-dir build --output-on-failure + override_dh_auto_install: DESTDIR=$(CURDIR)/debian/fastflowlm cmake --install build --prefix=/opt/fastflowlm rmdir --ignore-fail-on-non-empty debian/fastflowlm/opt/fastflowlm/bin diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6773768d..fedede22 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,8 @@ cmake_minimum_required(VERSION 3.22) project(flm LANGUAGES CXX VERSION 0.1.0) +include(CTest) + if(NOT DEFINED FLM_VERSION) message(FATAL_ERROR "FLM_VERSION must be specified externally. Use -DFLM_VERSION= when running cmake.") endif() @@ -809,3 +811,7 @@ if(NOT WIN32 AND NOT FLM_PORTABLE_BUILD AND NOT CMAKE_INSTALL_PREFIX STREQUAL "/ \"$ENV{DESTDIR}/usr/local/bin/flm\")" ) endif() + +if(BUILD_TESTING) + add_subdirectory(test/openai_tool_policy) +endif() diff --git a/src/include/openai_tool_policy.hpp b/src/include/openai_tool_policy.hpp index b67d664f..cab0d761 100644 --- a/src/include/openai_tool_policy.hpp +++ b/src/include/openai_tool_policy.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -44,6 +45,10 @@ struct ToolPolicy { } }; +struct StreamOptions { + bool include_usage = false; +}; + inline ToolPolicyError invalid_request(const std::string& message, const std::string& param) { return ToolPolicyError(message, "invalid_request_error", param, 400); } @@ -52,6 +57,72 @@ inline ToolPolicyError model_contract_error(const std::string& message) { return ToolPolicyError(message, "model_error", "tool_choice", 500); } +inline StreamOptions parse_stream_options(const json& request) { + StreamOptions options; + if (!request.contains("stream_options")) { + return options; + } + if (!request["stream_options"].is_object()) { + throw invalid_request("stream_options must be an object", "stream_options"); + } + + const auto& stream_options = request["stream_options"]; + if (stream_options.contains("include_usage")) { + if (!stream_options["include_usage"].is_boolean()) { + throw invalid_request( + "stream_options.include_usage must be a boolean", + "stream_options.include_usage"); + } + options.include_usage = stream_options["include_usage"].get(); + } + return options; +} + +inline std::vector build_buffered_stream_chunks( + const json& response, + const StreamOptions& options) { + const auto& choice = response["choices"][0]; + json content_chunk = { + {"id", response["id"]}, + {"object", "chat.completion.chunk"}, + {"created", response["created"]}, + {"model", response["model"]}, + {"choices", json::array({{ + {"index", 0}, + {"delta", choice["message"]}, + {"finish_reason", nullptr}, + }})}, + }; + + json finish_chunk = { + {"id", response["id"]}, + {"object", "chat.completion.chunk"}, + {"created", response["created"]}, + {"model", response["model"]}, + {"choices", json::array({{ + {"index", 0}, + {"delta", json::object()}, + {"finish_reason", choice["finish_reason"]}, + }})}, + }; + + if (!options.include_usage) { + return {std::move(content_chunk), std::move(finish_chunk)}; + } + + content_chunk["usage"] = nullptr; + finish_chunk["usage"] = nullptr; + json usage_chunk = { + {"id", response["id"]}, + {"object", "chat.completion.chunk"}, + {"created", response["created"]}, + {"model", response["model"]}, + {"choices", json::array()}, + {"usage", response["usage"]}, + }; + return {std::move(content_chunk), std::move(finish_chunk), std::move(usage_chunk)}; +} + inline bool valid_function_name(const std::string& name) { return !name.empty() && name.size() <= 64 && std::all_of(name.begin(), name.end(), [](unsigned char value) { return (value >= 'a' && value <= 'z') || (value >= 'A' && value <= 'Z') || @@ -72,6 +143,18 @@ inline std::string function_name(const json& tool, std::size_t index) { param + ".function.name must contain at most 64 letters, digits, underscores, or hyphens", param + ".function.name"); } + if (tool["function"].contains("description") && + !tool["function"]["description"].is_string()) { + throw invalid_request( + param + ".function.description must be a string", + param + ".function.description"); + } + if (tool["function"].contains("parameters") && + !tool["function"]["parameters"].is_object()) { + throw invalid_request( + param + ".function.parameters must be a JSON Schema object", + param + ".function.parameters"); + } if (tool["function"].contains("strict") && !tool["function"]["strict"].is_boolean()) { throw invalid_request(param + ".function.strict must be a boolean", param + ".function.strict"); } @@ -244,9 +327,12 @@ inline void validate_tool_calls(const json& choices, const ToolPolicy& policy) { if (policy.mode == ToolChoiceMode::none && has_calls) { throw model_contract_error("model emitted a tool call while tool_choice was 'none'"); } - if ((policy.mode == ToolChoiceMode::required || policy.mode == ToolChoiceMode::named) && !has_calls) { + if (policy.mode == ToolChoiceMode::required && !has_calls) { throw model_contract_error("model did not emit a tool call required by tool_choice"); } + if (policy.mode == ToolChoiceMode::named && call_count != 1) { + throw model_contract_error("model did not emit exactly one call to the function required by tool_choice"); + } if (!policy.parallel_tool_calls && call_count > 1) { throw model_contract_error("model emitted parallel tool calls while parallel_tool_calls was false"); } diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 898c7a63..596e9407 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -570,37 +570,14 @@ json RestHandler::build_nstream_response(std::string response_text) { namespace { -void send_buffered_chat_completion_stream(const json& response, - const StreamResponseCallback& send_streaming_response) { - const auto& choice = response["choices"][0]; - json content_chunk = { - {"id", response["id"]}, - {"object", "chat.completion.chunk"}, - {"created", response["created"]}, - {"model", response["model"]}, - {"choices", json::array({{ - {"index", 0}, - {"delta", choice["message"]}, - {"finish_reason", nullptr}, - }})}, - }; - send_streaming_response(json("data: " + content_chunk.dump() + "\n\n"), false); - - json final_chunk = { - {"id", response["id"]}, - {"object", "chat.completion.chunk"}, - {"created", response["created"]}, - {"model", response["model"]}, - {"choices", json::array({{ - {"index", 0}, - {"delta", json::object()}, - {"finish_reason", choice["finish_reason"]}, - }})}, - {"usage", response["usage"]}, - }; - send_streaming_response(json("data: " + final_chunk.dump() + "\n\n"), false); - send_streaming_response(json("data: [DONE]\n\n"), true); -} +void send_buffered_chat_completion_stream(const json& response, + const openai_tools::StreamOptions& stream_options, + const StreamResponseCallback& send_streaming_response) { + for (const auto& chunk : openai_tools::build_buffered_stream_chunks(response, stream_options)) { + send_streaming_response(json("data: " + chunk.dump() + "\n\n"), false); + } + send_streaming_response(json("data: [DONE]\n\n"), true); +} } // namespace @@ -1106,9 +1083,10 @@ void RestHandler::handle_openai_chat_completion(const json& request, json current_messages = request["messages"]; std::string model = request.value("model", current_model_tag); bool stream = request.value("stream", false); - int length_limit = request.value("max_tokens", request.value("max_completion_tokens", 4096)); - const auto tool_policy = openai_tools::parse_tool_policy(request); - json tools = tool_policy.tools; + int length_limit = request.value("max_tokens", request.value("max_completion_tokens", 4096)); + const auto tool_policy = openai_tools::parse_tool_policy(request); + const auto stream_options = openai_tools::parse_stream_options(request); + json tools = tool_policy.tools; json options = request.value("options", json::object()); auto load_start_time = time_utils::now(); @@ -1291,17 +1269,24 @@ void RestHandler::handle_openai_chat_completion(const json& request, if (meta_info.stop_reason == CANCEL_DETECTED) { header_print("❌ ", "Generation Cancelled!"); this->prompt_cache.reset(); - } - if (stream) { - send_buffered_chat_completion_stream(response, send_streaming_response); - } + } + if (stream) { + send_buffered_chat_completion_stream(response, stream_options, send_streaming_response); + } else { send_response(response); } } - } catch (const openai_tools::ToolPolicyError& e) { - send_response(openai_tools::error_response(e)); + } catch (const openai_tools::ToolPolicyError& e) { + if (e.type == "model_error") { + if (this->auto_chat_engine != nullptr) { + this->auto_chat_engine->reset_parser(); + this->auto_chat_engine->clear_context(); + } + this->prompt_cache.reset(); + } + send_response(openai_tools::error_response(e)); } catch (const std::exception& e) { json error_response = { {"error", { diff --git a/src/test/openai_tool_policy/CMakeLists.txt b/src/test/openai_tool_policy/CMakeLists.txt index d8f4d51c..5102dcee 100644 --- a/src/test/openai_tool_policy/CMakeLists.txt +++ b/src/test/openai_tool_policy/CMakeLists.txt @@ -1,8 +1,3 @@ -cmake_minimum_required(VERSION 3.22) -project(openai_tool_policy_test LANGUAGES CXX) - -enable_testing() - add_executable(openai_tool_policy_test test.cpp) target_compile_features(openai_tool_policy_test PRIVATE cxx_std_20) target_include_directories(openai_tool_policy_test PRIVATE ../../include) diff --git a/src/test/openai_tool_policy/test.cpp b/src/test/openai_tool_policy/test.cpp index 35286437..a314369d 100644 --- a/src/test/openai_tool_policy/test.cpp +++ b/src/test/openai_tool_policy/test.cpp @@ -25,6 +25,12 @@ json tools() { }); } +json minimal_tool() { + return json::array({ + {{"type", "function"}, {"function", {{"name", "ping"}}}}, + }); +} + json choices_with_calls(std::initializer_list names) { json calls = json::array(); for (const auto& name : names) { @@ -47,6 +53,21 @@ json choices_with_text() { }}); } +json buffered_response() { + return { + {"id", "chatcmpl-test"}, + {"object", "chat.completion"}, + {"created", 123}, + {"model", "test-model"}, + {"choices", choices_with_calls({"read_file"})}, + {"usage", { + {"prompt_tokens", 10}, + {"completion_tokens", 5}, + {"total_tokens", 15}, + }}, + }; +} + void require(bool condition) { if (!condition) { throw std::runtime_error("contract assertion failed"); @@ -92,6 +113,9 @@ int main() { require(named.tools[0]["function"]["name"] == "write_file"); openai_tools::validate_tool_calls(choices_with_calls({"write_file"}), named); expect_error([&] { openai_tools::validate_tool_calls(choices_with_calls({"read_file"}), named); }, 500); + expect_error([&] { + openai_tools::validate_tool_calls(choices_with_calls({"write_file", "write_file"}), named); + }, 500); const json allowed_choice = { {"type", "allowed_tools"}, @@ -150,6 +174,63 @@ int main() { strict_tools[0]["function"]["strict"] = true; expect_error([&] { openai_tools::parse_tool_policy({{"tools", strict_tools}}); }, 400); + auto invalid_description = tools(); + invalid_description[0]["function"]["description"] = json::array(); + expect_error([&] { + openai_tools::parse_tool_policy({{"tools", invalid_description}}); + }, 400); + + auto invalid_parameters = tools(); + invalid_parameters[0]["function"]["parameters"] = "not a schema"; + expect_error([&] { + openai_tools::parse_tool_policy({{"tools", invalid_parameters}}); + }, 400); + + auto invalid_strict = tools(); + invalid_strict[0]["function"]["strict"] = "false"; + expect_error([&] { + openai_tools::parse_tool_policy({{"tools", invalid_strict}}); + }, 400); + + const auto minimal = openai_tools::parse_tool_policy({{"tools", minimal_tool()}}); + require(minimal.tools.size() == 1); + + auto unicode_description = minimal_tool(); + unicode_description[0]["function"]["description"] = "Meteo pour Sao Paulo, Tokyo, and \U0001f30d"; + const auto unicode = openai_tools::parse_tool_policy({{"tools", unicode_description}}); + require(unicode.tools[0]["function"]["description"] == + unicode_description[0]["function"]["description"]); + + const auto default_stream = openai_tools::parse_stream_options(json::object()); + require(!default_stream.include_usage); + const auto chunks_without_usage = + openai_tools::build_buffered_stream_chunks(buffered_response(), default_stream); + require(chunks_without_usage.size() == 2); + require(!chunks_without_usage[0].contains("usage")); + require(!chunks_without_usage[1].contains("usage")); + require(chunks_without_usage[1]["choices"][0]["finish_reason"] == "tool_calls"); + + const auto usage_stream = openai_tools::parse_stream_options({ + {"stream_options", {{"include_usage", true}}}, + }); + require(usage_stream.include_usage); + const auto chunks_with_usage = + openai_tools::build_buffered_stream_chunks(buffered_response(), usage_stream); + require(chunks_with_usage.size() == 3); + require(chunks_with_usage[0]["usage"].is_null()); + require(chunks_with_usage[1]["usage"].is_null()); + require(chunks_with_usage[2]["choices"].empty()); + require(chunks_with_usage[2]["usage"]["total_tokens"] == 15); + + expect_error([&] { + openai_tools::parse_stream_options({{"stream_options", json::array()}}); + }, 400); + expect_error([&] { + openai_tools::parse_stream_options({ + {"stream_options", {{"include_usage", "true"}}}, + }); + }, 400); + const auto prompted = openai_tools::apply_policy_prompt( json::array({{{"role", "user"}, {"content", "write it"}}}), named); require(prompted.size() == 2); From 11c4081a5f93a09c805d0c7a3350412f02e485f7 Mon Sep 17 00:00:00 2001 From: William Watson Date: Mon, 3 Aug 2026 20:40:28 -0400 Subject: [PATCH 3/4] docs: define supported tool-calling contract --- docs/docs/instructions/server/tool_calling.md | 128 +++++++++++++++--- 1 file changed, 107 insertions(+), 21 deletions(-) diff --git a/docs/docs/instructions/server/tool_calling.md b/docs/docs/instructions/server/tool_calling.md index b74bc843..0f685d80 100644 --- a/docs/docs/instructions/server/tool_calling.md +++ b/docs/docs/instructions/server/tool_calling.md @@ -71,8 +71,28 @@ Please refer to the [model card](https://fastflowlm.com/docs/models/) to see whe ## OpenAI tool-selection behavior -The `/v1/chat/completions` endpoint accepts the OpenAI `tool_choice` values -`none`, `auto`, and `required`. It also accepts a named function choice: +FastFlowLM implements function tools at the `/v1/chat/completions` boundary. +The following table describes the supported contract. It does not imply support +for every tool feature available in other OpenAI endpoints. + +| Capability | Status | Behavior | +|---|---|---| +| Function tools | Supported | Each tool must have `type: "function"` and a valid function name. | +| `tool_choice: "none"` | Supported | No function definitions are sent to the model and tool calls are rejected. | +| `tool_choice: "auto"` | Supported | The model may return text or one or more calls. This is the default when tools are present. | +| `tool_choice: "required"` | Supported | The model must return one or more valid calls. | +| Named function choice | Supported | The model must return exactly one call to the selected function. | +| `allowed_tools` | Supported | Restricts the model to a named subset in `auto` or `required` mode. | +| `parallel_tool_calls` | Supported | `false` limits each assistant response to at most one call. | +| Streaming tool responses | Supported with buffering | Calls are validated before any Server-Sent Events (SSE) are emitted. | +| Strict JSON Schema | Not supported | A function definition with `strict: true` returns a request error. | +| Custom or hosted tools | Not supported | Only function tools can be passed to FastFlowLM model templates. | +| Responses API tools | Not supported | This contract applies only to Chat Completions. | + +### Selecting tools + +The endpoint accepts `none`, `auto`, and `required` as string values. A named +function choice uses this object form: ```json { @@ -83,27 +103,93 @@ The `/v1/chat/completions` endpoint accepts the OpenAI `tool_choice` values } ``` -`parallel_tool_calls` is a boolean. When it is `false`, a response containing -more than one function call is rejected. A named choice must refer to a function -in the request's `tools` array. `required` and named choices must produce at -least one function call; FastFlowLM returns a model error instead of silently -returning a successful text response when the model violates that contract. -The Chat Completions `allowed_tools` form is also supported for limiting the -request to a named subset of its function tools in `auto` or `required` mode. -Hosted and custom tool types are not supported by FastFlowLM model templates. +The name must identify a function in the same request's `tools` array. The +`allowed_tools` form can expose only a subset of the definitions: + +```json +{ + "tool_choice": { + "type": "allowed_tools", + "allowed_tools": { + "mode": "required", + "tools": [ + {"type": "function", "function": {"name": "get_temperature"}} + ] + } + } +} +``` + +`parallel_tool_calls` must be a JSON boolean. When it is `false`, a response +containing more than one call is rejected. A named function choice always +requires exactly one call, regardless of the parallel setting. + +### Function definitions and validation + +Function names are limited to 64 ASCII letters, digits, underscores, or hyphens. +`description`, when present, must be a string and may contain UTF-8 text. +`parameters`, when present, must be a JSON Schema object. `strict` must be a +boolean; `strict: true` is rejected because FastFlowLM does not yet provide +constrained JSON Schema decoding. Tool-call responses are checked before they are returned. Each call must name an -available function, and `function.arguments` must be a string containing a JSON -object. Requests with tools are therefore buffered before an OpenAI-compatible -stream is emitted. This keeps streaming response shapes compatible but makes -time to first token approximately equal to the complete tool-call generation -time. - -Function definitions with `strict: true` currently return an explicit request -error. FastFlowLM does not yet have constrained JSON-schema decoding, so -claiming strict schema adherence would be incorrect. Applications must continue -to validate argument values against their function schema before executing a -tool. +available function, have a non-empty call ID, and provide `function.arguments` +as a string containing a JSON object. FastFlowLM validates the transport shape, +not every constraint in the function's parameter schema. Applications must +validate the decoded object against their schema before executing a function. + +If generation violates the selected policy, returns an undeclared function, +or produces malformed arguments, FastFlowLM returns HTTP 500 with a structured +model error instead of presenting the output as a successful call: + +```json +{ + "error": { + "message": "model called an unavailable function: call", + "type": "model_error", + "param": "tool_choice", + "code": 500 + } +} +``` + +The parser state, model context, and prompt cache are cleared after this error, +so a subsequent request does not reuse the rejected generation. Invalid request +fields instead return HTTP 400 with `type: "invalid_request_error"` and identify +the rejected parameter. + +### Streaming and usage + +Requests with tools are buffered until the complete assistant response can be +validated. Time to first token is therefore approximately equal to the complete +tool-call generation time. After validation, FastFlowLM emits Chat Completions +SSE chunks followed by `data: [DONE]`. + +By default, streamed chunks do not contain `usage`. To request token totals, set: + +```json +{ + "stream": true, + "stream_options": {"include_usage": true} +} +``` + +With `include_usage: true`, ordinary chunks contain `"usage": null`. A separate +final chunk has an empty `choices` array and contains the total usage. If the +stream is interrupted, that final usage chunk may not arrive. + +### Executing calls safely + +The application, not FastFlowLM, executes functions. Before dispatching a call: + +1. Confirm the function name is registered for the current request. +2. Decode `function.arguments` as JSON and validate it against the function schema. +3. Apply authorization, path, network, and side-effect controls appropriate to the function. +4. Return one `role: "tool"` message with the matching `tool_call_id` for each call. +5. Preserve the assistant's complete `tool_calls` message when sending the next turn. + +Do not execute an unknown function, malformed argument object, or repeated +side-effecting call merely because it appeared in model output. ## 🐍 How to use tool calling with FLM via Python script From 3082411af6fec7e3df95ba0335f1e5f71cf69a16 Mon Sep 17 00:00:00 2001 From: William Watson Date: Mon, 3 Aug 2026 21:02:07 -0400 Subject: [PATCH 4/4] test: preserve live tool-calling contract harness --- src/test/openai_tool_policy/README.md | 52 ++ .../test_server_contract.py | 528 ++++++++++++++++++ 2 files changed, 580 insertions(+) create mode 100644 src/test/openai_tool_policy/README.md create mode 100755 src/test/openai_tool_policy/test_server_contract.py diff --git a/src/test/openai_tool_policy/README.md b/src/test/openai_tool_policy/README.md new file mode 100644 index 00000000..da1469d4 --- /dev/null +++ b/src/test/openai_tool_policy/README.md @@ -0,0 +1,52 @@ +# OpenAI Tool-Calling Contract Tests + +This directory contains two layers of contract coverage: + +- `test.cpp` verifies the policy and validation logic through CTest. +- `test_server_contract.py` exercises the OpenAI-compatible HTTP and SSE + boundaries against a live FastFlowLM server. + +The Python harness is intentionally manual. It requires an NPU, a downloaded +model, and a running FastFlowLM server, so it is not part of ordinary CI. + +## Run the live contract test + +Start one model at a time: + +```bash +flm serve gemma4-it:e4b --ctx-len 65536 --port 52625 +``` + +In another terminal, run the baseline contract and optional multi-turn +workflow: + +```bash +python3 src/test/openai_tool_policy/test_server_contract.py \ + gemma4-it:e4b --workflow +``` + +The model argument must match the model loaded by the server. To test another +model, stop the server, start that model, and run the harness again. Keep this +serial because concurrent model loading can exhaust NPU memory. + +Use a different endpoint or timeout when needed: + +```bash +python3 src/test/openai_tool_policy/test_server_contract.py \ + qwen3-it:4b \ + --base-url http://127.0.0.1:1234/v1 \ + --timeout 600 +``` + +## Coverage + +The live harness checks request validation; `none`, `required`, named, and +allowed-tool selection; fail-closed behavior and recovery; multi-turn tool +result chaining; and streamed usage accounting. `--workflow` adds a synthetic +read/edit/write skill flow to probe whether the loaded model can sustain a +tool-use conversation. + +The harness does not execute real tools or modify files. It supplies synthetic +tool results in memory and exits nonzero when any check fails. Very small token +limits are used deliberately in one negative-path check, so a logged +`model_error` can be expected while that check is running. diff --git a/src/test/openai_tool_policy/test_server_contract.py b/src/test/openai_tool_policy/test_server_contract.py new file mode 100755 index 00000000..f9e6812f --- /dev/null +++ b/src/test/openai_tool_policy/test_server_contract.py @@ -0,0 +1,528 @@ +#!/usr/bin/env python3 + +import argparse +import copy +import json +import sys +import urllib.error +import urllib.request + + +ENDPOINT = "http://127.0.0.1:52625/v1/chat/completions" +REQUEST_TIMEOUT = 300.0 + +WEATHER_TOOL = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, +} + +TIME_TOOL = { + "type": "function", + "function": { + "name": "get_time", + "description": "Get the current time for a location", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, +} + +CHAIN_TOOLS = [ + { + "type": "function", + "function": { + "name": "read_dataset", + "description": "Read a dataset by identifier", + "parameters": { + "type": "object", + "properties": {"dataset_id": {"type": "string"}}, + "required": ["dataset_id"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "calculate_total", + "description": "Calculate the total of a list of integers", + "parameters": { + "type": "object", + "properties": { + "values": {"type": "array", "items": {"type": "integer"}} + }, + "required": ["values"], + }, + }, + }, +] + +WORKFLOW_TOOLS = [ + { + "type": "function", + "function": { + "name": "list_directory", + "description": "List files and subdirectories at a path", + "parameters": { + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "read_file", + "description": "Read a text file at a path", + "parameters": { + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "write_file", + "description": "Create or overwrite a text file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "content": {"type": "string"}, + }, + "required": ["path", "content"], + }, + }, + }, +] + + +def post(payload): + request = urllib.request.Request( + ENDPOINT, + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT) as response: + return response.status, json.loads(response.read()) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read()) + + +def post_sse(payload): + request = urllib.request.Request( + ENDPOINT, + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT) as response: + events = [] + for raw_line in response.read().decode("utf-8").splitlines(): + if not raw_line.startswith("data: "): + continue + data = raw_line[6:] + events.append(data if data == "[DONE]" else json.loads(data)) + return response.status, events + + +def base_request(model, messages, tools, tool_choice="auto", max_tokens=256): + return { + "model": model, + "messages": copy.deepcopy(messages), + "tools": copy.deepcopy(tools), + "tool_choice": copy.deepcopy(tool_choice), + "parallel_tool_calls": False, + "temperature": 0.1, + "max_tokens": max_tokens, + } + + +def response_message(status, body): + if status != 200: + raise AssertionError(f"HTTP {status}: {body}") + return body["choices"][0]["message"], body["choices"][0]["finish_reason"] + + +def single_call(message, finish_reason, expected_name): + calls = message.get("tool_calls", []) + if finish_reason != "tool_calls" or len(calls) != 1: + raise AssertionError( + f"expected one {expected_name} call; finish={finish_reason}, " + f"calls={calls}, content={message.get('content')}" + ) + call = calls[0] + if call["function"]["name"] != expected_name: + raise AssertionError(f"expected {expected_name}, got {call['function']['name']}") + arguments = json.loads(call["function"]["arguments"]) + if not isinstance(arguments, dict): + raise AssertionError(f"arguments are not an object: {arguments!r}") + return call, arguments + + +def expect_request_error(model, mutate, expected_param): + payload = base_request( + model, + [{"role": "user", "content": "Get the weather for Boston."}], + [WEATHER_TOOL], + ) + mutate(payload) + status, body = post(payload) + error = body.get("error", {}) + if status != 400 or error.get("type") != "invalid_request_error": + raise AssertionError(f"expected request error, got HTTP {status}: {body}") + if error.get("param") != expected_param: + raise AssertionError(f"expected param {expected_param}, got {error.get('param')}: {body}") + + +def test_request_validation(model): + expect_request_error( + model, + lambda payload: payload["tools"][0]["function"].update({"description": []}), + "tools[0].function.description", + ) + expect_request_error( + model, + lambda payload: payload["tools"][0]["function"].update({"parameters": "bad"}), + "tools[0].function.parameters", + ) + expect_request_error( + model, + lambda payload: payload["tools"][0]["function"].update({"strict": True}), + "tools[0].function.strict", + ) + expect_request_error( + model, + lambda payload: payload.update( + {"stream": True, "stream_options": {"include_usage": "true"}} + ), + "stream_options.include_usage", + ) + + +def test_none(model): + status, body = post( + base_request( + model, + [{"role": "user", "content": "Call get_weather for Boston."}], + [WEATHER_TOOL], + "none", + max_tokens=128, + ) + ) + message, finish = response_message(status, body) + if message.get("tool_calls") or finish == "tool_calls": + raise AssertionError(f"tool_choice none returned calls: {body}") + + +def test_required(model): + status, body = post( + base_request( + model, + [{"role": "user", "content": "Get the weather for Boston using the function."}], + [WEATHER_TOOL], + "required", + ) + ) + message, finish = response_message(status, body) + _, arguments = single_call(message, finish, "get_weather") + if not isinstance(arguments.get("location"), str): + raise AssertionError(f"location is not a string: {arguments!r}") + + +def test_named(model): + choice = {"type": "function", "function": {"name": "get_weather"}} + status, body = post( + base_request( + model, + [{"role": "user", "content": "Get the weather for Tokyo using a function."}], + [WEATHER_TOOL, TIME_TOOL], + choice, + ) + ) + message, finish = response_message(status, body) + single_call(message, finish, "get_weather") + + +def test_allowed_tools(model): + choice = { + "type": "allowed_tools", + "allowed_tools": { + "mode": "required", + "tools": [{"type": "function", "function": {"name": "get_weather"}}], + }, + } + status, body = post( + base_request( + model, + [{"role": "user", "content": "Get the weather for Paris using a function."}], + [WEATHER_TOOL, TIME_TOOL], + choice, + ) + ) + message, finish = response_message(status, body) + single_call(message, finish, "get_weather") + + +def test_fail_closed_and_recovery(model): + status, body = post( + base_request( + model, + [{"role": "user", "content": "Get the weather for Miami using the function."}], + [WEATHER_TOOL], + "required", + max_tokens=1, + ) + ) + if status < 500 or body.get("error", {}).get("type") != "model_error": + raise AssertionError(f"truncated call escaped as HTTP {status}: {body}") + test_required(model) + + +def test_chain(model): + messages = [ + { + "role": "system", + "content": "Complete the workflow using functions. Do not guess tool results.", + }, + { + "role": "user", + "content": ( + "Read dataset alpha. After receiving it, calculate the total of its values. " + "After receiving the total, answer with the final total." + ), + }, + ] + status, body = post(base_request(model, messages, CHAIN_TOOLS)) + first_message, first_finish = response_message(status, body) + first_call, first_args = single_call(first_message, first_finish, "read_dataset") + if first_args.get("dataset_id") != "alpha": + raise AssertionError(f"wrong dataset arguments: {first_args!r}") + + messages.extend( + [ + first_message, + { + "role": "tool", + "tool_call_id": first_call["id"], + "name": "read_dataset", + "content": json.dumps({"dataset_id": "alpha", "values": [2, 3, 5]}), + }, + ] + ) + status, body = post(base_request(model, messages, CHAIN_TOOLS)) + second_message, second_finish = response_message(status, body) + second_call, second_args = single_call(second_message, second_finish, "calculate_total") + if second_args.get("values") != [2, 3, 5]: + raise AssertionError(f"tool result not carried into second call: {second_args!r}") + + messages.extend( + [ + second_message, + { + "role": "tool", + "tool_call_id": second_call["id"], + "name": "calculate_total", + "content": json.dumps({"total": 10}), + }, + ] + ) + status, body = post(base_request(model, messages, CHAIN_TOOLS)) + final_message, final_finish = response_message(status, body) + content = final_message.get("content") or "" + if final_message.get("tool_calls") or final_finish != "stop" or "10" not in content: + raise AssertionError(f"wrong final response: {body}") + + +def test_stream_usage(model): + for include_usage in (False, True): + payload = base_request( + model, + [{"role": "user", "content": "Get the weather for Boston using the function."}], + [WEATHER_TOOL], + "required", + ) + payload["stream"] = True + payload["stream_options"] = {"include_usage": include_usage} + status, events = post_sse(payload) + if status != 200 or not events or events[-1] != "[DONE]": + raise AssertionError(f"invalid SSE response: HTTP {status}: {events}") + chunks = events[:-1] + if include_usage: + if len(chunks) != 3: + raise AssertionError(f"expected three usage chunks: {chunks}") + if any(chunk.get("usage", "missing") is not None for chunk in chunks[:-1]): + raise AssertionError(f"ordinary chunks must contain usage null: {chunks}") + if chunks[-1].get("choices") != [] or not isinstance(chunks[-1].get("usage"), dict): + raise AssertionError(f"invalid final usage chunk: {chunks[-1]}") + elif any("usage" in chunk for chunk in chunks): + raise AssertionError(f"usage appeared without include_usage: {chunks}") + + +def test_skill_workflow(model): + messages = [ + { + "role": "system", + "content": ( + "You are an autonomous tool-using agent. Follow every step in order: " + "list .skills, read .skills/task-manager/SKILL.md, read tasks.md, write " + "the updated tasks.md, then answer. Never guess results or repeat a step." + ), + }, + {"role": "user", "content": "Add 'Buy bread, eggs and milk' to my todo list."}, + ] + read_skill = False + read_tasks = False + seen = set() + write_message = None + write_call = None + + for _ in range(8): + status, body = post(base_request(model, messages, WORKFLOW_TOOLS, max_tokens=768)) + message, finish = response_message(status, body) + calls = message.get("tool_calls", []) + if finish != "tool_calls" or len(calls) != 1: + raise AssertionError(f"workflow stopped before write: {body}") + call = calls[0] + name = call["function"]["name"] + arguments = json.loads(call["function"]["arguments"]) + signature = (name, json.dumps(arguments, sort_keys=True)) + if signature in seen: + raise AssertionError(f"workflow repeated a call: {signature!r}") + seen.add(signature) + path = arguments.get("path") + + if name == "list_directory" and path in (".", ".skills"): + result = ( + {"subdirectories": [".skills"], "files": ["tasks.md"]} + if path == "." + else {"subdirectories": ["task-manager"], "files": []} + ) + elif name == "list_directory" and path == ".skills/task-manager": + result = {"subdirectories": [], "files": ["SKILL.md"]} + elif name == "read_file" and path == ".skills/task-manager/SKILL.md": + read_skill = True + result = { + "content": "Read tasks.md, preserve it, and append the new unchecked task with write_file." + } + elif name == "read_file" and path == "tasks.md": + read_tasks = True + result = {"content": "# My Tasks\n- [ ] Call the dentist"} + elif name == "write_file": + content = arguments.get("content", "") + if not read_skill or not read_tasks: + raise AssertionError("workflow wrote before reading its skill and current tasks") + if path != "tasks.md" or "Call the dentist" not in content or "bread" not in content.lower(): + raise AssertionError(f"invalid write call: {arguments!r}") + write_message = message + write_call = call + break + else: + raise AssertionError(f"unexpected workflow call: {name}({arguments!r})") + + messages.extend( + [ + message, + { + "role": "tool", + "tool_call_id": call["id"], + "name": name, + "content": json.dumps(result), + }, + ] + ) + + if write_call is None: + raise AssertionError(f"workflow did not reach write_file: {seen!r}") + + messages.extend( + [ + write_message, + { + "role": "tool", + "tool_call_id": write_call["id"], + "name": "write_file", + "content": json.dumps({"status": "success", "path": "tasks.md"}), + }, + ] + ) + status, body = post(base_request(model, messages, WORKFLOW_TOOLS, max_tokens=768)) + message, finish = response_message(status, body) + if message.get("tool_calls") or finish != "stop" or "bread" not in (message.get("content") or "").lower(): + raise AssertionError(f"workflow did not finish correctly: {body}") + + +def run(model, include_workflow): + tests = [ + ("request_validation", test_request_validation), + ("none", test_none), + ("required", test_required), + ("named", test_named), + ("allowed_tools", test_allowed_tools), + ("fail_closed_and_recovery", test_fail_closed_and_recovery), + ("chain", test_chain), + ("stream_usage", test_stream_usage), + ] + if include_workflow: + tests.append(("skill_workflow", test_skill_workflow)) + + results = {} + for name, test in tests: + try: + test(model) + results[name] = {"status": "pass"} + except Exception as error: + results[name] = {"status": "fail", "error": str(error)} + return results + + +def main(): + global ENDPOINT, REQUEST_TIMEOUT + + parser = argparse.ArgumentParser( + description=( + "Run the OpenAI tool-calling contract tests against a live " + "FastFlowLM server." + ) + ) + parser.add_argument("model", help="Model tag loaded by the server") + parser.add_argument( + "--base-url", + default="http://127.0.0.1:52625/v1", + help="OpenAI-compatible API base URL (default: %(default)s)", + ) + parser.add_argument( + "--timeout", + type=float, + default=300.0, + help="Per-request timeout in seconds (default: %(default)s)", + ) + parser.add_argument( + "--workflow", + action="store_true", + help="Also run a multi-turn synthetic skill workflow", + ) + args = parser.parse_args() + ENDPOINT = f"{args.base_url.rstrip('/')}/chat/completions" + REQUEST_TIMEOUT = args.timeout + report = {args.model: run(args.model, args.workflow)} + print(json.dumps(report, indent=2)) + if any(result["status"] != "pass" for result in report[args.model].values()): + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main())