diff --git a/common/chat.cpp b/common/chat.cpp index 22d2ee4a2a11..c441021b97fe 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -15,11 +15,13 @@ #include "nlohmann/json.hpp" +#include #include #include #include #include #include +#include #include #include @@ -1855,12 +1857,89 @@ static common_chat_params common_chat_params_init_gigachat_v3( return data; } +// The DeepSeek V4 reference implementation renders consecutive tool results into a single +// user block, ordered by the tool call order of the preceding assistant message (matched +// by tool call id) rather than by the order they appear in the conversation. +static json deepseek_v4_sort_tool_results(const json & messages) { + json adjusted = messages; + std::map call_order; + + for (size_t i = 0; i < adjusted.size();) { + const auto & msg = adjusted[i]; + const auto role = msg.value("role", ""); + + if (role == "assistant" && msg.contains("tool_calls") && + msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) { + call_order.clear(); + const auto & tool_calls = msg.at("tool_calls"); + for (size_t idx = 0; idx < tool_calls.size(); idx++) { + auto id = tool_calls[idx].value("id", ""); + if (!id.empty()) { + call_order[id] = idx; + } + } + i++; + continue; + } + + if (role != "user" && role != "tool") { + i++; + continue; + } + + // collect a maximal run of user/tool messages - they render into one user block + std::vector tool_positions; + size_t run_end = i; + for (; run_end < adjusted.size(); run_end++) { + const auto r = adjusted[run_end].value("role", ""); + if (r == "tool") { + tool_positions.push_back(run_end); + } else if (r != "user") { + break; + } + } + + if (tool_positions.size() > 1 && !call_order.empty()) { + std::vector results; + results.reserve(tool_positions.size()); + for (auto pos : tool_positions) { + results.push_back(adjusted[pos]); + } + std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) { + const auto order = [&](const json & m) { + auto it = call_order.find(m.value("tool_call_id", "")); + return it == call_order.end() ? (size_t) 0 : it->second; + }; + return order(a) < order(b); + }); + for (size_t k = 0; k < tool_positions.size(); k++) { + adjusted[tool_positions[k]] = std::move(results[k]); + } + } + + i = run_end; + } + + return adjusted; +} + static common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs) { common_chat_params data; - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + // V4 uses the same DSML markup as V3.2, but names the tool call block "tool_calls" + // instead of "function_calls", renders tool results in tool call order and its + // non-thinking generation prompt ends with a bare instead of an empty + // pair. + const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos; + + std::optional adjusted_messages; + if (is_v4) { + adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages); + } + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages); data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; data.supports_thinking = true; data.thinking_start_tag = ""; @@ -1879,8 +1958,9 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha const std::string DSML = "｜DSML｜"; const std::string THINK_START = ""; const std::string THINK_END = ""; - const std::string FC_START = "<" + DSML + "function_calls>"; - const std::string FC_END = ""; + const std::string TC_BLOCK = is_v4 ? "tool_calls" : "function_calls"; + const std::string FC_START = "<" + DSML + TC_BLOCK + ">"; + const std::string FC_END = ""; const std::string INVOKE_START = "<" + DSML + "invoke"; const std::string INVOKE_END = ""; const std::string PARAM_START = "<" + DSML + "parameter"; @@ -1907,8 +1987,11 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); } else if (extract_reasoning) { // Thinking disabled but reasoning extraction requested: the generation prompt - // contains an empty pair that must still be consumed. - reasoning = p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END)); + // contains an empty pair (V3.2) or a bare (V4) that + // must still be consumed. + reasoning = is_v4 + ? p.optional(p.literal(THINK_END)) + : p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END)); } if (has_response_format) { @@ -2612,12 +2695,14 @@ std::optional common_chat_try_specialized_template( return common_chat_params_init_gigachat_v3(tmpl, params); } - // DeepSeek V3.2 format detection: template defines dsml_token and uses it for tool calls. + // DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls. // The template source contains the token as a variable assignment, not as a literal in markup. + // V3.2 names the tool call block "function_calls", V4 names it "tool_calls". if (src.find("dsml_token") != std::string::npos && - src.find("function_calls") != std::string::npos && - src.find("DSML") != std::string::npos) { - LOG_DBG("Using specialized template: DeepSeek V3.2\n"); + src.find("DSML") != std::string::npos && + (src.find("function_calls") != std::string::npos || + src.find("tool_calls") != std::string::npos)) { + LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n"); return common_chat_params_init_deepseek_v3_2(tmpl, params); } diff --git a/common/jinja/caps.cpp b/common/jinja/caps.cpp index ae378ebd4fd0..0d9336c0d90e 100644 --- a/common/jinja/caps.cpp +++ b/common/jinja/caps.cpp @@ -23,6 +23,7 @@ void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled) { ctx.set_val("preserve_thinking", mk_val(enabled)); ctx.set_val("clear_thinking", mk_val(!enabled)); ctx.set_val("truncate_history_thinking", mk_val(!enabled)); + ctx.set_val("drop_reasoning", mk_val(!enabled)); } static void caps_try_execute(jinja::program & prog, diff --git a/models/templates/deepseek-ai-DeepSeek-V4.jinja b/models/templates/deepseek-ai-DeepSeek-V4.jinja index f19f787b1b7e..ad1ff8ce23aa 100644 --- a/models/templates/deepseek-ai-DeepSeek-V4.jinja +++ b/models/templates/deepseek-ai-DeepSeek-V4.jinja @@ -8,12 +8,15 @@ {%- set thinking = false -%} {%- endif -%} {%- endif -%} +{%- if not drop_thinking is defined -%} + {%- set drop_thinking = false -%} +{%- endif -%} {%- set dsml_token = '｜DSML｜' -%} {%- set thinking_start_token = '' -%} {%- set thinking_end_token = '' -%} {%- set tools_header = '## Tools\n\nYou have access to a set of tools to help answer the user\'s question. You can invoke tools by writing a "<' + dsml_token + 'tool_calls>" block like the following:\n\n<' + dsml_token + 'tool_calls>\n<' + dsml_token + 'invoke name="$TOOL_NAME">\n<' + dsml_token + 'parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE\n...\n\n<' + dsml_token + 'invoke name="$TOOL_NAME2">\n...\n\n\n\nString parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.\n\nIf thinking_mode is enabled (triggered by ' + thinking_start_token + '), you MUST output your complete reasoning inside ' + thinking_start_token + '...' + thinking_end_token + ' BEFORE any tool calls or final response.\n\nOtherwise, output directly after ' + thinking_end_token + ' with tool calls or final response.\n\n### Available Tool Schemas\n\n' -%} {%- set tools_footer = '\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n' -%} -{%- set ns = namespace(system_prompt='', is_first_sp=true) -%} +{%- set ns = namespace(system_prompt='', is_first_sp=true, has_tool_calls=false) -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} {%- if ns.is_first_sp -%} @@ -46,6 +49,11 @@ {%- endif -%} {%- endfor -%} {%- set state = namespace(in_user=false) -%} +{%- for message in messages -%} + {%- if message['role'] == 'tool' -%} + {%- set ns.has_tool_calls = true -%} + {%- endif -%} +{%- endfor -%} {%- for message in messages -%} {%- if message['role'] == 'user' or message['role'] == 'developer' -%} {%- if state.in_user -%} @@ -67,7 +75,8 @@ {%- set state.in_user = false -%} {{- '<｜Assistant｜>' -}} {%- set is_after_last_user = loop.index0 > last_user_idx.value -%} - {%- if is_after_last_user and thinking -%} + {%- set retain_reasoning = (not drop_thinking) or (is_after_last_user or ns.has_tool_calls) -%} + {%- if retain_reasoning and thinking -%} {{- thinking_start_token -}} {%- if message['reasoning_content'] is defined and message['reasoning_content'] -%} {{- message['reasoning_content'] -}} diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index e1e0a59e6de6..606d80f296af 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -109,6 +109,15 @@ static void assert_contains(const std::string & haystack, const std::string & ne } } +static void assert_not_contains(const std::string & haystack, const std::string & needle) { + if (haystack.find(needle) != std::string::npos) { + LOG_ERR("Expected NOT to contain: %s\n", needle.c_str()); + LOG_ERR("Actual: %s\n", haystack.c_str()); + common_log_flush(common_log_main()); + throw std::runtime_error("Test failed"); + } +} + static void assert_ends_with(const std::string & str, const std::string & suffix) { if (str.size() < suffix.size() || str.compare(str.size() - suffix.size(), suffix.size(), suffix) != 0) { @@ -4016,6 +4025,132 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .run(); } + // DeepSeek V4 tests - same DSML markup as V3.2, but the tool call block is named + // "tool_calls" and the non-thinking generation prompt ends in a bare + // instead of an empty pair. + { + auto tst = peg_tester("models/templates/deepseek-ai-DeepSeek-V4.jinja", detailed_debug); + + // Pure content (non-thinking mode; generation prompt ends with ) + tst.test("Hello, world!\nWhat's up?") + .enable_thinking(false) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .expect(message_assist) + .run(); + + // Thinking + content + tst.test("I'm\nthinkingHello, world!\nWhat's up?") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .expect(message_assist_thoughts) + .run(); + + // Thinking + tool call (single, string param) + tst.test( + "Let me check the time\n\n" + "<｜DSML｜tool_calls>\n" + "<｜DSML｜invoke name=\"get_time\">\n" + "<｜DSML｜parameter name=\"city\" string=\"true\">Tokyo\n" + "\n" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ get_time_tool }) + .expect(message_with_tool_calls_and_reasoning("get_time", R"({"city": "Tokyo"})", "Let me check the time")) + .run(); + + // Tool call without reasoning (non-thinking mode), integer param (string="false") + tst.test( + "<｜DSML｜tool_calls>\n" + "<｜DSML｜invoke name=\"special_function\">\n" + "<｜DSML｜parameter name=\"arg1\" string=\"false\">1\n" + "\n" + "") + .enable_thinking(false) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ special_function_tool }) + .expect(message_assist_call) + .run(); + + // Multiple parallel tool calls with reasoning + tst.test( + "Calling both\n\n" + "<｜DSML｜tool_calls>\n" + "<｜DSML｜invoke name=\"get_time\">\n" + "<｜DSML｜parameter name=\"city\" string=\"true\">Paris\n" + "\n" + "<｜DSML｜invoke name=\"get_weather\">\n" + "<｜DSML｜parameter name=\"city\" string=\"true\">Paris\n" + "\n" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .parallel_tool_calls(true) + .tools({ get_time_tool, get_weather_tool }) + .expect(message_with_reasoning_content_and_multiple_tool_calls( + "Calling both", "", + { { "get_time", R"({"city": "Paris"})" }, { "get_weather", R"({"city": "Paris"})" } })) + .run(); + + // Tool call with content before tool calls + tst.test( + "Thinking about it" + "Let me call the function.\n\n" + "<｜DSML｜tool_calls>\n" + "<｜DSML｜invoke name=\"special_function\">\n" + "<｜DSML｜parameter name=\"arg1\" string=\"false\">1\n" + "\n" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ special_function_tool }) + .expect_reasoning("Thinking about it") + .expect_content("Let me call the function.") + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + }) + .run(); + + // Tool call with multiple params (mixed types) + tst.test( + "Multi-arg call\n\n" + "<｜DSML｜tool_calls>\n" + "<｜DSML｜invoke name=\"magic_int\">\n" + "<｜DSML｜parameter name=\"ref\" string=\"false\">42\n" + "<｜DSML｜parameter name=\"name\" string=\"true\">foo bar\n" + "\n" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ magic_int_tool }) + .expect_reasoning("Multi-arg call") + .expect_tool_calls({ + { "magic_int", R"({"ref": 42, "name": "foo bar"})", {} }, + }) + .run(); + + // Continuation tests + tst.test("world!\nWhat's up?") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .enable_thinking(true) + .messages({ message_user, message_assist_prefill_content }) + .add_generation_prompt(false) + .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT) + .expect_reasoning("I'm thinking") + .expect_content("Hello, world!\nWhat's up?") + .run(); + + tst.test(" thinkingHello, world!\nWhat's up?") + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .enable_thinking(true) + .messages({ message_user, message_assist_prefill_reasoning }) + .add_generation_prompt(false) + .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING) + .expect_reasoning("I'm thinking") + .expect_content("Hello, world!\nWhat's up?") + .run(); + } + // GLM-4.6 tests - format: function_name\n...\n...\n { auto tst = peg_tester("models/templates/GLM-4.6.jinja", detailed_debug); @@ -5911,6 +6046,144 @@ static void test_developer_role_to_system_workaround() { } } +// Verify reasoning-trace retention rules in the DeepSeek-V4 template: +// all traces are retained unless drop_thinking is true AND the conversation +// has no tool calls, in which case only the last (after-final-user) trace is +// kept and earlier ones are dropped. +static void test_deepseek_v4_thinking_retention() { + LOG_DBG("%s\n", __func__); + + auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-V4.jinja"); + + common_chat_msg user_q1; user_q1.role = "user"; user_q1.content = "Question 1"; + common_chat_msg user_q2; user_q2.role = "user"; user_q2.content = "Question 2"; + common_chat_msg asst_a1 = simple_assist_msg("Answer 1", "thinking A1"); + common_chat_msg asst_a2 = simple_assist_msg("Answer 2", "thinking A2"); + + common_chat_msg tool_assist = message_with_tool_calls("special_function", "{\"arg1\": 1}"); + common_chat_msg tool_result; tool_result.role = "tool"; + tool_result.tool_name = "special_function"; tool_result.tool_call_id = "0"; tool_result.content = "result"; + + // The template uses U+FF5C as the role separator and literal think tags + // for the reasoning block. + const std::string asst_marker = "<\xef\xbd\x9c" "Assistant" "\xef\xbd\x9c>"; + // Built via concatenation so the thinking tokens are not interpreted by + // tooling processing this source file. + const std::string think_start = "<" "think" ">"; + const std::string think_end = ""; + + const std::string think_a1 = asst_marker + think_start + "thinking A1" + think_end; + const std::string think_a2 = asst_marker + think_start + "thinking A2" + think_end; + const std::string asst_no_think = asst_marker + think_end; + + auto render = [&](const std::vector & messages, bool drop_thinking) { + common_chat_templates_inputs inputs; + inputs.messages = messages; + inputs.add_generation_prompt = false; + inputs.chat_template_kwargs["thinking"] = "true"; + inputs.chat_template_kwargs["drop_thinking"] = drop_thinking ? "true" : "false"; + return common_chat_templates_apply(tmpls.get(), inputs).prompt; + }; + + // No tools, drop_thinking=false: all reasoning is retained. + { + auto prompt = render({ user_q1, asst_a1, user_q2, asst_a2 }, /* drop_thinking = */ false); + assert_contains(prompt, think_a1); + assert_contains(prompt, think_a2); + } + + // No tools, drop_thinking=true: only the last reasoning trace is kept, + // earlier ones are dropped (the assistant block emits just the end token). + { + auto prompt = render({ user_q1, asst_a1, user_q2, asst_a2 }, /* drop_thinking = */ true); + assert_not_contains(prompt, think_a1); + assert_contains(prompt, think_a2); + // The dropped assistant turn still opens with the marker + bare end token. + assert_contains(prompt, asst_no_think + "Answer 1"); + } + + // Single assistant turn, drop_thinking=true: the only trace is the last + // one, so it must be retained even with drop_thinking set. + { + auto prompt = render({ user_q1, asst_a1 }, /* drop_thinking = */ true); + assert_contains(prompt, think_a1); + } + + // Single assistant turn, drop_thinking=false: reasoning is retained. + { + auto prompt = render({ user_q1, asst_a1 }, /* drop_thinking = */ false); + assert_contains(prompt, think_a1); + } + + // With tool calls, drop_thinking=true: tool presence forces all reasoning + // to be retained, including the pre-tool-call trace. + { + auto prompt = render({ user_q1, asst_a1, user_q2, tool_assist, tool_result, asst_a2 }, + /* drop_thinking = */ true); + assert_contains(prompt, think_a1); + assert_contains(prompt, think_a2); + } + + // With tool calls, drop_thinking=false: all reasoning retained. + { + auto prompt = render({ user_q1, asst_a1, user_q2, tool_assist, tool_result, asst_a2 }, + /* drop_thinking = */ false); + assert_contains(prompt, think_a1); + assert_contains(prompt, think_a2); + } +} + +// Verify that consecutive tool results are rendered in the tool call order of the +// preceding assistant message (matched by tool call id), as required by the reference +// DeepSeek-V4 implementation. +static void test_deepseek_v4_tool_result_ordering() { + LOG_DBG("%s\n", __func__); + + auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-V4.jinja"); + + common_chat_msg user_q; user_q.role = "user"; user_q.content = "Question"; + + common_chat_msg assist_calls; + assist_calls.role = "assistant"; + assist_calls.tool_calls.push_back({ "get_time", "{\"city\": \"Paris\"}", "call_1" }); + assist_calls.tool_calls.push_back({ "get_weather", "{\"city\": \"Paris\"}", "call_2" }); + + common_chat_msg time_result; time_result.role = "tool"; + time_result.tool_name = "get_time"; time_result.tool_call_id = "call_1"; time_result.content = "12:00"; + common_chat_msg weather_result; weather_result.role = "tool"; + weather_result.tool_name = "get_weather"; weather_result.tool_call_id = "call_2"; weather_result.content = "sunny"; + + auto render = [&](const std::vector & messages) { + common_chat_templates_inputs inputs; + inputs.messages = messages; + inputs.add_generation_prompt = false; + return common_chat_templates_apply(tmpls.get(), inputs).prompt; + }; + + // Results sent out of order are reordered to match the tool call order. + { + auto prompt = render({ user_q, assist_calls, weather_result, time_result }); + assert_contains(prompt, "12:00\n\nsunny"); + } + + // Results already in call order stay put. + { + auto prompt = render({ user_q, assist_calls, time_result, weather_result }); + assert_contains(prompt, "12:00\n\nsunny"); + } + + // Without tool call ids there is nothing to match against; order is preserved. + { + auto no_id_calls = assist_calls; + no_id_calls.tool_calls[0].id = ""; + no_id_calls.tool_calls[1].id = ""; + auto no_id_weather = weather_result; no_id_weather.tool_call_id = ""; + auto no_id_time = time_result; no_id_time.tool_call_id = ""; + auto prompt = render({ user_q, no_id_calls, no_id_weather, no_id_time }); + assert_contains(prompt, "sunny\n\n12:00"); + } +} + static void test_msg_diffs_compute() { LOG_DBG("%s\n", __func__); { @@ -6067,6 +6340,8 @@ int main(int argc, char ** argv) { test_tools_oaicompat_json_conversion(); test_convert_responses_to_chatcmpl(); test_developer_role_to_system_workaround(); + test_deepseek_v4_thinking_retention(); + test_deepseek_v4_tool_result_ordering(); test_template_generation_prompt(); test_template_output_peg_parsers(detailed_debug); std::cout << "\n[chat] All tests passed!" << '\n';