Skip to content

Dsv4 parsing encoding fixes #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tarruda wants to merge 2 commits into
base: dsv4
Choose a base branch
from
+381 −95
Open

Dsv4 parsing encoding fixes #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7212fcb
common: specialize DeepSeek V4 DSML parsing
tarruda Jul 4, 2026
8b00186
common: align DeepSeek DSML tool encoding
tarruda Jul 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
305 changes: 212 additions & 93 deletions common/chat.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,11 @@ static bool has_content_or_tool_calls(const common_chat_msg & msg) {
return !msg.content.empty() || !msg.tool_calls.empty();
}

static bool common_chat_template_uses_deepseek_dsml(const std::string & src) {
return src.find("dsml_token") != std::string::npos &&
src.find("DSML") != std::string::npos;
}

std::string common_chat_msg::render_content(const std::string & delimiter) const {
if (!content.empty() && !content_parts.empty()) {
throw std::runtime_error("Cannot specify both content and content_parts");
Expand Down Expand Up @@ -1856,7 +1861,8 @@ static common_chat_params common_chat_params_init_gigachat_v3(
}

static common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl,
const autoparser::generation_params & inputs) {
const autoparser::generation_params & inputs,
const std::string & tool_calls_tag = "function_calls") {
common_chat_params data;

data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
Expand All @@ -1879,8 +1885,8 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
const std::string DSML = "｜DSML｜";
const std::string THINK_START = "<think>";
const std::string THINK_END = "</think>";
const std::string FC_START = "<" + DSML + "function_calls>";
const std::string FC_END = "</" + DSML + "function_calls>";
const std::string FC_START = "<" + DSML + tool_calls_tag + ">";
const std::string FC_END = "</" + DSML + tool_calls_tag + ">";
const std::string INVOKE_START = "<" + DSML + "invoke";
const std::string INVOKE_END = "</" + DSML + "invoke>";
const std::string PARAM_START = "<" + DSML + "parameter";
Expand All @@ -1902,112 +1908,149 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
auto generation_prompt = p.literal(GEN_PROMPT);
auto end = p.end();

auto reasoning = p.eps();
if (extract_reasoning && inputs.enable_thinking) {
reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
} else if (extract_reasoning) {
// Thinking disabled but reasoning extraction requested: the generation prompt
// contains an empty <think></think> pair that must still be consumed.
reasoning = p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));
}

auto after_reasoning = p.eps();
if (has_response_format) {
auto response_format = p.rule("response-format",
p.literal("```json") + p.space() +
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
p.space() + p.literal("```"));
return generation_prompt + reasoning + response_format + end;
}
after_reasoning = response_format;
} else if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
after_reasoning = p.content(p.rest());
} else {
auto tool_choice = p.choice();
foreach_function(inputs.tools, [&](const json & tool) {
const auto & function = tool.at("function");
std::string name = function.at("name");
auto params = function.contains("parameters") ? function.at("parameters") : json::object();
const auto & props = params.contains("properties") ? params.at("properties") : json::object();

if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
return generation_prompt + reasoning + p.content(p.rest()) + end;
}
std::set<std::string> required;
if (params.contains("required")) {
params.at("required").get_to(required);
}

auto tool_choice = p.choice();
foreach_function(inputs.tools, [&](const json & tool) {
const auto & function = tool.at("function");
std::string name = function.at("name");
auto params = function.contains("parameters") ? function.at("parameters") : json::object();
const auto & props = params.contains("properties") ? params.at("properties") : json::object();
auto schema_info = common_schema_info();
schema_info.resolve_refs(params);

std::vector<size_t> required_indices;
std::vector<common_peg_parser> arg_parsers;
for (const auto & [param_name, param_schema] : props.items()) {
bool is_required = required.find(param_name) != required.end();
bool is_string = schema_info.resolves_to_string(param_schema);

auto arg = p.tool_arg(
p.tool_arg_open(
p.literal(PARAM_START + " name=\"") +
p.tool_arg_name(p.literal(param_name)) +
p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) +
(is_string
? p.tool_arg_string_value(p.until(PARAM_END))
: p.tool_arg_json_value(p.schema(p.json(),
"tool-" + name + "-arg-" + param_name + "-schema",
param_schema, false))) +
p.tool_arg_close(p.literal(PARAM_END)));

auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
const auto arg_idx = arg_parsers.size();
arg_parsers.push_back(named_arg);
if (is_required) {
required_indices.push_back(arg_idx);
}
}

std::set<std::string> required;
if (params.contains("required")) {
params.at("required").get_to(required);
}
auto contains_index = [](const std::vector<size_t> & values, size_t needle) {
for (const auto value : values) {
if (value == needle) {
return true;
}
}
return false;
};

auto schema_info = common_schema_info();
schema_info.resolve_refs(params);

std::vector<common_peg_parser> required_parsers;
std::vector<common_peg_parser> optional_parsers;
for (const auto & [param_name, param_schema] : props.items()) {
bool is_required = required.find(param_name) != required.end();
bool is_string = schema_info.resolves_to_string(param_schema);

auto arg = p.tool_arg(
p.tool_arg_open(
p.literal(PARAM_START + " name=\"") +
p.tool_arg_name(p.literal(param_name)) +
p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) +
(is_string
? p.tool_arg_string_value(p.until(PARAM_END))
: p.tool_arg_json_value(p.schema(p.json(),
"tool-" + name + "-arg-" + param_name + "-schema",
param_schema, false))) +
p.tool_arg_close(p.literal(PARAM_END)));

auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
if (is_required) {
required_parsers.push_back(named_arg);
} else {
optional_parsers.push_back(named_arg);
}
}
std::function<common_peg_parser(const std::vector<size_t> &)> build_args_seq =
[&](const std::vector<size_t> & remaining_required) {
if (arg_parsers.empty()) {
return p.eps();
}

common_peg_parser args_seq = p.eps();
for (size_t i = 0; i < required_parsers.size(); i++) {
if (i > 0) {
args_seq = args_seq + p.space();
}
args_seq = args_seq + required_parsers[i];
}
common_peg_parser skippable_args = p.choice();
bool has_skippable_args = false;
for (size_t i = 0; i < arg_parsers.size(); i++) {
if (contains_index(remaining_required, i)) {
continue;
}
skippable_args |= arg_parsers[i];
has_skippable_args = true;
}

if (!optional_parsers.empty()) {
common_peg_parser any_opt = p.choice();
for (const auto & opt : optional_parsers) {
any_opt |= opt;
}
args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);
}
auto skipped = has_skippable_args ? p.repeat(skippable_args + p.space(), 0, -1) : p.eps();
if (remaining_required.empty()) {
return skipped;
}

common_peg_parser args_choice = p.choice();
for (size_t i = 0; i < remaining_required.size(); i++) {
std::vector<size_t> next_required;
next_required.reserve(remaining_required.size() - 1);
for (size_t j = 0; j < remaining_required.size(); j++) {
if (i != j) {
next_required.push_back(remaining_required[j]);
}
}

auto required_arg = arg_parsers[remaining_required[i]] + p.space();
args_choice |= skipped + required_arg + build_args_seq(next_required);
}
return args_choice;
};

common_peg_parser invoke_body = args_seq;
auto func_parser = p.tool(
p.tool_open(p.literal(INVOKE_START + " name=\"") +
p.tool_name(p.literal(name)) + p.literal("\">\n")) +
invoke_body + p.space() +
p.tool_close(p.literal(INVOKE_END)));
common_peg_parser args_seq = build_args_seq(required_indices);

tool_choice |= p.rule("tool-" + name, func_parser);
});
common_peg_parser invoke_body = args_seq;
auto func_parser = p.tool(
p.tool_open(p.literal(INVOKE_START + " name=\"") +
p.tool_name(p.literal(name)) + p.literal("\">\n")) +
invoke_body + p.space() +
p.tool_close(p.literal(INVOKE_END)));

auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
tool_choice |= p.rule("tool-" + name, func_parser);
});

common_peg_parser tool_calls = p.eps();
if (inputs.parallel_tool_calls) {
tool_calls = p.trigger_rule("tool-call",
p.literal(FC_START) + p.space() + tool_choice +
p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
} else {
tool_calls = p.trigger_rule("tool-call",
p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;

common_peg_parser tool_calls = p.eps();
if (inputs.parallel_tool_calls) {
tool_calls = p.trigger_rule("tool-call",
p.literal(FC_START) + p.space() + tool_choice +
p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
} else {
tool_calls = p.trigger_rule("tool-call",
p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
}

if (!require_tools) {
tool_calls = p.optional(tool_calls);
}

auto content_before_tools = p.content(p.until(FC_START));
after_reasoning = content_before_tools + tool_calls;
}

if (!require_tools) {
tool_calls = p.optional(tool_calls);
if (extract_reasoning && inputs.enable_thinking) {
auto closed_reasoning = p.literal(THINK_START) + p.reasoning(p.until(THINK_END)) + p.literal(THINK_END) + after_reasoning;
auto open_reasoning = p.literal(THINK_START) + p.reasoning(p.rest());
return generation_prompt + p.choice({ closed_reasoning, open_reasoning }) + end;
} else if (extract_reasoning) {
// V3.2 pre-fills <think></think>, while V4 pre-fills </think> directly.
auto reasoning = p.optional(p.choice({
p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END),
p.literal(THINK_END),
}));
return generation_prompt + reasoning + after_reasoning + end;
}

auto content_before_tools = p.content(p.until(FC_START));
return generation_prompt + reasoning + content_before_tools + tool_calls + end;
return generation_prompt + after_reasoning + end;
});

data.parser = parser.save();
Expand Down Expand Up @@ -2214,6 +2257,71 @@ static void requires_non_null_content(json & messages) {
}
}

static void sort_tool_results_by_previous_tool_calls(json & messages) {
GGML_ASSERT(messages.is_array());

for (size_t i = 0; i + 1 < messages.size(); i++) {
auto & assistant = messages[i];
if (!assistant.is_object() ||
assistant.value("role", "") != "assistant" ||
!assistant.contains("tool_calls") ||
!assistant.at("tool_calls").is_array() ||
assistant.at("tool_calls").empty()) {
continue;
}

size_t begin = i + 1;
size_t end = begin;
while (end < messages.size() &&
messages[end].is_object() &&
messages[end].value("role", "") == "tool") {
end++;
}
if (begin == end) {
continue;
}

std::vector<bool> used(end - begin, false);
std::vector<json> ordered;
ordered.reserve(end - begin);

for (const auto & tool_call : assistant.at("tool_calls")) {
if (!tool_call.is_object() ||
!tool_call.contains("id") ||
!tool_call.at("id").is_string()) {
continue;
}

const auto id = tool_call.at("id").get<std::string>();
for (size_t j = begin; j < end; j++) {
auto & tool_result = messages[j];
if (used[j - begin] ||
!tool_result.contains("tool_call_id") ||
!tool_result.at("tool_call_id").is_string() ||
tool_result.at("tool_call_id").get<std::string>() != id) {
continue;
}

ordered.push_back(tool_result);
used[j - begin] = true;
break;
}
}

for (size_t j = begin; j < end; j++) {
if (!used[j - begin]) {
ordered.push_back(messages[j]);
}
}

for (size_t j = 0; j < ordered.size(); j++) {
messages[begin + j] = ordered[j];
}

i = end - 1;
}
}

// Gemma4 uses a custom tool_responses field instead of role:tool messages.
//
// This will transform a sequence of messages:
Expand Down Expand Up @@ -2597,13 +2705,20 @@ std::optional<common_chat_params> common_chat_try_specialized_template(

// DeepSeek V3.2 format detection: template defines dsml_token and uses it for tool calls.
// The template source contains the token as a variable assignment, not as a literal in markup.
if (src.find("dsml_token") != std::string::npos &&
src.find("function_calls") != std::string::npos &&
src.find("DSML") != std::string::npos) {
if (common_chat_template_uses_deepseek_dsml(src) &&
src.find("function_calls") != std::string::npos) {
LOG_DBG("Using specialized template: DeepSeek V3.2\n");
return common_chat_params_init_deepseek_v3_2(tmpl, params);
}

// DeepSeek V4 format detection: same DSML invoke/parameter shape as V3.2,
// but the outer block is tool_calls instead of function_calls.
if (common_chat_template_uses_deepseek_dsml(src) &&
src.find("tool_calls>") != std::string::npos) {
LOG_DBG("Using specialized template: DeepSeek V4\n");
return common_chat_params_init_deepseek_v3_2(tmpl, params, "tool_calls");
}

// Gemma4 format detection
if (src.find("'<|tool_call>call:'") != std::string::npos) {
if (src.find("{#- OpenAI Chat Completions:") == std::string::npos) {
Expand Down Expand Up @@ -2685,6 +2800,10 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
workaround::func_args_not_string(params.messages);
}

if (common_chat_template_uses_deepseek_dsml(src)) {
workaround::sort_tool_results_by_previous_tool_calls(params.messages);
}

params.extra_context = common_chat_extra_context();
for (auto el : inputs.chat_template_kwargs) {
params.extra_context[el.first] = json::parse(el.second);
Expand Down
Loading