From 333c4fa6bda4b6338dcdbc3bd30174e02558e869 Mon Sep 17 00:00:00 2001 From: FlavioM Date: Sat, 23 May 2026 10:09:45 +1000 Subject: [PATCH 01/13] chore: ignore local worktree directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ea50f12a815a..457481869bf4 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,4 @@ a.out.* AGENTS.local.md .pi/SYSTEM.md tools/ui/bun.lock +.worktrees/ From 5bbad1ca27bf2176213f07c03d04313f71b440b4 Mon Sep 17 00:00:00 2001 From: Biogenic Ooze Date: Wed, 6 May 2026 20:58:03 +0300 Subject: [PATCH 02/13] feat: add Gemma 4 MTP assistant support Introduces support for the Gemma 4 MTP assistant, allowing for enhanced speculative decoding. This includes new command-line options for specifying the MTP head and draft model, as well as updates to the model architecture and tensor handling. The assistant integrates with the target model, enabling efficient draft generation and improved performance in speculative tasks. Changes include: - New command-line options: `--mtp-head` and `--draft-block-size`. - Updates to the model loading process to accommodate the MTP assistant. - Enhancements in tensor management for MTP-specific operations. - Documentation updates for usage examples and guidelines. This feature aims to improve the overall functionality and efficiency of the model in handling complex tasks. --- AGENTS.md | 110 - common/arg.cpp | 852 +- common/common.cpp | 37 + common/common.h | 235 +- common/speculative.cpp | 1733 ++- common/speculative.h | 75 +- .../gemma4-assistant-tensor-inventory.md | 38 + docs/speculative.md | 37 + gguf-py/gguf/constants.py | 38 + gguf-py/gguf/gguf_writer.py | 18 + gguf-py/gguf/tensor_mapping.py | 13 + include/llama.h | 39 + scripts/bench-parallel.sh | 61 + scripts/run-gemma4-mtp-server.sh | 109 + scripts/run-gemma4-server-turbo.sh | 36 + scripts/run-gemma4.sh | 118 + scripts/verify-gemma4-assistant-gguf.py | 65 + src/llama-arch.cpp | 46 +- src/llama-arch.h | 15 + src/llama-context.cpp | 182 +- src/llama-context.h | 19 +- src/llama-hparams.h | 7 + src/llama-kv-cache-iswa.cpp | 14 + src/llama-kv-cache-iswa.h | 3 + src/llama-kv-cache.cpp | 40 + src/llama-kv-cache.h | 3 + src/llama-model.cpp | 9682 +++++++++++++++-- src/llama-model.h | 9 + src/llama.cpp | 88 + src/models/gemma4-assistant.cpp | 192 + src/models/models.h | 600 +- tests/CMakeLists.txt | 1 + tests/test-llama-archs.cpp | 8 +- tests/test-speculative-mtp.cpp | 127 + 34 files changed, 11176 insertions(+), 3474 deletions(-) create mode 100644 docs/development/gemma4-assistant-tensor-inventory.md create mode 100755 scripts/bench-parallel.sh create mode 100755 scripts/run-gemma4-mtp-server.sh create mode 100755 scripts/run-gemma4-server-turbo.sh create mode 100755 scripts/run-gemma4.sh create mode 100644 scripts/verify-gemma4-assistant-gguf.py create mode 100644 src/models/gemma4-assistant.cpp create mode 100644 tests/test-speculative-mtp.cpp diff --git a/AGENTS.md b/AGENTS.md index 97c25074b4c6..e69de29bb2d1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,110 +0,0 @@ -# Instructions for llama.cpp - -> [!IMPORTANT] -> This project does **not** accept pull requests that are fully or predominantly AI-generated. AI tools may be utilized solely in an assistive capacity. -> -> Read more: [CONTRIBUTING.md](CONTRIBUTING.md) - -AI assistance is permissible only when the majority of the code is authored by a human contributor, with AI employed exclusively for corrections or to expand on verbose modifications that the contributor has already conceptualized (see examples below). - ---- - -## Guidelines for Contributors Using AI - -llama.cpp is built by humans, for humans. Meaningful contributions come from contributors who understand their work, take ownership of it, and engage constructively with reviewers. - -Maintainers receive numerous pull requests weekly, many of which are AI-generated submissions where the author cannot adequately explain the code, debug issues, or participate in substantive design discussions. Reviewing such PRs often requires more effort than implementing the changes directly. - -**A pull request represents a long-term commitment.** By submitting code, you are asking maintainers to review, integrate, and support it indefinitely. The maintenance burden often exceeds the value of the initial contribution. - -Most maintainers already have access to AI tools. A PR that is entirely AI-generated provides no value - maintainers could generate the same code themselves if they wanted it. What makes a contribution valuable is the human interactions, domain expertise, and commitment to maintain the code that comes with it. - -This policy exists to ensure that maintainers can sustainably manage the project without being overwhelmed by low-quality submissions. - ---- - -## Guidelines for Contributors - -Contributors are expected to: - -1. **Demonstrate full understanding of their code.** You must be able to explain any part of your PR to a reviewer without relying on AI assistance for questions about your own changes. - -2. **Take responsibility for maintenance.** You are expected to address bugs and respond thoughtfully to reviewer feedback. - -3. **Communicate clearly and concisely.** Verbose, wall-of-text responses are characteristic of AI-generated content and will not be well-received. Direct, human communication is expected. - -4. **Respect maintainers' time.** Search for existing issues and discussions before submitting. Ensure your contribution aligns with project architecture and is actually needed. - -Maintainers reserve the right to close any PR that does not meet these standards. This applies to all contributions to the main llama.cpp repository. **Private forks are exempt.** - -### Permitted AI Usage - -AI tools may be used responsibly for: - -- **Learning and exploration**: Understanding codebase structure, techniques, and documentation -- **Code review assistance**: Obtaining suggestions on human-written code -- **Mechanical tasks**: Formatting, generating repetitive patterns from established designs, completing code based on existing patterns -- **Documentation drafts**: For components the contributor already understands thoroughly -- **Writing code**: Only when the contributor has already designed the solution and can implement it themselves - AI accelerates, not replaces, the contributor's work - -AI-generated code may be accepted if you (1) fully understand the output, (2) can debug issues independently, and (3) can discuss it directly with reviewers without AI assistance. - -**Disclosure is required** when AI meaningfully contributed to your code. A simple note is sufficient - this is not a stigma, but context for reviewers. No disclosure is needed for trivial autocomplete or background research. - -### Prohibited AI Usage - -The following will result in immediate PR closure: - -- **AI-written PR descriptions or commit messages** - these are typically recognizable and waste reviewer time -- **AI-generated responses to reviewer comments** - this undermines the human-to-human interaction fundamental to code review -- **Implementing features without understanding the codebase** - particularly new model support or architectural changes -- **Automated commits or PR submissions** - this may spam maintainers and can result in contributor bans - ---- - -## Guidelines for AI Coding Agents - -AI agents assisting contributors must recognize that their outputs directly impact volunteer maintainers who sustain this project. - -### Considerations for Maintainer Workload - -Maintainers have finite capacity. Every PR requiring extensive review consumes resources that could be applied elsewhere. Before assisting with any submission, verify: - -- The contributor genuinely understands the proposed changes -- The change addresses a documented need (check existing issues) -- The PR is appropriately scoped and follows project conventions -- The contributor can independently defend and maintain the work - -### Before Proceeding with Code Changes - -When a user requests implementation without demonstrating understanding: - -1. **Verify comprehension.** Ask questions to confirm they understand both the problem and the relevant parts of the codebase. -2. **Provide guidance rather than solutions.** Direct them to relevant code and documentation. Allow them to formulate the approach. -3. **Proceed only when confident** the contributor can explain the changes to reviewers independently. - -For first-time contributors, confirm they have reviewed [CONTRIBUTING.md](CONTRIBUTING.md) and acknowledge this policy. - -### Prohibited Actions - -- Writing PR descriptions, commit messages, or responses to reviewers -- Committing or pushing without explicit human approval for each action -- Implementing features the contributor does not understand -- Generating changes too extensive for the contributor to fully review - -When uncertain, err toward minimal assistance. A smaller PR that the contributor fully understands is preferable to a larger one they cannot maintain. - -### Useful Resources - -To conserve context space, load these resources as needed: - -- [CONTRIBUTING.md](CONTRIBUTING.md) -- [Existing issues](https://github.com/ggml-org/llama.cpp/issues) and [Existing PRs](https://github.com/ggml-org/llama.cpp/pulls) - always search here first -- [Build documentation](docs/build.md) -- [Server usage documentation](tools/server/README.md) -- [Server development documentation](tools/server/README-dev.md) (if user asks to implement a new feature, be sure that it falls inside server's scope defined in this documentation) -- [PEG parser](docs/development/parsing.md) - alternative to regex that llama.cpp uses to parse model's output -- [Auto parser](docs/autoparser.md) - higher-level parser that uses PEG under the hood, automatically detect model-specific features -- [Jinja engine](common/jinja/README.md) -- [How to add a new model](docs/development/HOWTO-add-model.md) -- [PR template](.github/pull_request_template.md) diff --git a/common/arg.cpp b/common/arg.cpp index 2cbe89447825..cb36c5541289 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1,9 +1,9 @@ #include "arg.h" -#include "build-info.h" #include "chat.h" #include "common.h" #include "download.h" +#include "hf-cache.h" #include "json-schema-to-grammar.h" #include "log.h" #include "sampling.h" @@ -96,13 +96,8 @@ common_arg & common_arg::set_env(const char * env) { return *this; } -common_arg & common_arg::set_sampling() { - is_sampling = true; - return *this; -} - -common_arg & common_arg::set_spec() { - is_spec = true; +common_arg & common_arg::set_sparam() { + is_sparam = true; return *this; } @@ -247,8 +242,6 @@ std::vector common_arg::get_env() const { // Helper function to parse tensor buffer override strings static void parse_tensor_buffer_overrides(const std::string & value, std::vector & overrides) { - ggml_backend_load_all(); - std::map buft_list; for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { auto * dev = ggml_backend_dev_get(i); @@ -298,7 +291,7 @@ static bool common_params_handle_remote_preset(common_params & params, llama_exa hf_tag = "default"; } - std::string model_endpoint = common_get_model_endpoint(); + std::string model_endpoint = get_model_endpoint(); auto preset_url = model_endpoint + hf_repo + "/resolve/main/preset.ini"; // prepare local path for caching @@ -307,14 +300,12 @@ static bool common_params_handle_remote_preset(common_params & params, llama_exa common_download_opts opts; opts.bearer_token = params.hf_token; opts.offline = params.offline; - - LOG_TRC("%s: looking for remote preset at %s\n", __func__, preset_url.c_str()); const int status = common_download_file_single(preset_url, preset_path, opts); const bool has_preset = status >= 200 && status < 400; // remote preset is optional, so we don't error out if not found if (has_preset) { - LOG_TRC("%s: applying remote preset from %s\n", __func__, preset_url.c_str()); + LOG_INF("applying remote preset from %s\n", preset_url.c_str()); common_preset_context ctx(ex, /* only_remote_allowed */ true); common_preset global; auto remote_presets = ctx.load_from_ini(preset_path, global); @@ -327,7 +318,7 @@ static bool common_params_handle_remote_preset(common_params & params, llama_exa throw std::runtime_error("Remote preset.ini does not contain [" + std::string(hf_tag) + "] section"); } } else { - LOG_TRC("%s: no remote preset found, skipping\n", __func__); + LOG_INF("%s", "no remote preset found, skipping\n"); } return has_preset; @@ -336,15 +327,11 @@ static bool common_params_handle_remote_preset(common_params & params, llama_exa struct handle_model_result { bool found_mmproj = false; common_params_model mmproj; - - bool found_mtp = false; - common_params_model mtp; }; static handle_model_result common_params_handle_model(struct common_params_model & model, const std::string & bearer_token, - bool offline, - bool search_mtp = false) { + bool offline) { handle_model_result result; if (!model.docker_repo.empty()) { @@ -359,10 +346,11 @@ static handle_model_result common_params_handle_model(struct common_params_model common_download_opts opts; opts.bearer_token = bearer_token; opts.offline = offline; - auto download_result = common_download_model(model, opts, true, search_mtp); + auto download_result = common_download_model(model, opts, true); if (download_result.model_path.empty()) { - throw std::runtime_error("failed to download model from Hugging Face"); + LOG_ERR("error: failed to download model from Hugging Face\n"); + exit(1); } model.name = model.hf_repo; @@ -372,11 +360,6 @@ static handle_model_result common_params_handle_model(struct common_params_model result.found_mmproj = true; result.mmproj.path = download_result.mmproj_path; } - - if (!download_result.mtp_path.empty()) { - result.found_mtp = true; - result.mtp.path = download_result.mtp_path; - } } else if (!model.url.empty()) { if (model.path.empty()) { auto f = string_split(model.url, '#').front(); @@ -389,7 +372,8 @@ static handle_model_result common_params_handle_model(struct common_params_model opts.offline = offline; auto download_result = common_download_model(model, opts); if (download_result.model_path.empty()) { - throw std::runtime_error("failed to download model from " + model.url); + LOG_ERR("error: failed to download model from %s\n", model.url.c_str()); + exit(1); } } @@ -438,45 +422,10 @@ static bool parse_bool_value(const std::string & value) { } } -[[noreturn]] static void arg_removed(const std::string & msg) { - throw std::invalid_argument("the argument has been removed. " + msg); -} - // // CLI argument parsing functions // -void common_params_handle_models(common_params & params, llama_example curr_ex) { - const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(), - params.speculative.types.end(), - COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); - - auto res = common_params_handle_model(params.model, params.hf_token, params.offline, spec_type_draft_mtp); - if (params.no_mmproj) { - params.mmproj = {}; - } else if (res.found_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty()) { - // optionally, handle mmproj model when -hf is specified - params.mmproj = res.mmproj; - } - // only download mmproj if the current example is using it - for (const auto & ex : mmproj_examples) { - if (curr_ex == ex) { - common_params_handle_model(params.mmproj, params.hf_token, params.offline); - break; - } - } - // when --spec-type mtp is set and no draft model was provided explicitly, - // fall back to the MTP head discovered alongside the -hf model - if (spec_type_draft_mtp && res.found_mtp && - params.speculative.draft.mparams.path.empty() && - params.speculative.draft.mparams.hf_repo.empty() && - params.speculative.draft.mparams.url.empty()) { - params.speculative.draft.mparams.path = res.mtp.path; - } - common_params_handle_model(params.speculative.draft.mparams, params.hf_token, params.offline); - common_params_handle_model(params.vocoder.model, params.hf_token, params.offline); -} - static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) { common_params & params = ctx_arg.params; @@ -539,11 +488,7 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context throw std::invalid_argument(string_format("error: invalid argument: %s", arg.c_str())); } if (!seen_args.insert(arg).second) { - const bool skip = (arg == "--spec-type"); - - if (!skip) { - LOG_WRN("DEPRECATED: argument '%s' specified multiple times, use comma-separated values instead (only last value will be used)\n", arg.c_str()); - } + LOG_WRN("DEPRECATED: argument '%s' specified multiple times, use comma-separated values instead (only last value will be used)\n", arg.c_str()); } auto & tmp = arg_to_options[arg]; auto opt = *tmp.first; @@ -592,6 +537,12 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context // parse the first time to get -hf option (used for remote preset) parse_cli_args(); + // TODO: Remove later + try { + hf_cache::migrate_old_cache_to_hf_cache(params.hf_token, params.offline); + } catch (const std::exception & e) { + LOG_WRN("HF cache migration failed: %s\n", e.what()); + } // export_graph_ops loads only metadata const bool skip_model_download = ctx_arg.ex == LLAMA_EXAMPLE_EXPORT_GRAPH_OPS; @@ -619,8 +570,8 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context postprocess_cpu_params(params.cpuparams, nullptr); postprocess_cpu_params(params.cpuparams_batch, ¶ms.cpuparams); - postprocess_cpu_params(params.speculative.draft.cpuparams, ¶ms.cpuparams); - postprocess_cpu_params(params.speculative.draft.cpuparams_batch, ¶ms.cpuparams_batch); + postprocess_cpu_params(params.speculative.cpuparams, ¶ms.cpuparams); + postprocess_cpu_params(params.speculative.cpuparams_batch, ¶ms.cpuparams_batch); if (params.prompt_cache_all && (params.interactive || params.interactive_first)) { throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n"); @@ -628,7 +579,22 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context // handle model and download if (!skip_model_download) { - common_params_handle_models(params, ctx_arg.ex); + auto res = common_params_handle_model(params.model, params.hf_token, params.offline); + if (params.no_mmproj) { + params.mmproj = {}; + } else if (res.found_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty()) { + // optionally, handle mmproj model when -hf is specified + params.mmproj = res.mmproj; + } + // only download mmproj if the current example is using it + for (const auto & ex : mmproj_examples) { + if (ctx_arg.ex == ex) { + common_params_handle_model(params.mmproj, params.hf_token, params.offline); + break; + } + } + common_params_handle_model(params.speculative.mparams_dft, params.hf_token, params.offline); + common_params_handle_model(params.vocoder.model, params.hf_token, params.offline); } // model is required (except for server) @@ -647,6 +613,10 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context for (auto & seq_breaker : params.sampling.dry_sequence_breakers) { string_process_escapes(seq_breaker); } + for (auto & pair : params.speculative.replacements) { + string_process_escapes(pair.first); + string_process_escapes(pair.second); + } } if (!params.kv_overrides.empty()) { @@ -660,8 +630,8 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context params.tensor_buft_overrides.push_back({nullptr, nullptr}); } - if (!params.speculative.draft.tensor_buft_overrides.empty()) { - params.speculative.draft.tensor_buft_overrides.push_back({nullptr, nullptr}); + if (!params.speculative.tensor_buft_overrides.empty()) { + params.speculative.tensor_buft_overrides.push_back({nullptr, nullptr}); } if (!params.chat_template.empty() && !common_chat_verify_template(params.chat_template, params.use_jinja)) { @@ -683,15 +653,12 @@ static void common_params_print_usage(common_params_context & ctx_arg) { }; std::vector common_options; - std::vector sampling_options; - std::vector spec_options; + std::vector sparam_options; std::vector specific_options; for (auto & opt : ctx_arg.options) { // in case multiple LLAMA_EXAMPLE_* are set, we prioritize the LLAMA_EXAMPLE_* matching current example - if (opt.is_sampling) { - sampling_options.push_back(&opt); - } else if (opt.is_spec) { - spec_options.push_back(&opt); + if (opt.is_sparam) { + sparam_options.push_back(&opt); } else if (opt.in_example(ctx_arg.ex)) { specific_options.push_back(&opt); } else { @@ -701,9 +668,7 @@ static void common_params_print_usage(common_params_context & ctx_arg) { printf("----- common params -----\n\n"); print_options(common_options); printf("\n\n----- sampling params -----\n\n"); - print_options(sampling_options); - printf("\n\n----- speculative params -----\n\n"); - print_options(spec_options); + print_options(sparam_options); // TODO: maybe convert enum llama_example to string printf("\n\n----- example-specific params -----\n\n"); print_options(specific_options); @@ -711,15 +676,12 @@ static void common_params_print_usage(common_params_context & ctx_arg) { static void common_params_print_completion(common_params_context & ctx_arg) { std::vector common_options; - std::vector sampling_options; - std::vector spec_options; + std::vector sparam_options; std::vector specific_options; for (auto & opt : ctx_arg.options) { - if (opt.is_sampling) { - sampling_options.push_back(&opt); - } else if (opt.is_spec) { - spec_options.push_back(&opt); + if (opt.is_sparam) { + sparam_options.push_back(&opt); } else if (opt.in_example(ctx_arg.ex)) { specific_options.push_back(&opt); } else { @@ -743,8 +705,7 @@ static void common_params_print_completion(common_params_context & ctx_arg) { }; print_options(common_options); - print_options(sampling_options); - print_options(spec_options); + print_options(sparam_options); print_options(specific_options); printf("\"\n\n"); @@ -830,7 +791,6 @@ static std::vector parse_device_list(const std::string & val if (dev_names.size() == 1 && dev_names[0] == "none") { devices.push_back(nullptr); } else { - ggml_backend_load_all(); for (const auto & device : dev_names) { auto * dev = ggml_backend_dev_by_name(device.c_str()); if (!dev || ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) { @@ -848,7 +808,6 @@ static void add_rpc_devices(const std::string & servers) { if (rpc_servers.empty()) { throw std::invalid_argument("no RPC servers specified"); } - ggml_backend_load_all(); ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); if (!rpc_reg) { throw std::invalid_argument("failed to find RPC backend"); @@ -900,11 +859,7 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map devices; for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { auto * dev = ggml_backend_dev_get(i); @@ -2328,6 +2285,12 @@ common_params_context common_params_parser_init(common_params & params, llama_ex parse_tensor_buffer_overrides(value, params.tensor_buft_overrides); } ).set_env("LLAMA_ARG_OVERRIDE_TENSOR")); + add_opt(common_arg( + {"-otd", "--override-tensor-draft"}, "=,...", + "override tensor buffer type for draft model", [](common_params & params, const std::string & value) { + parse_tensor_buffer_overrides(value, params.speculative.tensor_buft_overrides); + } + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( {"-cmoe", "--cpu-moe"}, "keep all Mixture of Experts (MoE) weights in the CPU", @@ -2350,6 +2313,27 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } } ).set_env("LLAMA_ARG_N_CPU_MOE")); + add_opt(common_arg( + {"-cmoed", "--cpu-moe-draft"}, + "keep all Mixture of Experts (MoE) weights in the CPU for the draft model", + [](common_params & params) { + params.speculative.tensor_buft_overrides.push_back(llm_ffn_exps_cpu_override()); + } + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_CPU_MOE_DRAFT")); + add_opt(common_arg( + {"-ncmoed", "--n-cpu-moe-draft"}, "N", + "keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model", + [](common_params & params, int value) { + if (value < 0) { + throw std::invalid_argument("invalid value"); + } + for (int i = 0; i < value; ++i) { + static std::list buft_overrides_draft; + buft_overrides_draft.push_back(llm_ffn_exps_block_regex(i)); + params.speculative.tensor_buft_overrides.push_back({buft_overrides_draft.back().c_str(), ggml_backend_cpu_buffer_type()}); + } + } + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_N_CPU_MOE_DRAFT")); GGML_ASSERT(params.n_gpu_layers < 0); // string_format would need to be extended for a default >= 0 add_opt(common_arg( {"-ngl", "--gpu-layers", "--n-gpu-layers"}, "N", @@ -2444,20 +2428,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } } ).set_env("LLAMA_ARG_FIT")); - add_opt(common_arg( - { "-fitp", "--fit-print" }, "[on|off]", - string_format("print the estimated required memory ('on' or 'off', default: '%s')", params.fit_params_print ? "on" : "off"), - [](common_params & params, const std::string & value) { - if (is_truthy(value)) { - params.fit_params_print = true; - } else if (is_falsey(value)) { - params.fit_params_print = false; - } else { - throw std::runtime_error( - string_format("error: unknown value for --fit-print: '%s'\n", value.c_str())); - } - } - ).set_examples({LLAMA_EXAMPLE_FIT_PARAMS}).set_env("LLAMA_ARG_FIT_ESTIMATE")); add_opt(common_arg( { "-fitt", "--fit-target" }, "MiB0,MiB1,MiB2,...", string_format("target margin per device for --fit, comma-separated list of values, " @@ -2632,6 +2602,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.model.hf_repo = value; } ).set_env("LLAMA_ARG_HF_REPO")); + add_opt(common_arg( + {"-hfd", "-hfrd", "--hf-repo-draft"}, "/[:quant]", + "Same as --hf-repo, but for the draft model (default: unused)", + [](common_params & params, const std::string & value) { + params.speculative.mparams_dft.hf_repo = value; + } + ).set_env("LLAMA_ARG_HFD_REPO")); add_opt(common_arg( {"-hff", "--hf-file"}, "FILE", "Hugging Face model file. If specified, it will override the quant in --hf-repo (default: unused)", @@ -2812,7 +2789,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, int value) { params.embd_normalize = value; } - ).set_examples({LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_DEBUG})); + ).set_examples({LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_DEBUG})); add_opt(common_arg( {"--embd-output-format"}, "FORMAT", "empty = default, \"array\" = [[],[]...], \"json\" = openai style, \"json+\" = same \"json\" + cosine similarity matrix, \"raw\" = plain whitespace-delimited output (one embedding per line)", @@ -2869,93 +2846,45 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.api_prefix = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_API_PREFIX")); - // Deprecated: use --ui-config instead (kept for backward compat) add_opt(common_arg( {"--webui-config"}, "JSON", - "[DEPRECATED: use --ui-config] JSON that provides default WebUI settings (overrides WebUI defaults)", + "JSON that provides default WebUI settings (overrides WebUI defaults)", [](common_params & params, const std::string & value) { - params.ui_config_json = value; params.webui_config_json = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI_CONFIG")); - - add_opt(common_arg( - {"--ui-config"}, "JSON", - "JSON that provides default UI settings (overrides UI defaults)", - [](common_params & params, const std::string & value) { - params.ui_config_json = value; - params.webui_config_json = value; - } - ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_CONFIG")); - - // Deprecated: use --ui-config-file instead (kept for backward compat) add_opt(common_arg( {"--webui-config-file"}, "PATH", - "[DEPRECATED: use --ui-config-file] JSON file that provides default WebUI settings (overrides WebUI defaults)", + "JSON file that provides default WebUI settings (overrides WebUI defaults)", [](common_params & params, const std::string & value) { - params.ui_config_json = read_file(value); - params.webui_config_json = params.ui_config_json; + params.webui_config_json = read_file(value); } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI_CONFIG_FILE")); - - add_opt(common_arg( - {"--ui-config-file"}, "PATH", - "JSON file that provides default UI settings (overrides UI defaults)", - [](common_params & params, const std::string & value) { - params.ui_config_json = read_file(value); - params.webui_config_json = params.ui_config_json; - } - ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_CONFIG_FILE")); - - // Deprecated: use --ui-mcp-proxy instead (kept for backward compat) add_opt(common_arg( {"--webui-mcp-proxy"}, {"--no-webui-mcp-proxy"}, - "[DEPRECATED: use --ui-mcp-proxy/--no-ui-mcp-proxy] experimental: whether to enable MCP CORS proxy", + string_format("experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: %s)", params.webui_mcp_proxy ? "enabled" : "disabled"), [](common_params & params, bool value) { - params.ui_mcp_proxy = value; params.webui_mcp_proxy = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI_MCP_PROXY")); - - add_opt(common_arg( - {"--ui-mcp-proxy"}, - {"--no-ui-mcp-proxy"}, - "experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)", - [](common_params & params, bool value) { - params.ui_mcp_proxy = value; - params.webui_mcp_proxy = value; - } - ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_MCP_PROXY")); add_opt(common_arg( {"--tools"}, "TOOL1,TOOL2,...", "experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n" "specify \"all\" to enable all tools\n" - "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, apply_diff, get_datetime", + "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, apply_diff", [](common_params & params, const std::string & value) { params.server_tools = parse_csv_row(value); } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS")); - // Deprecated: use --ui/--no-ui instead (kept for backward compat) add_opt(common_arg( {"--webui"}, {"--no-webui"}, - "[DEPRECATED: use --ui/--no-ui] whether to enable the Web UI", + string_format("whether to enable the Web UI (default: %s)", params.webui ? "enabled" : "disabled"), [](common_params & params, bool value) { - params.ui = value; params.webui = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI")); - - add_opt(common_arg( - {"--ui"}, - {"--no-ui"}, - string_format("whether to enable the Web UI (default: %s)", params.ui ? "enabled" : "disabled"), - [](common_params & params, bool value) { - params.ui = value; - params.webui = value; - } - ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI")); add_opt(common_arg( {"--embedding", "--embeddings"}, string_format("restrict to only support embedding use case; use only with dedicated embedding models (default: %s)", params.embedding ? "enabled" : "disabled"), @@ -3181,14 +3110,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex "token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)", [](common_params & params, int value) { if (value < -1) { throw std::invalid_argument("invalid value"); } - params.sampling.reasoning_budget_tokens = value; + params.reasoning_budget = value; } ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_THINK_BUDGET")); add_opt(common_arg( {"--reasoning-budget-message"}, "MESSAGE", "message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)", [](common_params & params, const std::string & value) { - params.sampling.reasoning_budget_message = value; + params.reasoning_budget_message = value; } ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_THINK_BUDGET_MESSAGE")); add_opt(common_arg( @@ -3367,8 +3296,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex " - 1: error\n" " - 2: warning\n" " - 3: info\n" - " - 4: trace (more info)\n" - " - 5: debug\n" + " - 4: debug\n" "(default: %d)\n", params.verbosity), [](common_params & params, int value) { params.verbosity = value; @@ -3377,252 +3305,183 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_env("LLAMA_LOG_VERBOSITY")); add_opt(common_arg( {"--log-prefix"}, - {"--no-log-prefix"}, "Enable prefix in log messages", - [](common_params &, bool value) { - common_log_set_prefix(common_log_main(), value); + [](common_params &) { + common_log_set_prefix(common_log_main(), true); } - ).set_env("LLAMA_ARG_LOG_PREFIX")); + ).set_env("LLAMA_LOG_PREFIX")); add_opt(common_arg( {"--log-timestamps"}, - {"--no-log-timestamps"}, "Enable timestamps in log messages", - [](common_params &, bool value) { - common_log_set_timestamps(common_log_main(), value); + [](common_params &) { + common_log_set_timestamps(common_log_main(), true); } - ).set_env("LLAMA_ARG_LOG_TIMESTAMPS")); + ).set_env("LLAMA_LOG_TIMESTAMPS")); - // // speculative parameters - // - add_opt(common_arg( - {"--spec-draft-hf", "-hfd", "-hfrd", "--hf-repo-draft"}, "/[:quant]", - "Same as --hf-repo, but for the draft model (default: unused)", - [](common_params & params, const std::string & value) { - params.speculative.draft.mparams.hf_repo = value; - } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_HF_REPO")); - add_opt(common_arg( - {"--spec-draft-threads", "-td", "--threads-draft"}, "N", + {"-td", "--threads-draft"}, "N", "number of threads to use during generation (default: same as --threads)", [](common_params & params, int value) { - params.speculative.draft.cpuparams.n_threads = value; - if (params.speculative.draft.cpuparams.n_threads <= 0) { - params.speculative.draft.cpuparams.n_threads = std::thread::hardware_concurrency(); + params.speculative.cpuparams.n_threads = value; + if (params.speculative.cpuparams.n_threads <= 0) { + params.speculative.cpuparams.n_threads = std::thread::hardware_concurrency(); } } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( - {"--spec-draft-threads-batch", "-tbd", "--threads-batch-draft"}, "N", + {"-tbd", "--threads-batch-draft"}, "N", "number of threads to use during batch and prompt processing (default: same as --threads-draft)", [](common_params & params, int value) { - params.speculative.draft.cpuparams_batch.n_threads = value; - if (params.speculative.draft.cpuparams_batch.n_threads <= 0) { - params.speculative.draft.cpuparams_batch.n_threads = std::thread::hardware_concurrency(); + params.speculative.cpuparams_batch.n_threads = value; + if (params.speculative.cpuparams_batch.n_threads <= 0) { + params.speculative.cpuparams_batch.n_threads = std::thread::hardware_concurrency(); } } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( - {"--spec-draft-cpu-mask", "-Cd", "--cpu-mask-draft"}, "M", + {"-Cd", "--cpu-mask-draft"}, "M", "Draft model CPU affinity mask. Complements cpu-range-draft (default: same as --cpu-mask)", [](common_params & params, const std::string & mask) { - params.speculative.draft.cpuparams.mask_valid = true; - if (!parse_cpu_mask(mask, params.speculative.draft.cpuparams.cpumask)) { + params.speculative.cpuparams.mask_valid = true; + if (!parse_cpu_mask(mask, params.speculative.cpuparams.cpumask)) { throw std::invalid_argument("invalid cpumask"); } } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE})); add_opt(common_arg( - {"--spec-draft-cpu-range", "-Crd", "--cpu-range-draft"}, "lo-hi", + {"-Crd", "--cpu-range-draft"}, "lo-hi", "Ranges of CPUs for affinity. Complements --cpu-mask-draft", [](common_params & params, const std::string & range) { - params.speculative.draft.cpuparams.mask_valid = true; - if (!parse_cpu_range(range, params.speculative.draft.cpuparams.cpumask)) { + params.speculative.cpuparams.mask_valid = true; + if (!parse_cpu_range(range, params.speculative.cpuparams.cpumask)) { throw std::invalid_argument("invalid range"); } } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE})); add_opt(common_arg( - {"--spec-draft-cpu-strict", "--cpu-strict-draft"}, "<0|1>", + {"--cpu-strict-draft"}, "<0|1>", "Use strict CPU placement for draft model (default: same as --cpu-strict)", [](common_params & params, int value) { - params.speculative.draft.cpuparams.strict_cpu = value; + params.speculative.cpuparams.strict_cpu = value; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE})); add_opt(common_arg( - {"--spec-draft-prio", "--prio-draft"}, "N", - string_format("set draft process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.speculative.draft.cpuparams.priority), + {"--prio-draft"}, "N", + string_format("set draft process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.speculative.cpuparams.priority), [](common_params & params, int prio) { if (prio < 0 || prio > 3) { throw std::invalid_argument("invalid value"); } - params.speculative.draft.cpuparams.priority = (enum ggml_sched_priority) prio; + params.speculative.cpuparams.priority = (enum ggml_sched_priority) prio; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE})); add_opt(common_arg( - {"--spec-draft-poll", "--poll-draft"}, "<0|1>", - "Use polling to wait for draft model work (default: same as --poll)", + {"--poll-draft"}, "<0|1>", + "Use polling to wait for draft model work (default: same as --poll])", [](common_params & params, int value) { - params.speculative.draft.cpuparams.poll = value; + params.speculative.cpuparams.poll = value; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE})); add_opt(common_arg( - {"--spec-draft-cpu-mask-batch", "-Cbd", "--cpu-mask-batch-draft"}, "M", + {"-Cbd", "--cpu-mask-batch-draft"}, "M", "Draft model CPU affinity mask. Complements cpu-range-draft (default: same as --cpu-mask)", [](common_params & params, const std::string & mask) { - params.speculative.draft.cpuparams_batch.mask_valid = true; - if (!parse_cpu_mask(mask, params.speculative.draft.cpuparams_batch.cpumask)) { + params.speculative.cpuparams_batch.mask_valid = true; + if (!parse_cpu_mask(mask, params.speculative.cpuparams_batch.cpumask)) { throw std::invalid_argument("invalid cpumask"); } } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE})); add_opt(common_arg( - {"--spec-draft-cpu-range-batch", "-Crbd", "--cpu-range-batch-draft"}, "lo-hi", + {"-Crbd", "--cpu-range-batch-draft"}, "lo-hi", "Ranges of CPUs for affinity. Complements --cpu-mask-draft-batch)", [](common_params & params, const std::string & range) { - params.speculative.draft.cpuparams_batch.mask_valid = true; - if (!parse_cpu_range(range, params.speculative.draft.cpuparams_batch.cpumask)) { + params.speculative.cpuparams_batch.mask_valid = true; + if (!parse_cpu_range(range, params.speculative.cpuparams_batch.cpumask)) { throw std::invalid_argument("invalid cpumask"); } } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE})); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE})); add_opt(common_arg( - {"--spec-draft-cpu-strict-batch", "--cpu-strict-batch-draft"}, "<0|1>", + {"--cpu-strict-batch-draft"}, "<0|1>", "Use strict CPU placement for draft model (default: --cpu-strict-draft)", [](common_params & params, int value) { - params.speculative.draft.cpuparams_batch.strict_cpu = value; + params.speculative.cpuparams_batch.strict_cpu = value; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE})); add_opt(common_arg( - {"--spec-draft-prio-batch", "--prio-batch-draft"}, "N", - string_format("set draft process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.speculative.draft.cpuparams_batch.priority), + {"--prio-batch-draft"}, "N", + string_format("set draft process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.speculative.cpuparams_batch.priority), [](common_params & params, int prio) { if (prio < 0 || prio > 3) { throw std::invalid_argument("invalid value"); } - params.speculative.draft.cpuparams_batch.priority = (enum ggml_sched_priority) prio; + params.speculative.cpuparams_batch.priority = (enum ggml_sched_priority) prio; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE})); add_opt(common_arg( - {"--spec-draft-poll-batch", "--poll-batch-draft"}, "<0|1>", + {"--poll-batch-draft"}, "<0|1>", "Use polling to wait for draft model work (default: --poll-draft)", [](common_params & params, int value) { - params.speculative.draft.cpuparams_batch.poll = value; - } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); - add_opt(common_arg( - {"--spec-draft-type-k", "-ctkd", "--cache-type-k-draft"}, "TYPE", - string_format( - "KV cache data type for K for the draft model\n" - "allowed values: %s\n" - "(default: %s)", - get_all_kv_cache_types().c_str(), - ggml_type_name(params.speculative.draft.cache_type_k) - ), - [](common_params & params, const std::string & value) { - params.speculative.draft.cache_type_k = kv_cache_type_from_str(value); + params.speculative.cpuparams_batch.poll = value; } - ).set_env("LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K")); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE})); add_opt(common_arg( - {"--spec-draft-type-v", "-ctvd", "--cache-type-v-draft"}, "TYPE", - string_format( - "KV cache data type for V for the draft model\n" - "allowed values: %s\n" - "(default: %s)", - get_all_kv_cache_types().c_str(), - ggml_type_name(params.speculative.draft.cache_type_v) - ), - [](common_params & params, const std::string & value) { - params.speculative.draft.cache_type_v = kv_cache_type_from_str(value); - } - ).set_env("LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V")); - add_opt(common_arg( - {"--spec-draft-override-tensor", "-otd", "--override-tensor-draft"}, "=,...", - "override tensor buffer type for draft model", [](common_params & params, const std::string & value) { - parse_tensor_buffer_overrides(value, params.speculative.draft.tensor_buft_overrides); - } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); - add_opt(common_arg( - {"--spec-draft-cpu-moe", "-cmoed", "--cpu-moe-draft"}, - "keep all Mixture of Experts (MoE) weights in the CPU for the draft model", - [](common_params & params) { - params.speculative.draft.tensor_buft_overrides.push_back(llm_ffn_exps_cpu_override()); - } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_CPU_MOE")); - add_opt(common_arg( - {"--spec-draft-n-cpu-moe", "--spec-draft-ncmoe", "-ncmoed", "--n-cpu-moe-draft"}, "N", - "keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model", - [](common_params & params, int value) { - if (value < 0) { - throw std::invalid_argument("invalid value"); - } - for (int i = 0; i < value; ++i) { - static std::list buft_overrides_draft; - buft_overrides_draft.push_back(llm_ffn_exps_block_regex(i)); - params.speculative.draft.tensor_buft_overrides.push_back({buft_overrides_draft.back().c_str(), ggml_backend_cpu_buffer_type()}); - } - } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE")); - - add_opt(common_arg( - {"--spec-draft-n-max"}, "N", - string_format("number of tokens to draft for speculative decoding (default: %d)", params.speculative.draft.n_max), + {"--draft", "--draft-n", "--draft-max"}, "N", + string_format("number of tokens to draft for speculative decoding (default: %d)", params.speculative.n_max), [](common_params & params, int value) { - params.speculative.draft.n_max = value; + params.speculative.n_max = value; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MAX")); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_DRAFT_MAX")); add_opt(common_arg( - {"--spec-draft-n-min"}, "N", - string_format("minimum number of draft tokens to use for speculative decoding (default: %d)", params.speculative.draft.n_min), + {"--draft-min", "--draft-n-min"}, "N", + string_format("minimum number of draft tokens to use for speculative decoding (default: %d)", params.speculative.n_min), [](common_params & params, int value) { - params.speculative.draft.n_min = value; + params.speculative.n_min = value; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN")); - + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_DRAFT_MIN")); add_opt(common_arg( - {"--spec-draft-p-split", "--draft-p-split"}, "P", - string_format("speculative decoding split probability (default: %.2f)", (double)params.speculative.draft.p_split), + {"--draft-p-split"}, "P", + string_format("speculative decoding split probability (default: %.2f)", (double)params.speculative.p_split), [](common_params & params, const std::string & value) { - params.speculative.draft.p_split = std::stof(value); + params.speculative.p_split = std::stof(value); } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_P_SPLIT")); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE}).set_env("LLAMA_ARG_DRAFT_P_SPLIT")); add_opt(common_arg( - {"--spec-draft-p-min", "--draft-p-min"}, "P", - string_format("minimum speculative decoding probability (greedy) (default: %.2f)", (double)params.speculative.draft.p_min), + {"--draft-p-min"}, "P", + string_format("minimum speculative decoding probability (greedy) (default: %.2f)", (double)params.speculative.p_min), [](common_params & params, const std::string & value) { - params.speculative.draft.p_min = std::stof(value); + params.speculative.p_min = std::stof(value); } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_P_MIN")); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_DRAFT_P_MIN")); add_opt(common_arg( - {"--spec-draft-backend-sampling"}, - {"--no-spec-draft-backend-sampling"}, - string_format("offload draft sampling to the backend (default: %s)", - params.speculative.draft.backend_sampling ? "enabled" : "disabled"), - [](common_params & params, bool value) { - params.speculative.draft.backend_sampling = value; + {"-cd", "--ctx-size-draft"}, "N", + string_format("size of the prompt context for the draft model (default: %d, 0 = loaded from model)", params.speculative.n_ctx), + [](common_params & params, int value) { + params.speculative.n_ctx = value; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING")); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_CTX_SIZE_DRAFT")); add_opt(common_arg( - {"--spec-draft-device", "-devd", "--device-draft"}, "", + {"-devd", "--device-draft"}, "", "comma-separated list of devices to use for offloading the draft model (none = don't offload)\n" "use --list-devices to see a list of available devices", [](common_params & params, const std::string & value) { - params.speculative.draft.devices = parse_device_list(value); + params.speculative.devices = parse_device_list(value); } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); - GGML_ASSERT(params.speculative.draft.n_gpu_layers < 0); // string_format would need to be extended for a default >= 0 + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + GGML_ASSERT(params.speculative.n_gpu_layers < 0); // string_format would need to be extended for a default >= 0 add_opt(common_arg( - {"--spec-draft-ngl", "-ngld", "--gpu-layers-draft", "--n-gpu-layers-draft"}, "N", + {"-ngld", "--gpu-layers-draft", "--n-gpu-layers-draft"}, "N", string_format("max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: %s)", - params.speculative.draft.n_gpu_layers == -1 ? "auto" : "all"), + params.speculative.n_gpu_layers == -1 ? "auto" : "all"), [](common_params & params, const std::string & value) { if (value == "auto") { - params.speculative.draft.n_gpu_layers = -1; + params.speculative.n_gpu_layers = -1; } else if (value == "all") { - params.speculative.draft.n_gpu_layers = -2; + params.speculative.n_gpu_layers = -2; } else { - params.speculative.draft.n_gpu_layers = std::stoi(value); + params.speculative.n_gpu_layers = std::stoi(value); } if (!llama_supports_gpu_offload()) { fprintf(stderr, "warning: no usable GPU found, --gpu-layers-draft option will be ignored\n"); @@ -3630,191 +3489,122 @@ common_params_context common_params_parser_init(common_params & params, llama_ex fprintf(stderr, "warning: consult docs/build.md for compilation instructions\n"); } } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_N_GPU_LAYERS_DRAFT")); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_N_GPU_LAYERS_DRAFT")); add_opt(common_arg( - {"--spec-draft-model", "-md", "--model-draft"}, "FNAME", - "draft model for speculative decoding (default: unused)", + {"-md", "--model-draft"}, "FNAME", + "draft model for speculative decoding, or Gemma 4 MTP assistant GGUF when using --spec-type mtp (default: unused)", [](common_params & params, const std::string & value) { - params.speculative.draft.mparams.path = value; + params.speculative.mparams_dft.path = value; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_MODEL")); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_MODEL_DRAFT")); add_opt(common_arg( - {"--spec-type"}, common_speculative_all_types_str(), - string_format("comma-separated list of types of speculative decoding to use (default: %s)\n", - common_speculative_type_name_str(params.speculative.types).c_str()), + {"--mtp-head"}, "FNAME", + "alias for Gemma 4 MTP: path to gemma4_assistant GGUF (loaded into the target; use with --spec-type mtp)", [](common_params & params, const std::string & value) { - const auto types_str = string_split(value, ','); - auto types = common_speculative_types_from_names(types_str); - params.speculative.types.insert(params.speculative.types.end(), types.begin(), types.end()); - } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_TYPE")); - add_opt(common_arg( - {"--spec-ngram-mod-n-min"}, "N", - string_format("minimum number of ngram tokens to use for ngram-based speculative decoding (default: %d)", params.speculative.ngram_mod.n_min), - [](common_params & params, int value) { - if (value < 0 || value > 1024) { - throw std::invalid_argument("ngram n-min must be between 0 and 1024 inclusive"); - } - params.speculative.ngram_mod.n_min = value; - } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); - add_opt(common_arg( - {"--spec-ngram-mod-n-max"}, "N", - string_format("maximum number of ngram tokens to use for ngram-based speculative decoding (default: %d)", params.speculative.ngram_mod.n_max), - [](common_params & params, int value) { - if (value < 0 || value > 1024) { - throw std::invalid_argument("ngram n-max must be between 0 and 1024 inclusive"); - } - params.speculative.ngram_mod.n_max = value; - } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); - add_opt(common_arg( - {"--spec-ngram-mod-n-match"}, "N", - string_format("ngram-mod lookup length (default: %d)", params.speculative.ngram_mod.n_match), - [](common_params & params, int value) { - if (value < 1 || value > 1024) { - throw std::invalid_argument("ngram size N must be between 1 and 1024 inclusive"); - } - params.speculative.ngram_mod.n_match = value; - } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); - - add_opt(common_arg( - {"--spec-ngram-simple-size-n"}, "N", - string_format("ngram size N for ngram-simple speculative decoding, length of lookup n-gram (default: %d)", params.speculative.ngram_simple.size_n), - [](common_params & params, int value) { - if (value < 1 || value > 1024) { - throw std::invalid_argument("ngram size N must be between 1 and 1024 inclusive"); - } - params.speculative.ngram_simple.size_n = value; + params.speculative.mparams_dft.path = value; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); - add_opt(common_arg( - {"--spec-ngram-simple-size-m"}, "N", - string_format("ngram size M for ngram-simple speculative decoding, length of draft m-gram (default: %d)", params.speculative.ngram_simple.size_m), - [](common_params & params, int value) { - if (value < 1 || value > 1024) { - throw std::invalid_argument("ngram size M must be between 1 and 1024 inclusive"); - } - params.speculative.ngram_simple.size_m = value; - } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); - add_opt(common_arg( - {"--spec-ngram-simple-min-hits"}, "N", - string_format("minimum hits for ngram-simple speculative decoding (default: %d)", params.speculative.ngram_simple.min_hits), - [](common_params & params, int value) { - if (value < 1) { - throw std::invalid_argument("ngram min hits must be at least 1"); - } - params.speculative.ngram_simple.min_hits = value; - } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); - + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_MTP_HEAD")); add_opt(common_arg( - {"--spec-ngram-map-k-size-n"}, "N", - string_format("ngram size N for ngram-map-k speculative decoding, length of lookup n-gram (default: %d)", params.speculative.ngram_map_k.size_n), - [](common_params & params, int value) { - if (value < 1 || value > 1024) { - throw std::invalid_argument("ngram size N must be between 1 and 1024 inclusive"); - } - params.speculative.ngram_map_k.size_n = value; + {"--spec-replace"}, "TARGET", "DRAFT", + "translate the string in TARGET into DRAFT if the draft model and main model are not compatible", + [](common_params & params, const std::string & tgt, const std::string & dft) { + params.speculative.replacements.push_back({ tgt, dft }); } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( - {"--spec-ngram-map-k-size-m"}, "N", - string_format("ngram size M for ngram-map-k speculative decoding, length of draft m-gram (default: %d)", params.speculative.ngram_map_k.size_m), - [](common_params & params, int value) { - if (value < 1 || value > 1024) { - throw std::invalid_argument("ngram size M must be between 1 and 1024 inclusive"); + {"--spec-type"}, "[none|draft|mtp|eagle3|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod]", + string_format("type of speculative decoding (default: %s). For Gemma 4 MTP use --spec-type mtp and --mtp-head (or --model-draft) pointing at gemma4_assistant GGUF.\n", + common_speculative_type_to_str(params.speculative.type).c_str()), + [](common_params & params, const std::string & value) { + if (value == "none") { + params.speculative.type = COMMON_SPECULATIVE_TYPE_NONE; + } else if (value == "draft") { + params.speculative.type = COMMON_SPECULATIVE_TYPE_DRAFT; + } else if (value == "mtp") { + params.speculative.type = COMMON_SPECULATIVE_TYPE_MTP; + } else if (value == "eagle3") { + params.speculative.type = COMMON_SPECULATIVE_TYPE_EAGLE3; + } else if (value == "ngram-cache") { + params.speculative.type = COMMON_SPECULATIVE_TYPE_NGRAM_CACHE; + } else if (value == "ngram-simple") { + params.speculative.type = COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE; + } else if (value == "ngram-map-k") { + params.speculative.type = COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K; + } else if (value == "ngram-map-k4v") { + params.speculative.type = COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V; + } else if (value == "ngram-mod") { + params.speculative.type = COMMON_SPECULATIVE_TYPE_NGRAM_MOD; + } else { + throw std::invalid_argument("unknown speculative decoding type"); } - params.speculative.ngram_map_k.size_m = value; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_SPEC_TYPE")); add_opt(common_arg( - {"--spec-ngram-map-k-min-hits"}, "N", - string_format("minimum hits for ngram-map-k speculative decoding (default: %d)", params.speculative.ngram_map_k.min_hits), + {"--draft-block-size"}, "N", + string_format("MTP draft block size B (drafts B-1 tokens per round; default: %d)", params.speculative.draft_block_size), [](common_params & params, int value) { - if (value < 1) { - throw std::invalid_argument("ngram min hits must be at least 1"); + if (value < 2 || value > 32) { + throw std::invalid_argument("draft block size must be between 2 and 32"); } - params.speculative.ngram_map_k.min_hits = value; + params.speculative.draft_block_size = value; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); - + ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_SPECULATIVE}).set_env("LLAMA_ARG_DRAFT_BLOCK_SIZE")); add_opt(common_arg( - {"--spec-ngram-map-k4v-size-n"}, "N", - string_format("ngram size N for ngram-map-k4v speculative decoding, length of lookup n-gram (default: %d)", params.speculative.ngram_map_k4v.size_n), + {"--spec-ngram-size-n"}, "N", + string_format("ngram size N for ngram-simple/ngram-map speculative decoding, length of lookup n-gram (default: %d)", params.speculative.ngram_size_n), [](common_params & params, int value) { if (value < 1 || value > 1024) { throw std::invalid_argument("ngram size N must be between 1 and 1024 inclusive"); } - params.speculative.ngram_map_k4v.size_n = value; + params.speculative.ngram_size_n = value; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( - {"--spec-ngram-map-k4v-size-m"}, "N", - string_format("ngram size M for ngram-map-k4v speculative decoding, length of draft m-gram (default: %d)", params.speculative.ngram_map_k4v.size_m), + {"--spec-ngram-size-m"}, "N", + string_format("ngram size M for ngram-simple/ngram-map speculative decoding, length of draft m-gram (default: %d)", params.speculative.ngram_size_m), [](common_params & params, int value) { if (value < 1 || value > 1024) { throw std::invalid_argument("ngram size M must be between 1 and 1024 inclusive"); } - params.speculative.ngram_map_k4v.size_m = value; + params.speculative.ngram_size_m = value; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + ).set_examples({LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( - {"--spec-ngram-map-k4v-min-hits"}, "N", - string_format("minimum hits for ngram-map-k4v speculative decoding (default: %d)", params.speculative.ngram_map_k4v.min_hits), + {"--spec-ngram-min-hits"}, "N", + string_format("minimum hits for ngram-map speculative decoding (default: %d)", params.speculative.ngram_min_hits), [](common_params & params, int value) { if (value < 1) { throw std::invalid_argument("ngram min hits must be at least 1"); } - params.speculative.ngram_map_k4v.min_hits = value; - } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); - - // - // removed params - // - - add_opt(common_arg( - {"--draft", "--draft-n", "--draft-max"}, "N", - "the argument has been removed. use --spec-draft-n-max or --spec-ngram-mod-n-max", - [](common_params & /*params*/, int /*value*/) { - arg_removed("use --spec-draft-n-max or --spec-ngram-mod-n-max"); - } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_DRAFT_MAX")); - add_opt(common_arg( - {"--draft-min", "--draft-n-min"}, "N", - "the argument has been removed. use --spec-draft-n-min or --spec-ngram-mod-n-min", - [](common_params & /*params*/, int /*value*/) { - arg_removed("use --spec-draft-n-min or --spec-ngram-mod-n-min"); - } - ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_DRAFT_MIN")); - add_opt(common_arg( - {"--spec-ngram-size-n"}, "N", - "the argument has been removed. use the respective --spec-ngram-*-size-n or --spec-ngram-mod-n-match", - [](common_params & /*params*/, int /*value*/) { - arg_removed("use the respective --spec-ngram-*-size-n"); + params.speculative.ngram_min_hits = value; } - ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER})); + ).set_examples({LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( - {"--spec-ngram-size-m"}, "N", - "the argument has been removed. use the respective --spec-ngram-*-size-m", - [](common_params & /*params*/, int /*value*/) { - arg_removed("use the respective --spec-ngram-*-size-m"); + {"-ctkd", "--cache-type-k-draft"}, "TYPE", + string_format( + "KV cache data type for K for the draft model\n" + "allowed values: %s\n" + "(default: %s)", + get_all_kv_cache_types().c_str(), + ggml_type_name(params.speculative.cache_type_k) + ), + [](common_params & params, const std::string & value) { + params.speculative.cache_type_k = kv_cache_type_from_str(value); } - ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER})); + ).set_env("LLAMA_ARG_CACHE_TYPE_K_DRAFT")); add_opt(common_arg( - {"--spec-ngram-min-hits"}, "N", - "the argument has been removed. use the respective --spec-ngram-*-min-hits", - [](common_params & /*params*/, int /*value*/) { - arg_removed("use the respective --spec-ngram-*-min-hits"); + {"-ctvd", "--cache-type-v-draft"}, "TYPE", + string_format( + "KV cache data type for V for the draft model\n" + "allowed values: %s\n" + "(default: %s)", + get_all_kv_cache_types().c_str(), + ggml_type_name(params.speculative.cache_type_v) + ), + [](common_params & params, const std::string & value) { + params.speculative.cache_type_v = kv_cache_type_from_str(value); } - ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER})); - - // - // TTS params - // + ).set_env("LLAMA_ARG_CACHE_TYPE_V_DRAFT")); add_opt(common_arg( {"-mv", "--model-vocoder"}, "FNAME", @@ -3838,10 +3628,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } ).set_examples({LLAMA_EXAMPLE_TTS})); - // - // diffusion params - // - add_opt(common_arg( {"--diffusion-steps"}, "N", string_format("number of diffusion steps (default: %d)", params.diffusion.steps), @@ -3859,10 +3645,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_examples({ LLAMA_EXAMPLE_DIFFUSION })); add_opt(common_arg( {"--diffusion-algorithm"}, "N", - string_format( - "diffusion algorithm: 0=DIFFUSION_ALGORITHM_ORIGIN, 1=DIFFUSION_ALGORITHM_ENTROPY_BASED, " - "2=DIFFUSION_ALGORITHM_MARGIN_BASED, 3=DIFFUSION_ALGORITHM_RANDOM, " - "4=DIFFUSION_ALGORITHM_CONFIDENCE_BASED (default: %d)", params.diffusion.algorithm), + string_format("diffusion algorithm: 0=ORIGIN, 1=ENTROPY_BASED, 2=MARGIN_BASED, 3=RANDOM, 4=LOW_CONFIDENCE (default: %d)", params.diffusion.algorithm), [](common_params & params, int value) { params.diffusion.algorithm = value; } ).set_examples({ LLAMA_EXAMPLE_DIFFUSION })); add_opt(common_arg( @@ -4029,8 +3812,8 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params) { params.model.hf_repo = "ggml-org/Qwen2.5-Coder-7B-Q8_0-GGUF"; params.model.hf_file = "qwen2.5-coder-7b-q8_0.gguf"; - params.speculative.draft.mparams.hf_repo = "ggml-org/Qwen2.5-Coder-0.5B-Q8_0-GGUF"; - params.speculative.draft.mparams.hf_file = "qwen2.5-coder-0.5b-q8_0.gguf"; + params.speculative.mparams_dft.hf_repo = "ggml-org/Qwen2.5-Coder-0.5B-Q8_0-GGUF"; + params.speculative.mparams_dft.hf_file = "qwen2.5-coder-0.5b-q8_0.gguf"; params.port = 8012; params.n_ubatch = 1024; params.n_batch = 1024; @@ -4045,8 +3828,8 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params) { params.model.hf_repo = "ggml-org/Qwen2.5-Coder-14B-Q8_0-GGUF"; params.model.hf_file = "qwen2.5-coder-14b-q8_0.gguf"; - params.speculative.draft.mparams.hf_repo = "ggml-org/Qwen2.5-Coder-0.5B-Q8_0-GGUF"; - params.speculative.draft.mparams.hf_file = "qwen2.5-coder-0.5b-q8_0.gguf"; + params.speculative.mparams_dft.hf_repo = "ggml-org/Qwen2.5-Coder-0.5B-Q8_0-GGUF"; + params.speculative.mparams_dft.hf_file = "qwen2.5-coder-0.5b-q8_0.gguf"; params.port = 8012; params.n_ubatch = 1024; params.n_batch = 1024; @@ -4130,23 +3913,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); - add_opt(common_arg( - {"--spec-default"}, - string_format("enable default speculative decoding config"), - [](common_params & params) { - params.speculative.types.push_back(COMMON_SPECULATIVE_TYPE_NGRAM_MOD); - params.speculative.ngram_mod.n_match = 24; - params.speculative.ngram_mod.n_min = 48; - params.speculative.ngram_mod.n_max = 64; - - // TODO: not sure if this is a good config - explore more settings and potentially enable it - //params.speculative.types.push_back(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V); - //params.speculative.ngram_map_k4v.size_n = 8; - //params.speculative.ngram_map_k4v.size_m = 24; - //params.speculative.ngram_map_k4v.min_hits = 2; - } - ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); - return ctx_arg; } diff --git a/common/common.cpp b/common/common.cpp index 2ee73f2ddce9..c520f9405e5e 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1214,6 +1214,43 @@ common_init_result::common_init_result(common_params & params, bool model_only) pimpl->lora.emplace_back(std::move(lora)); // copy to list of loaded adapters } + if (params.speculative.has_dft() && params.speculative.type == COMMON_SPECULATIVE_TYPE_MTP) { + if (params.speculative.mparams_dft.path.empty() && params.speculative.mparams_dft.hf_repo.empty()) { + LOG_ERR("%s: MTP speculative decoding requires --mtp-head or --model-draft with a local path\n", __func__); + pimpl->model.reset(); + return; + } + if (params.speculative.mparams_dft.path.empty()) { + LOG_ERR("%s: MTP assistant must be loaded from a local GGUF path (HF repo download not wired here)\n", __func__); + pimpl->model.reset(); + return; + } + + common_params p_mtp = params; + p_mtp.n_parallel = 1; + p_mtp.n_ctx = params.speculative.n_ctx > 0 ? params.speculative.n_ctx : params.n_ctx; + p_mtp.n_batch = std::max(p_mtp.n_ctx, params.n_batch); + p_mtp.devices = params.speculative.devices; + p_mtp.model = params.speculative.mparams_dft; + p_mtp.n_gpu_layers = params.speculative.n_gpu_layers; + p_mtp.cache_type_k = params.speculative.cache_type_k; + p_mtp.cache_type_v = params.speculative.cache_type_v; + if (params.speculative.cpuparams.n_threads > 0) { + p_mtp.cpuparams = params.speculative.cpuparams; + p_mtp.cpuparams_batch = params.speculative.cpuparams_batch; + } + p_mtp.tensor_buft_overrides = params.speculative.tensor_buft_overrides; + + llama_model_params mparams_mtp = common_model_params_to_llama(p_mtp); + const char * path_mtp = params.speculative.mparams_dft.path.c_str(); + if (llama_model_load_mtp_from_file(model, path_mtp, mparams_mtp) != 0) { + LOG_ERR("%s: failed to load MTP assistant from '%s'\n", __func__, path_mtp); + pimpl->model.reset(); + return; + } + params.speculative.model_dft = nullptr; + } + // updates params.sampling // TODO: fix naming common_init_sampler_from_model(model, params.sampling); diff --git a/common/common.h b/common/common.h index dec90456afab..d32d78e8fb11 100644 --- a/common/common.h +++ b/common/common.h @@ -2,18 +2,17 @@ #pragma once -#include "llama-cpp.h" - #include "ggml-opt.h" #include "ggml.h" +#include "llama-cpp.h" #include #include #include #include +#include #include #include -#include #if defined(_WIN32) && !defined(_WIN32_WINNT) #define _WIN32_WINNT 0x0A00 @@ -28,6 +27,11 @@ #define die(msg) do { fputs("error: " msg "\n", stderr); exit(1); } while (0) #define die_fmt(fmt, ...) do { fprintf(stderr, "error: " fmt "\n", __VA_ARGS__); exit(1); } while (0) +#define print_build_info() do { \ + fprintf(stderr, "%s: build = %d (%s)\n", __func__, LLAMA_BUILD_NUMBER, LLAMA_COMMIT); \ + fprintf(stderr, "%s: built with %s for %s\n", __func__, LLAMA_COMPILER, LLAMA_BUILD_TARGET); \ +} while(0) + struct common_time_meas { common_time_meas(int64_t & t_acc, bool disable = false); ~common_time_meas(); @@ -49,13 +53,21 @@ struct common_adapter_lora_info { using llama_tokens = std::vector; +// build info +extern int LLAMA_BUILD_NUMBER; +extern const char * LLAMA_COMMIT; +extern const char * LLAMA_COMPILER; +extern const char * LLAMA_BUILD_TARGET; + +const static std::string build_info("b" + std::to_string(LLAMA_BUILD_NUMBER) + "-" + LLAMA_COMMIT); + struct common_control_vector_load_info; // // CPU utils // -struct common_cpu_params { +struct cpu_params { int n_threads = -1; bool cpumask[GGML_MAX_N_THREADS] = {false}; // CPU affinity mask. bool mask_valid = false; // Default: any CPU @@ -64,8 +76,8 @@ struct common_cpu_params { uint32_t poll = 50; // Polling (busywait) level (0 - no polling, 100 - mostly polling) }; -int32_t common_cpu_get_num_physical_cores(); -int32_t common_cpu_get_num_math(); +int32_t cpu_get_num_physical_cores(); +int32_t cpu_get_num_math(); // // Common params @@ -158,14 +170,14 @@ enum common_params_sampling_config : uint64_t { enum common_speculative_type { COMMON_SPECULATIVE_TYPE_NONE, // no speculative decoding - COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding - COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding - COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction - COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams + COMMON_SPECULATIVE_TYPE_DRAFT, // draft model + COMMON_SPECULATIVE_TYPE_EAGLE3, // eagle draft model + COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values COMMON_SPECULATIVE_TYPE_NGRAM_MOD, COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache + COMMON_SPECULATIVE_TYPE_MTP, // Gemma 4 MTP assistant drafter COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type }; @@ -276,7 +288,6 @@ struct common_params_sampling { std::vector reasoning_budget_start; // start tag token sequence std::vector reasoning_budget_end; // end tag token sequence std::vector reasoning_budget_forced; // forced sequence (message + end tag) - std::string reasoning_budget_message; // message injected before end tag when budget exhausted bool backend_sampling = false; @@ -297,84 +308,65 @@ struct common_params_model { std::string name = ""; // in format /[:] (tag is optional) // NOLINT }; -// draft-model-based speculative decoding parameters -struct common_params_speculative_draft { - int32_t n_max = 3; // maximum number of tokens to draft during speculative decoding - int32_t n_min = 0; // minimum number of draft tokens to use for speculative decoding +struct common_ngram_mod; - float p_split = 0.1f; // speculative decoding split probability - float p_min = 0.0f; // minimum speculative decoding probability (greedy) +struct common_params_speculative { + common_speculative_type type = COMMON_SPECULATIVE_TYPE_NONE; // type of speculative decoding - bool backend_sampling = true; // offload draft sampling to the backend (default: on) + // general-purpose speculative decoding parameters - common_params_model mparams; + int32_t n_max = 16; // maximum number of tokens to draft during speculative decoding + int32_t n_min = 0; // minimum number of draft tokens to use for speculative decoding - llama_context * ctx_tgt = nullptr; - llama_context * ctx_dft = nullptr; + // MTP (Gemma 4 assistant): draft block size B produces B-1 draft tokens per round (default 4) + int32_t draft_block_size = 4; + float p_split = 0.1f; // speculative decoding split probability + float p_min = 0.75f; // minimum speculative decoding probability (greedy) - int32_t n_gpu_layers = -1; // number of layers to store in VRAM for the draft model (-1 - use default) + // ngram-based speculative decoding - ggml_type cache_type_k = GGML_TYPE_F16; // KV cache data type for the K - ggml_type cache_type_v = GGML_TYPE_F16; // KV cache data type for the V + uint16_t ngram_size_n = 12; // ngram size for lookup + uint16_t ngram_size_m = 48; // mgram size for speculative tokens + uint16_t ngram_min_hits = 1; // minimum hits at ngram/mgram lookup for mgram to be proposed - common_cpu_params cpuparams; - common_cpu_params cpuparams_batch; + std::shared_ptr ngram_mod; - std::vector devices; // devices to use for offloading + std::string lookup_cache_static; // path of static ngram cache file for lookup decoding // NOLINT + std::string lookup_cache_dynamic; // path of dynamic ngram cache file for lookup decoding // NOLINT - std::vector tensor_buft_overrides; -}; + // draft-model speculative decoding -struct common_params_speculative_ngram_mod { - int32_t n_match = 24; + struct common_params_model mparams_dft; - int32_t n_max = 64; - int32_t n_min = 48; -}; + llama_model * model_dft = nullptr; // a llama_model that can be shared by multiple speculative contexts -struct common_params_speculative_ngram_map { - uint16_t size_n = 12; // ngram size for lookup - uint16_t size_m = 48; // mgram size for speculative tokens - uint16_t min_hits = 1; // minimum hits at ngram/mgram lookup for mgram to be proposed -}; + llama_context_params cparams_dft; // these are the parameters for the draft llama_context -struct common_params_speculative_ngram_cache { - std::string lookup_cache_static; // path of static ngram cache file for lookup decoding - std::string lookup_cache_dynamic; // path of dynamic ngram cache file for lookup decoding -}; + int32_t n_ctx = 0; // draft context size + int32_t n_gpu_layers = -1; // number of layers to store in VRAM for the draft model (-1 - use default) -struct common_params_speculative { - std::vector types = { COMMON_SPECULATIVE_TYPE_NONE }; + ggml_type cache_type_k = GGML_TYPE_F16; // KV cache data type for the K + ggml_type cache_type_v = GGML_TYPE_F16; // KV cache data type for the V - // used by Simple, MTP, Eagle3, etc. - all methods that require some kind of draft model - common_params_speculative_draft draft; + struct cpu_params cpuparams; + struct cpu_params cpuparams_batch; - common_params_speculative_ngram_mod ngram_mod; - common_params_speculative_ngram_map ngram_simple; - common_params_speculative_ngram_map ngram_map_k; - common_params_speculative_ngram_map ngram_map_k4v; + std::vector devices; // devices to use for offloading - common_params_speculative_ngram_cache ngram_cache; + std::vector> replacements; // main to speculative model replacements + std::vector tensor_buft_overrides; bool has_dft() const { - return !draft.mparams.path.empty() || !draft.mparams.hf_repo.empty(); - } - - uint32_t need_n_rs_seq() const { - bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) { - return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP; - }); - - return needs_rs_seq ? draft.n_max : 0u; + return !mparams_dft.path.empty() || !mparams_dft.hf_repo.empty(); } }; struct common_params_vocoder { struct common_params_model model; - std::string speaker_file; // speaker file path + std::string speaker_file = ""; // speaker file path // NOLINT - bool use_guide_tokens = false; // enable guide tokens to improve TTS accuracy + bool use_guide_tokens = false; // enable guide tokens to improve TTS accuracy // NOLINT }; struct common_params_diffusion { @@ -445,20 +437,19 @@ struct common_params { // offload params std::vector devices; // devices to use for offloading - int32_t n_gpu_layers = -1; // number of layers to store in VRAM, -1 is auto, <= -2 is all - int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors - float tensor_split[128] = {0}; // how split tensors should be distributed across GPUs - bool fit_params = true; // whether to fit unset model/context parameters to free device memory - bool fit_params_print = false; // print the estimated required memory to run the model - int32_t fit_params_min_ctx = 4096; // minimum context size to set when trying to reduce memory use + int32_t n_gpu_layers = -1; // number of layers to store in VRAM, -1 is auto, <= -2 is all + int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors + float tensor_split[128] = {0}; // how split tensors should be distributed across GPUs + bool fit_params = true; // whether to fit unset model/context parameters to free device memory + int32_t fit_params_min_ctx = 4096; // minimum context size to set when trying to reduce memory use // margin per device in bytes for fitting parameters to free memory: std::vector fit_params_target = std::vector(llama_max_devices(), 1024 * 1024*1024); enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs - common_cpu_params cpuparams; - common_cpu_params cpuparams_batch; + struct cpu_params cpuparams; + struct cpu_params cpuparams_batch; ggml_backend_sched_eval_callback cb_eval = nullptr; void * cb_eval_user_data = nullptr; @@ -592,7 +583,7 @@ struct common_params { int32_t n_threads_http = -1; // number of threads to process HTTP requests (TODO: support threadpool) int32_t n_cache_reuse = 0; // min chunk size to reuse from the cache via KV shifting bool cache_prompt = true; // whether to enable prompt caching - bool cache_idle_slots = true; // save and clear idle slots upon starting a new task + bool clear_idle = true; // save and clear idle slots upon starting a new task int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot int32_t checkpoint_every_nt = 8192; // make a checkpoint every n tokens during prefill int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. @@ -606,6 +597,8 @@ struct common_params { bool force_pure_content_parser = false; common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK; int enable_reasoning = -1; // -1 = auto, 0 = disable, 1 = enable + int reasoning_budget = -1; + std::string reasoning_budget_message; // message injected before end tag when budget exhausted bool prefill_assistant = true; // if true, any trailing assistant message will be prefilled into the response int sleep_idle_seconds = -1; // if >0, server will sleep after this many seconds of idle time @@ -616,21 +609,11 @@ struct common_params { std::map default_template_kwargs; - // UI configs -#ifdef LLAMA_UI_DEFAULT_ENABLED - bool ui = LLAMA_UI_DEFAULT_ENABLED != 0; -#else - bool ui = true; // default to enabled when not set -#endif - - // Deprecated: use ui, ui_mcp_proxy, ui_config_json instead - bool webui = ui; + // webui configs + bool webui = true; bool webui_mcp_proxy = false; std::string webui_config_json; - bool ui_mcp_proxy = false; - std::string ui_config_json; - // "advanced" endpoints are disabled by default for better security bool endpoint_slots = true; bool endpoint_props = false; // only control POST requests, not GET @@ -708,12 +691,11 @@ struct common_params { // initializes the logging system and prints info about the build void common_init(); -void common_params_print_info(const common_params & params, bool print_devices = true); std::string common_params_get_system_info(const common_params & params); bool parse_cpu_range(const std::string & range, bool(&boolmask)[GGML_MAX_N_THREADS]); bool parse_cpu_mask(const std::string & mask, bool(&boolmask)[GGML_MAX_N_THREADS]); -void postprocess_cpu_params(common_cpu_params & cpuparams, const common_cpu_params * role_model = nullptr); +void postprocess_cpu_params(cpu_params & cpuparams, const cpu_params * role_model = nullptr); bool set_process_priority(enum ggml_sched_priority prio); // @@ -781,11 +763,6 @@ inline bool string_starts_with(std::string_view str, std::string_view prefix) { str.compare(0, prefix.size(), prefix) == 0; } -// remove when moving to c++20 -inline bool string_starts_with(std::string_view str, char prefix) { - return !str.empty() && str.front() == prefix; -} - // remove when moving to c++20 inline bool string_ends_with(std::string_view str, std::string_view suffix) { return str.size() >= suffix.size() && @@ -859,7 +836,7 @@ struct common_sampler; // note: defines the model, context, samplers, ets. lifetimes struct common_init_result { - common_init_result(common_params & params, bool model_only = false); + common_init_result(common_params & params); ~common_init_result(); llama_model * model(); @@ -877,37 +854,16 @@ struct common_init_result { using common_init_result_ptr = std::unique_ptr; -common_init_result_ptr common_init_from_params(common_params & params, bool model_only = false); +common_init_result_ptr common_init_from_params(common_params & params); struct llama_model_params common_model_params_to_llama ( common_params & params); struct llama_context_params common_context_params_to_llama(const common_params & params); -struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params); +struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const cpu_params & params); // clear LoRA adapters from context, then apply new list of adapters void common_set_adapter_lora(struct llama_context * ctx, std::vector & lora); -// model endpoint from env -std::string common_get_model_endpoint(); - -// -// Context utils -// - -enum common_context_seq_rm_type { - COMMON_CONTEXT_SEQ_RM_TYPE_NO = 0, // seq_rm not supported (e.g. no memory module) - COMMON_CONTEXT_SEQ_RM_TYPE_PART = 1, // can seq_rm partial sequences - COMMON_CONTEXT_SEQ_RM_TYPE_FULL = 2, // can seq_rm full sequences only - COMMON_CONTEXT_SEQ_RM_TYPE_RS = 3, // can seq_rm partial sequences, bounded by n_rs_seq -}; - -// check if the llama_context can remove sequences -// note: clears the memory of the context -common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx); - -// aborts execution on failure -void common_context_seq_rm (llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1); -void common_context_seq_add(llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta); -void common_context_seq_cp (llama_context * ctx, llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1); +std::string get_model_endpoint(); // // Batch utils @@ -1046,50 +1002,3 @@ ggml_opt_dataset_t common_opt_dataset_init(struct llama_context * ctx, const std // "adamw" or "sgd" (case insensitive) enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *); - -// -// prompt utils -// - -struct common_prompt_checkpoint { - int64_t n_tokens; - - llama_pos pos_min; - llama_pos pos_max; - - std::vector data_tgt; - std::vector data_dft; - - size_t size() const; - - bool empty() const; - void clear(); - - void update_pos( - int64_t n_tokens, - llama_pos pos_min, - llama_pos pos_max); - - void update_tgt( - llama_context * ctx, - llama_seq_id seq_id, - llama_state_seq_flags flags); - - void update_dft( - llama_context * ctx, - llama_seq_id seq_id, - llama_state_seq_flags flags); - - void load_tgt( - llama_context * ctx, - llama_seq_id seq_id, - llama_state_seq_flags flags) const; - - void load_dft( - llama_context * ctx, - llama_seq_id seq_id, - llama_state_seq_flags flags) const; - - void clear_tgt(); - void clear_dft(); -}; diff --git a/common/speculative.cpp b/common/speculative.cpp index 1962cbe828c1..644b9551e582 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -3,7 +3,6 @@ #include "common.h" #include "ggml.h" #include "llama.h" -#include "../src/llama-ext.h" // staging API: llama_set_embeddings_pre_norm / llama_get_embeddings_pre_norm_ith (used by MTP) #include "log.h" #include "ngram-cache.h" #include "ngram-map.h" @@ -11,42 +10,40 @@ #include "sampling.h" #include -#include +#include #include #include #include -#include +#include +#include #define SPEC_VOCAB_MAX_SIZE_DIFFERENCE 128 #define SPEC_VOCAB_CHECK_START_TOKEN_ID 5 -const std::map common_speculative_type_from_name_map = { - {"none", COMMON_SPECULATIVE_TYPE_NONE}, - {"draft-simple", COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE}, - {"draft-eagle3", COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3}, - {"draft-mtp", COMMON_SPECULATIVE_TYPE_DRAFT_MTP}, - {"ngram-simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE}, - {"ngram-map-k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K}, - {"ngram-map-k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V}, - {"ngram-mod", COMMON_SPECULATIVE_TYPE_NGRAM_MOD}, - {"ngram-cache", COMMON_SPECULATIVE_TYPE_NGRAM_CACHE} -}; -static std::string common_speculative_get_devices_str(const std::vector & devices) { - if (devices.empty()) { - return "default"; - } +const std::vector common_speculative_types = { + COMMON_SPECULATIVE_TYPE_NONE, + COMMON_SPECULATIVE_TYPE_DRAFT, + COMMON_SPECULATIVE_TYPE_EAGLE3, + COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, + COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, + COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, + COMMON_SPECULATIVE_TYPE_NGRAM_MOD, + COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, + COMMON_SPECULATIVE_TYPE_MTP +}; - std::string result; - for (size_t i = 0; i < devices.size(); i++) { - if (devices[i] == nullptr) { - continue; - } - if (!result.empty()) result += ", "; - result += ggml_backend_dev_name(devices[i]); - } - return result.empty() ? "default" : result; -} +const std::map common_speculative_type_from_name_map = { + {"none", COMMON_SPECULATIVE_TYPE_NONE}, + {"draft", COMMON_SPECULATIVE_TYPE_DRAFT}, + {"eagle3", COMMON_SPECULATIVE_TYPE_EAGLE3}, + {"ngram_simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE}, + {"ngram_map_k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K}, + {"ngram_map_k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V}, + {"ngram_mod", COMMON_SPECULATIVE_TYPE_NGRAM_MOD}, + {"ngram_cache", COMMON_SPECULATIVE_TYPE_NGRAM_CACHE}, + {"mtp", COMMON_SPECULATIVE_TYPE_MTP} +}; struct common_speculative_config { common_speculative_type type; @@ -56,6 +53,48 @@ struct common_speculative_config { const common_params_speculative & p = common_params_speculative{}) : type(t), params(p) {} }; +static bool common_speculative_mtp_arch_ok(const llama_model * model_tgt, const llama_model * model_dft) { + return std::strcmp(llama_model_arch_str(model_tgt), "gemma4") == 0 + && std::strcmp(llama_model_arch_str(model_dft), "gemma4_assistant") == 0; +} + +// MTP-specific vocab compatibility: +// the assistant (draft) only predicts the next token id from the same SentencePiece +// vocabulary; stop-condition / chat-template special tokens are owned by the target. +// Therefore we only require identical vocab_type, vocab size (within tolerance) and +// per-token text equality, and intentionally skip bos/eos id and add_bos/add_eos +// checks (target may be chat-tuned with eos==106, draft with eos==1). +static bool common_speculative_are_compatible_mtp( + const llama_model * model_tgt, + const llama_model * model_dft) { + const llama_vocab * vocab_tgt = llama_model_get_vocab(model_tgt); + const llama_vocab * vocab_dft = llama_model_get_vocab(model_dft); + + if (llama_vocab_type(vocab_tgt) != llama_vocab_type(vocab_dft)) { + return false; + } + + const int n_vocab_tgt = llama_vocab_n_tokens(vocab_tgt); + const int n_vocab_dft = llama_vocab_n_tokens(vocab_dft); + const int vocab_diff = n_vocab_tgt > n_vocab_dft + ? n_vocab_tgt - n_vocab_dft + : n_vocab_dft - n_vocab_tgt; + + if (vocab_diff > SPEC_VOCAB_MAX_SIZE_DIFFERENCE) { + return false; + } + + for (int i = SPEC_VOCAB_CHECK_START_TOKEN_ID; i < std::min(n_vocab_tgt, n_vocab_dft); ++i) { + const char * t_tgt = llama_vocab_get_text(vocab_tgt, i); + const char * t_dft = llama_vocab_get_text(vocab_dft, i); + if (std::strcmp(t_tgt, t_dft) != 0) { + return false; + } + } + + return true; +} + static bool common_speculative_are_compatible( const llama_model * model_tgt, const llama_model * model_dft) { @@ -69,26 +108,18 @@ static bool common_speculative_are_compatible( LOG_DBG("%s: vocab_type dft: %d\n", __func__, vocab_type_dft); if (vocab_type_tgt != vocab_type_dft) { - LOG_WRN("%s: draft model vocab type must match target model to use speculation but " - "vocab_type_dft = %d while vocab_type_tgt = %d\n", __func__, vocab_type_dft, vocab_type_tgt); + LOG_DBG("%s: draft model vocab type must match target model to use speculation but ", __func__); + LOG_DBG("vocab_type_dft = %d while vocab_type_tgt = %d\n", vocab_type_dft, vocab_type_tgt); return false; } - if (llama_vocab_get_add_bos(vocab_tgt) != llama_vocab_get_add_bos(vocab_dft) || - (llama_vocab_get_add_bos(vocab_tgt) && llama_vocab_bos(vocab_tgt) != llama_vocab_bos(vocab_dft))) { - LOG_WRN("%s: draft model bos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n", - __func__, - llama_vocab_get_add_bos(vocab_tgt), llama_vocab_get_add_bos(vocab_dft), - llama_vocab_bos(vocab_tgt), llama_vocab_bos(vocab_dft)); - return false; - } - - if (llama_vocab_get_add_eos(vocab_tgt) != llama_vocab_get_add_eos(vocab_dft) || - (llama_vocab_get_add_eos(vocab_tgt) && llama_vocab_eos(vocab_tgt) != llama_vocab_eos(vocab_dft))) { - LOG_WRN("%s: draft model eos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n", - __func__, - llama_vocab_get_add_eos(vocab_tgt), llama_vocab_get_add_eos(vocab_dft), - llama_vocab_eos(vocab_tgt), llama_vocab_eos(vocab_dft)); + if ( + llama_vocab_get_add_bos(vocab_tgt) != llama_vocab_get_add_bos(vocab_dft) || + llama_vocab_get_add_eos(vocab_tgt) != llama_vocab_get_add_eos(vocab_dft) || + llama_vocab_bos(vocab_tgt) != llama_vocab_bos(vocab_dft) || + llama_vocab_eos(vocab_tgt) != llama_vocab_eos(vocab_dft) + ) { + LOG_DBG("%s: draft model special tokens must match target model to use speculation\n", __func__); return false; } @@ -123,16 +154,12 @@ static bool common_speculative_are_compatible( return true; } -using common_speculative_draft_params_vec = std::vector; - // state of an implementation of speculative decoding // // each implementation has a unique type and a state that is implementation-specific -// in a subclass of common_speculative_impl -struct common_speculative_impl { - const common_speculative_type type; - - uint32_t n_seq; +// in a subclass of common_speculative_state +struct common_speculative_state { + const enum common_speculative_type type; size_t n_call_begin = 0; // number of times this implementation was called for refresh. size_t n_call_draft = 0; // number of times this implementation was called for generation. @@ -150,50 +177,44 @@ struct common_speculative_impl { int64_t t_draft_us = 0; // total time spent in generating drafts in this implementation in microseconds. int64_t t_accept_us = 0; // total time spent in accumulation of this implementation in microseconds. - common_speculative_impl(common_speculative_type type, uint32_t n_seq) : type(type), n_seq(n_seq) {} - - virtual ~common_speculative_impl() = default; - - virtual void begin(llama_seq_id seq_id, const llama_tokens & prompt) = 0; - - virtual bool process(const llama_batch & batch) = 0; + common_speculative_state(enum common_speculative_type type) : type(type) {} - virtual void draft(common_speculative_draft_params_vec & dparams) = 0; + virtual ~common_speculative_state() = default; - virtual void accept(llama_seq_id seq_id, uint16_t n_accepted, bool is_other) = 0; + virtual void begin(const llama_tokens & prompt) = 0; - // true if this implementation requires the target context to extract post-norm embeddings - virtual bool need_embd() const = 0; + virtual void draft( + const common_params_speculative & params, + const llama_tokens & prompt_tgt, + llama_token id_last, + llama_tokens & result) = 0; - // true if this implementation requires the target context to extract pre-norm embeddings - virtual bool need_embd_pre_norm() const { return false; } + virtual void accept(uint16_t n_accepted) = 0; }; -struct common_speculative_impl_draft_simple : public common_speculative_impl { - common_params_speculative_draft params; +struct common_speculative_state_draft : public common_speculative_state { + llama_context * ctx_tgt; // only used for retokenizing from ctx_dft + llama_context * ctx_dft; - llama_batch batch; + common_sampler * smpl; - std::vector smpls; + llama_batch batch; + llama_tokens prompt_dft; - common_speculative_impl_draft_simple(const common_params_speculative & params, uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, n_seq) - , params(params.draft) - { - auto * ctx_dft = this->params.ctx_dft; - auto * ctx_tgt = this->params.ctx_tgt; - - LOG_INF("%s: adding speculative implementation 'draft-simple'\n", __func__); - LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min); - LOG_INF("%s: - gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", __func__, - this->params.n_gpu_layers, - ggml_type_name(this->params.cache_type_k), - ggml_type_name(this->params.cache_type_v), - ctx_tgt ? "yes" : "no", - ctx_dft ? "yes" : "no", - common_speculative_get_devices_str(this->params.devices).c_str()); + bool vocab_cmpt = true; // whether retokenization is needed + std::unordered_map vocab_map; + common_speculative_state_draft( + enum common_speculative_type type, + llama_context * ctx_tgt, + llama_context * ctx_dft, + const std::vector> & replacements) + : common_speculative_state(type) + , ctx_tgt(ctx_tgt) + , ctx_dft(ctx_dft) + { batch = llama_batch_init(llama_n_batch(ctx_dft), 0, 1); + smpl = nullptr; // TODO: optimize or pass from outside? // { @@ -211,9 +232,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { // // result->smpl = common_sampler_init(llama_get_model(ctx_dft), params); // } - - smpls.resize(n_seq); - for (auto & smpl : smpls) { + { common_params_sampling params; params.no_perf = false; params.top_k = 10; @@ -221,725 +240,472 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { COMMON_SAMPLER_TYPE_TOP_K, }; - smpl.reset(common_sampler_init(llama_get_model(ctx_dft), params)); + smpl = common_sampler_init(llama_get_model(ctx_dft), params); } - const bool vocab_cmpt = common_speculative_are_compatible(llama_get_model(ctx_tgt), llama_get_model(ctx_dft)); - LOG_DBG("%s: vocab_cmpt = %d\n", __func__, vocab_cmpt); + vocab_cmpt = common_speculative_are_compatible(llama_get_model(ctx_tgt), llama_get_model(ctx_dft)); + LOG_DBG("vocab_cmpt = %d\n", vocab_cmpt); if (!vocab_cmpt) { - LOG_ERR("%s: the target and draft vocabs are not compatible\n", __func__); + LOG_WRN("the target and draft vocabs are not compatible - tokens will be translated between the two\n"); - throw std::runtime_error("draft model vocab type must match target model to use speculation"); + for (const auto & pair : replacements) { + vocab_map[pair.first] = pair.second; + } } + } - if (n_seq != llama_n_seq_max(ctx_dft)) { - LOG_ERR("%s: n_seq mismatch: %d != %d\n", __func__, n_seq, llama_n_seq_max(ctx_dft)); + ~common_speculative_state_draft() override { + llama_perf_context_print(ctx_dft); - throw std::runtime_error("the draft model number of sequences is incompatible with the speculative n_seq"); - } - } + llama_free(ctx_dft); + + common_sampler_free(smpl); - ~common_speculative_impl_draft_simple() override { llama_batch_free(batch); } - void begin(llama_seq_id /*seq_id*/, const llama_tokens & /*prompt*/) override { - // noop + void begin(const llama_tokens & prompt) override { + GGML_UNUSED(prompt); } - bool process(const llama_batch & batch) override { - auto * ctx_dft = params.ctx_dft; + void draft( + const common_params_speculative & params, + const llama_tokens & prompt_tgt, + llama_token id_last, + llama_tokens & result) override { + auto * spec = this; - const int ret = llama_decode(ctx_dft, batch); + auto & batch = spec->batch; + auto & ctx_tgt = spec->ctx_tgt; + auto & ctx_dft = spec->ctx_dft; + auto & smpl = spec->smpl; + auto & prompt_dft = spec->prompt_dft; - if (ret != 0) { - LOG_ERR("%s: failed to decode draft batch, ret = %d\n", __func__, ret); + auto * mem_dft = llama_get_memory(ctx_dft); - return false; - } + int reuse_i = 0; + int reuse_n = 0; - return true; - } + const int n_ctx = llama_n_ctx(ctx_dft) - params.n_max; - void draft(common_speculative_draft_params_vec & dparams) override { - auto & ctx_dft = params.ctx_dft; + llama_tokens prompt_cnv; + if (!spec->vocab_cmpt) { + std::string text; - common_batch_clear(batch); + text = common_detokenize(ctx_tgt, prompt_tgt, true); + text = replace_to_dft(text); - // keep track of which sequences are still drafting - int n_drafting = 0; - std::vector drafting(n_seq); + LOG_DBG("%s: main->draft detokenized string: '%s'\n", __func__, text.c_str()); - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - auto & dp = dparams[seq_id]; + prompt_cnv = common_tokenize(ctx_dft, text, false, true); - if (!dp.drafting) { - continue; - } + // convert id_last to draft vocab. llama_detokenize is called directly to avoid an allocation + const auto * model_tgt = llama_get_model(ctx_tgt); + const auto * vocab_tgt = llama_model_get_vocab(model_tgt); - n_drafting++; - drafting[seq_id] = true; - common_sampler_reset(smpls[seq_id].get()); + int32_t n_chars = llama_detokenize(vocab_tgt, &id_last, 1, nullptr, 0, false, false); + GGML_ASSERT(n_chars < 0 && "failed to detokenize id_last"); - common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true); - } + text.resize(-n_chars); + llama_detokenize(vocab_tgt, &id_last, 1, text.data(), text.size(), false, false); + text = replace_to_dft(text); - int ret = llama_decode(ctx_dft, batch); - if (ret != 0) { - LOG_WRN("%s: llama_decode returned %d\n", __func__, ret); - return; + LOG_DBG("main->draft detokenized id_last(%d): '%s'\n", id_last, text.c_str()); + id_last = common_tokenize(ctx_dft, text, false, true)[0]; } - int i = 0; - - while (n_drafting > 0) { - int i_batch = 0; - - common_batch_clear(batch); - - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - if (!drafting[seq_id]) { - continue; - } - - auto * smpl = smpls[seq_id].get(); - - common_sampler_sample(smpl, ctx_dft, i_batch, true); - ++i_batch; + const llama_tokens & prompt_cur = spec->vocab_cmpt ? prompt_tgt : prompt_cnv; - const auto * cur_p = common_sampler_get_candidates(smpl, true); + const int i_start = std::max(0, (int) prompt_cur.size() - n_ctx); - for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { - LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", - seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p, - common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); - } - - // add drafted token for each sequence - const llama_token id = cur_p->data[0].id; - - // only collect very high-confidence draft tokens - if (cur_p->data[0].p < params.p_min) { - drafting[seq_id] = false; - n_drafting--; - - continue; - } + // reuse as much as possible from the old draft context + // ideally, the draft context should be as big as the target context and we will always reuse the entire prompt + for (int i = 0; i < (int) prompt_dft.size(); ++i) { + int cur = 0; + while (i_start + cur < (int) prompt_cur.size() && + i + cur < (int) prompt_dft.size() && + prompt_cur[i_start + cur] == prompt_dft[i + cur]) { + cur++; + } - common_sampler_accept(smpl, id, true); + if ((cur >= 256 || n_ctx >= (int) prompt_cur.size()) && cur > reuse_n) { + reuse_i = i; + reuse_n = cur; + } + } - auto & dp = dparams.at(seq_id); - auto & result = *dp.result; + LOG_DBG("%s: reuse_i = %d, reuse_n = %d, prompt = %d\n", __func__, reuse_i, reuse_n, (int) prompt_dft.size()); - result.push_back(id); + result.clear(); + result.reserve(params.n_max); - if ((params.n_max <= (int) result.size()) || - (dp.n_max > 0 && dp.n_max <= (int) result.size())) { - drafting[seq_id] = false; - n_drafting--; - continue; + if (reuse_n == 0) { + llama_memory_clear(mem_dft, false); + prompt_dft.clear(); + } else { + // this happens when a previous draft has been discarded (for example, due to being too small), but the + // target model agreed with it. in this case, we simply pass back the previous results to save compute + if (reuse_i + reuse_n < (int) prompt_dft.size() && prompt_dft[reuse_i + reuse_n] == id_last) { + for (int i = reuse_i + reuse_n + 1; i < (int) prompt_dft.size(); ++i) { + result.push_back(prompt_dft[i]); + + if (params.n_max <= (int) result.size()) { + break; + } } - common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); - } - - if (batch.n_tokens == 0) { - break; - } - - // evaluate the drafted tokens on the draft model - ret = llama_decode(ctx_dft, batch); - if (ret != 0) { - LOG_WRN("%s: llama_decode[%d] returned %d\n", __func__, i, ret); - break; + return; } - ++i; - } + if (reuse_i > 0) { + llama_memory_seq_rm (mem_dft, 0, 0, reuse_i); + llama_memory_seq_add(mem_dft, 0, reuse_i, -1, -reuse_i); - for (auto & dp : dparams) { - if (!dp.drafting) { - continue; + prompt_dft.erase(prompt_dft.begin(), prompt_dft.begin() + reuse_i); } - if (dp.result->size() < (size_t) params.n_min) { - dp.result->clear(); + if (reuse_n < (int) prompt_dft.size()) { + llama_memory_seq_rm (mem_dft, 0, reuse_n, -1); + prompt_dft.erase(prompt_dft.begin() + reuse_n, prompt_dft.end()); } } - } - - void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { - // noop - } - - bool need_embd() const override { - return false; - } -}; -struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { - //common_params_speculative_eagle3 params; + // prepare a batch to evaluate any new tokens in the prompt + common_batch_clear(batch); - common_speculative_impl_draft_eagle3(const common_params_speculative & params, uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq) - { - LOG_INF("%s: adding speculative implementation 'draft-eagle3'\n", __func__); - LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%f\n", __func__, params.draft.n_max, params.draft.n_min, params.draft.p_min); - } + for (size_t i = i_start + reuse_n; i < prompt_cur.size(); ++i) { + //LOG_DBG("i = %d, i_start = %d, reuse_n = %d, i - i_start = %d, id = %6d\n", i, i_start, reuse_n, i - i_start, prompt_cur[i]); + common_batch_add(batch, prompt_cur[i], i - i_start, { 0 }, false); - void begin(llama_seq_id /*seq_id*/, const llama_tokens & /*prompt*/) override { - // noop - } - - bool process(const llama_batch & /*batch*/) override { - // TODO: implement - return true; - } + prompt_dft.push_back(prompt_cur[i]); + } - void draft(common_speculative_draft_params_vec & /*dparams*/) override { - // TODO: implement - } + // we should rarely end-up here during normal decoding + if (batch.n_tokens > 0) { + //LOG_DBG("%s: draft prompt batch: %s\n", __func__, string_from(ctx, batch).c_str()); - void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { - // noop - } - - bool need_embd() const override { - return false; - } -}; + llama_decode(ctx_dft, batch); + } -struct common_speculative_impl_draft_mtp : public common_speculative_impl { - common_params_speculative_draft params; // reuses the draft-model params slot (ctx_tgt/ctx_dft) + const llama_pos n_past = prompt_dft.size(); - llama_batch batch; + LOG_DBG("%s: n_past = %d\n", __func__, n_past); - std::vector smpls; + common_batch_clear(batch); + common_batch_add (batch, id_last, n_past, { 0 }, true); - // backend sampler chain per seq, attached to ctx_dft - std::vector backend_chains; + prompt_dft.push_back(id_last); - int32_t n_embd = 0; + LOG_DBG("%s: draft prompt: %s\n", __func__, string_from(ctx_dft, prompt_dft).c_str()); - // Per-sequence cross-batch carryover: pair (h_p, x_{p+1}) at MTP pos p+1. - // The last h-row of one process() call needs the first token of the NEXT - // call to pair with, so it's stashed here until that next call fires. - std::vector> pending_h; // [n_seq][n_embd] + llama_decode(ctx_dft, batch); - std::vector i_batch_beg; - std::vector i_batch_end; + common_sampler_reset(smpl); - // Hidden rows from the most recent target verification batch, grouped by seq. - // Row 0 corresponds to the sampled token, row N to the Nth accepted draft token. - std::vector> verify_h; - std::vector verify_h_rows; + // sample n_draft tokens from the draft model + for (int i = 0; i < params.n_max; ++i) { + common_batch_clear(batch); - // Per-seq draft length from the last draft() call, used in accept() to - // roll back ctx_dft's recurrent state past the AR draft's redundant - // pre-advancement before process() mirrored the verify batch. - std::vector last_n_drafted; + common_sampler_sample(smpl, ctx_dft, 0, true); - common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq) - , params(params.draft) - { - auto * ctx_tgt = this->params.ctx_tgt; - auto * ctx_dft = this->params.ctx_dft; - GGML_ASSERT(ctx_tgt && ctx_dft && "MTP requires ctx_tgt and ctx_dft to be set"); - - n_embd = llama_model_n_embd(llama_get_model(ctx_dft)); - - LOG_INF("%s: adding speculative implementation 'draft-mtp'\n", __func__); - LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f, n_embd=%d, backend_sampling=%d\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min, n_embd, (int) this->params.backend_sampling); - LOG_INF("%s: - gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", __func__, - this->params.n_gpu_layers, - ggml_type_name(this->params.cache_type_k), - ggml_type_name(this->params.cache_type_v), - ctx_tgt ? "yes" : "no", - ctx_dft ? "yes" : "no", - common_speculative_get_devices_str(this->params.devices).c_str()); - - const int32_t n_b = (int32_t) llama_n_batch(ctx_dft); - batch = llama_batch_init(/*n_tokens=*/ n_b, /*embd=*/ n_embd, /*n_seq_max=*/ 1); - // llama_batch_init allocates only one of token/embd; MTP needs both. - // TODO: fix, how to call without malloc - batch.token = (llama_token *) malloc(sizeof(llama_token) * n_b); - - smpls.resize(n_seq); - for (auto & s : smpls) { - common_params_sampling sparams; - sparams.no_perf = false; - sparams.top_k = 10; - sparams.samplers = { COMMON_SAMPLER_TYPE_TOP_K }; - s.reset(common_sampler_init(llama_get_model(ctx_dft), sparams)); - } + const auto * cur_p = common_sampler_get_candidates(smpl, true); - // offload draft sampling to the backend - backend_chains.assign(n_seq, nullptr); - if (this->params.backend_sampling) { - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params()); - llama_sampler_chain_add(chain, llama_sampler_init_top_k(10)); - - if (!llama_set_sampler(ctx_dft, seq_id, chain)) { - LOG_WRN("%s: backend offload failed for seq_id=%d; using CPU sampler\n", __func__, (int) seq_id); - llama_sampler_free(chain); - chain = nullptr; - } - backend_chains[seq_id] = chain; + for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { + LOG_DBG(" - draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", + k, i, cur_p->data[k].id, cur_p->data[k].p, common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); } - } - - llama_set_embeddings_pre_norm(ctx_tgt, true, /*masked*/ false); - llama_set_embeddings_pre_norm(ctx_dft, true, /*masked*/ true); - pending_h.assign(n_seq, std::vector(n_embd, 0.0f)); + // add drafted token for each sequence + const llama_token id = cur_p->data[0].id; - i_batch_beg.assign(n_seq, -1); - i_batch_end.assign(n_seq, -1); + common_sampler_accept(smpl, id, true); - verify_h.assign(n_seq, {}); - verify_h_rows.assign(n_seq, 0); + result.push_back(id); - last_n_drafted.assign(n_seq, 0); - } - - ~common_speculative_impl_draft_mtp() override { - auto * ctx_dft = this->params.ctx_dft; - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) backend_chains.size(); ++seq_id) { - if (backend_chains[seq_id] == nullptr) { - continue; - } - if (ctx_dft) { - llama_set_sampler(ctx_dft, seq_id, nullptr); + if (params.n_max <= (int) result.size()) { + break; } - llama_sampler_free(backend_chains[seq_id]); - } - backend_chains.clear(); - if (batch.token != nullptr) { - free(batch.token); - batch.token = nullptr; - } - llama_batch_free(batch); - } + // only collect very high-confidence draft tokens + if (cur_p->data[0].p < params.p_min) { + break; + } - void begin(llama_seq_id seq_id, const llama_tokens & prompt) override { - const int32_t N = (int32_t) prompt.size(); - if (N <= 0) { - return; - } - auto * ctx_dft = this->params.ctx_dft; - const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_dft), seq_id); - if (pos_max < N - 1) { - LOG_WRN("%s: ctx_dft pos_max=%d < N-1=%d - " - "process() hook may not have run on every prefill ubatch " - "(need_embd / logits=1 on every prompt position?). " - "Drafts may degrade.\n", - __func__, (int) pos_max, N - 1); - } - } + common_batch_add(batch, id, n_past + i + 1, { 0 }, true); - bool process(const llama_batch & batch_in) override { - if (batch_in.n_tokens <= 0) { - return true; - } + // evaluate the drafted tokens on the draft model + llama_decode(ctx_dft, batch); - // TODO: how to make it work with vision tokens? - if (batch_in.token == nullptr || batch_in.embd != nullptr) { - return true; + prompt_dft.push_back(id); } - const int32_t n_tokens = batch_in.n_tokens; - - // remember the frist and last batch index for each sequence - std::fill(i_batch_beg.begin(), i_batch_beg.end(), -1); - std::fill(i_batch_end.begin(), i_batch_end.end(), -1); - - for (int k = 0; k < n_tokens; ++k) { - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - GGML_ASSERT(batch_in.n_seq_id[k] == 1); - - if (batch_in.seq_id[k][0] == seq_id) { - i_batch_end[seq_id] = k; - if (i_batch_beg[seq_id] < 0) { - i_batch_beg[seq_id] = k; - } - } + if (!spec->vocab_cmpt) { + std::string detokenized = common_detokenize(ctx_dft, result, true); + detokenized = replace_to_tgt(detokenized); + LOG_DBG("draft->main detokenized string: '%s'\n", detokenized.c_str()); + result = common_tokenize(ctx_tgt, detokenized, false, true); + if (result.size() > (size_t)params.n_max) { + result.resize(params.n_max); } } + } - auto * ctx_tgt = this->params.ctx_tgt; - auto * ctx_dft = this->params.ctx_dft; - - const size_t row_bytes = (size_t) n_embd * sizeof(float); - - common_batch_clear(batch); - - for (int k = 0; k < n_tokens; ++k) { - common_batch_add(batch, batch_in.token[k], batch_in.pos[k], { batch_in.seq_id[k][0] }, 0); - } - - // shift the tgt embeddings to the right by one position - // assumes that the tokens in the batch are sequential for each sequence - // i.e. we cannot have seq_id like this: [0, 0, 0, 1, 1, 0, 1, 1] - // ^--- this is a problem - // TODO:this is generally true, but would be nice to assert it - { - const float * h_tgt = llama_get_embeddings_pre_norm(ctx_tgt); - std::memcpy(batch.embd + (size_t) 1 * n_embd, h_tgt, row_bytes * (n_tokens-1)); - - //{ - // // string with seq_ids in the batch - // std::stringstream ss; - // for (int i = 0; i < n_tokens; ++i) { - // ss << batch_in.seq_id[i][0] << ","; - // } - // LOG_WRN("%s: batch_in.seq_id = %s\n", __func__, ss.str().c_str()); - //} - } + void accept(uint16_t n_accepted) override { + // noop + GGML_UNUSED(n_accepted); + } - // fill the pending embeddings from a previous run - auto set_h = [&](int idx, const float * h_row) { - std::memcpy(batch.embd + (size_t) idx * n_embd, h_row, row_bytes); - }; + std::string replace_to_dft(const std::string & input) const { + std::string result = input; - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - if (i_batch_beg[seq_id] < 0) { - continue; + for (const auto & pair : this->vocab_map) { + size_t pos = result.find(pair.first); + while (pos != std::string::npos) { + result.replace(pos, pair.first.length(), pair.second); + pos = result.find(pair.first, pos + pair.second.length()); } - - set_h(i_batch_beg[seq_id], pending_h[seq_id].data()); } - const int32_t rc = llama_decode(ctx_dft, batch); - if (rc != 0) { - LOG_ERR("%s: llama_decode(ctx_dft) failed rc=%d (pos=%d)\n", __func__, (int) rc, (int) batch_in.pos[0]); - return false; - } - - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - if (i_batch_end[seq_id] < 0) { - continue; - } + return result; + } - const int32_t n_rows = i_batch_end[seq_id] - i_batch_beg[seq_id] + 1; - verify_h_rows[seq_id] = n_rows; - verify_h[seq_id].resize((size_t) n_rows * n_embd); + std::string replace_to_tgt(const std::string & input) const { + std::string result = input; - for (int32_t i = 0; i < n_rows; ++i) { - const float * h = llama_get_embeddings_pre_norm_ith(ctx_tgt, i_batch_beg[seq_id] + i); - std::memcpy(verify_h[seq_id].data() + (size_t) i * n_embd, h, row_bytes); + for (const auto & pair : this->vocab_map) { + size_t pos = result.find(pair.second); + while (pos != std::string::npos) { + result.replace(pos, pair.second.length(), pair.first); + pos = result.find(pair.second, pos + pair.first.length()); } - - std::memcpy(pending_h[seq_id].data(), - verify_h[seq_id].data() + (size_t) (n_rows - 1) * n_embd, row_bytes); } - return true; + return result; } +}; - void draft(common_speculative_draft_params_vec & dparams) override { - auto & ctx_dft = params.ctx_dft; +struct common_speculative_state_eagle3 : public common_speculative_state { + common_speculative_state_eagle3(enum common_speculative_type type) : common_speculative_state(type) {} - common_batch_clear(batch); + void begin(const llama_tokens & prompt) override { + GGML_UNUSED(prompt); + } - // keep track of which sequences are still drafting - int n_drafting = 0; - std::vector drafting(n_seq); + void draft( + const common_params_speculative & params, + const llama_tokens & prompt_tgt, + llama_token id_last, + llama_tokens & draft_tokens) override { + // TODO: implement + GGML_UNUSED(params); + GGML_UNUSED(prompt_tgt); + GGML_UNUSED(id_last); + GGML_UNUSED(draft_tokens); + } - const float * h_row = nullptr; - const size_t row_bytes = (size_t) n_embd * sizeof(float); + void accept(uint16_t n_accepted) override { + // noop + GGML_UNUSED(n_accepted); + } +}; - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - auto & dp = dparams[seq_id]; +struct common_speculative_state_mtp : public common_speculative_state { + llama_context * ctx_tgt; + llama_seq_id seq_id = 0; // target-side sequence id (set by host, e.g. server slot.id) + int h_idx = -1; // output index in target's last decode for h_prev (-1 = last) + // Adaptive skip after consecutive zero-accept batches: when MTP head consistently + // mispredicts (e.g. on numbers/code/rare tokens during long generation), drafting + // costs ~10ms but yields no accepted tokens. Detect this and fall back to plain + // verify-only for one batch; reset skip on next non-empty accept. + size_t prev_n_acc_drafts = 0; + int zero_accept_streak = 0; + int skip_streak_threshold = 2; - if (!dp.drafting) { - continue; - } + explicit common_speculative_state_mtp(enum common_speculative_type type, llama_context * ctx_tgt) + : common_speculative_state(type), ctx_tgt(ctx_tgt) { + // MTP reads last backbone hidden from the target; keep embeddings on across decodes. + llama_set_embeddings(ctx_tgt, true); + } - n_drafting++; - drafting[seq_id] = true; - common_sampler_reset(smpls[seq_id].get()); + void begin(const llama_tokens & prompt) override { + GGML_UNUSED(prompt); + llama_set_embeddings(ctx_tgt, true); + } - common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true); + void draft( + const common_params_speculative & params, + const llama_tokens & prompt_tgt, + llama_token id_last, + llama_tokens & draft_tokens) override { - h_row = pending_h[seq_id].data(); - std::memcpy(batch.embd + n_embd*(batch.n_tokens - 1), h_row, row_bytes); - } + draft_tokens.clear(); - int ret = llama_decode(ctx_dft, batch); - if (ret != 0) { - LOG_WRN("%s: llama_decode returned %d\n", __func__, ret); + const llama_model * model_tgt = llama_get_model(ctx_tgt); + const uint32_t n_bb_u = llama_model_mtp_n_embd_backbone(model_tgt); + if (n_bb_u == 0) { + LOG_ERR("%s: no MTP assistant on target model\n", __func__); return; } + const int32_t n_bb = (int32_t) n_bb_u; - int i = 0; - - while (n_drafting > 0) { - int i_batch = 0; - - common_batch_clear(batch); - - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - if (!drafting[seq_id]) { - continue; - } - - auto * smpl = smpls[seq_id].get(); - - common_sampler_sample(smpl, ctx_dft, i_batch, true); - h_row = llama_get_embeddings_pre_norm_ith(ctx_dft, i_batch); - ++i_batch; - - const auto * cur_p = common_sampler_get_candidates(smpl, true); - - for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { - LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", - seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p, - common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); - } - - // add drafted token for each sequence - const llama_token id = cur_p->data[0].id; - - // only collect very high-confidence draft tokens - if (cur_p->data[0].p < params.p_min) { - drafting[seq_id] = false; - n_drafting--; - - continue; - } - - common_sampler_accept(smpl, id, true); - - auto & dp = dparams.at(seq_id); - auto & result = *dp.result; - - result.push_back(id); - - if (params.n_max <= (int) result.size()) { - drafting[seq_id] = false; - n_drafting--; - continue; - } - - common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); - std::memcpy(batch.embd + n_embd*(batch.n_tokens - 1), h_row, row_bytes); - } - - if (batch.n_tokens == 0) { - break; + // Detect zero-accept of previous draft batch: n_acc_drafts only increments when + // common_speculative_accept is called with n_accepted>0. So if it didn't move + // since our previous draft() return, the previous batch produced 0 accepted drafts. + if (n_call_draft > 0) { + if (n_acc_drafts == prev_n_acc_drafts) { + ++zero_accept_streak; + } else { + zero_accept_streak = 0; } + } + // After threshold consecutive misses, skip MTP draft for one batch — drafting + // would cost ~10ms with no benefit; better to let server do a single-token + // verify (baseline path). + if (zero_accept_streak >= skip_streak_threshold) { + // Reset streak after one skip; if next batch still misses (streak resumes), we'll skip again. + zero_accept_streak = 0; + prev_n_acc_drafts = n_acc_drafts; + return; // empty draft_tokens — server falls back to single-token verify + } - // evaluate the drafted tokens on the draft model - ret = llama_decode(ctx_dft, batch); - if (ret != 0) { - LOG_WRN("%s: llama_decode[%d] returned %d\n", __func__, i, ret); - break; - } + int32_t n_steps_raw = params.draft_block_size > 1 ? params.draft_block_size - 1 : 0; + int32_t n_steps = std::min(n_steps_raw, params.n_max); - ++i; + if (n_steps <= 0) { + return; } - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - auto & dp = dparams[seq_id]; - if (!dp.drafting) { - continue; - } - - if (dp.result->size() < (size_t) params.n_min) { - dp.result->clear(); - } + llama_set_embeddings(ctx_tgt, true); - last_n_drafted[seq_id] = (uint16_t) dp.result->size(); + std::vector h_prev((size_t) n_bb, 0.0f); + bool h_tgt_used = false; + // Use the explicit h_idx pointing at the last accepted output (set by the host after + // sample_and_accept_n). If unset, fall back to -1 (last output) which is correct only + // when the previous decode was prefill or when ALL drafts of the previous batch were + // accepted (otherwise -1 points at a rejected draft's hidden state). + if (float * h_tgt = llama_get_embeddings_ith(ctx_tgt, h_idx)) { + const int32_t n_out_tgt = llama_model_n_embd_out(model_tgt); + const int32_t n_copy = std::min(n_bb, n_out_tgt); + std::memcpy(h_prev.data(), h_tgt, (size_t) n_copy * sizeof(float)); + h_tgt_used = true; } - } - void accept(llama_seq_id seq_id, uint16_t n_accepted, bool /*is_other*/) override { - if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { - return; + llama_memory_t mem = llama_get_memory(ctx_tgt); + llama_pos attn_pos = mem ? llama_memory_seq_pos_max(mem, seq_id) : (llama_pos) 0; + if (attn_pos < 0) { + attn_pos = 0; } - const int32_t n_rows = verify_h_rows[seq_id]; - if (n_rows <= 0) { - return; - } + draft_tokens.resize((size_t) n_steps); + const int32_t rc = llama_decode_mtp( + ctx_tgt, + seq_id, + attn_pos, + id_last, + h_prev.data(), + n_steps, + draft_tokens.data(), + /*out_logits*/ nullptr, + /*out_h_prev_last*/ nullptr); - const int32_t i_h = std::min(n_accepted, n_rows - 1); - const size_t row_bytes = (size_t) n_embd * sizeof(float); - std::memcpy(pending_h[seq_id].data(), verify_h[seq_id].data() + (size_t) i_h * n_embd, row_bytes); - } + if (rc != 0) { + LOG_ERR("%s: llama_decode_mtp failed (%d)\n", __func__, (int) rc); + draft_tokens.clear(); + } - bool need_embd() const override { - return false; + // Snapshot accepted-draft counter for next call's zero-accept detection. + prev_n_acc_drafts = n_acc_drafts; } - bool need_embd_pre_norm() const override { - return true; + void accept(uint16_t n_accepted) override { + GGML_UNUSED(n_accepted); } }; // state of self-speculation (simple implementation, not ngram-map) -struct common_speculative_impl_ngram_simple : public common_speculative_impl { - common_params_speculative_ngram_map params; - - // shared across all sequences +struct common_speculative_state_ngram_simple : public common_speculative_state { common_ngram_simple_config config; - common_speculative_impl_ngram_simple( - const common_params_speculative & params, uint32_t n_seq, + common_speculative_state_ngram_simple( + enum common_speculative_type type, common_ngram_simple_config config) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, n_seq) - , params(params.ngram_simple) - , config(config) - { - LOG_INF("%s: adding speculative implementation 'ngram-simple'\n", __func__); - LOG_INF("%s: - size_n=%d, size_m=%d, min_hits=%d\n", __func__, - this->params.size_n, this->params.size_m, this->params.min_hits); - } - - void begin(llama_seq_id /*seq_id*/, const llama_tokens & /*prompt*/) override { - // noop - } + : common_speculative_state(type), config(config) {} - bool process(const llama_batch & /*batch*/) override { - // TODO: implement - return true; + void begin(const llama_tokens & prompt) override { + GGML_UNUSED(prompt); } - void draft(common_speculative_draft_params_vec & dparams) override { - assert(dparams.size() == n_seq); - - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - auto & dp = dparams[seq_id]; - if (!dp.drafting) { - continue; - } + void draft( + const common_params_speculative & params, + const llama_tokens & prompt_tgt, + llama_token id_last, + llama_tokens & result) override { - *dp.result = common_ngram_simple_draft(config, *dp.prompt, dp.id_last); - } + result = common_ngram_simple_draft(config, prompt_tgt, id_last); + GGML_UNUSED(params); } - void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { + void accept(uint16_t n_accepted) override { // noop - } - - bool need_embd() const override { - return false; + GGML_UNUSED(n_accepted); } }; -struct common_speculative_impl_ngram_map_k : public common_speculative_impl { - // n_seq configs - std::vector config; - - common_speculative_impl_ngram_map_k( - const common_ngram_map & config, - uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, n_seq) - { - for (uint32_t i = 0; i < n_seq; i++) { - this->config.push_back(config); - } - - LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(this->type).c_str()); - LOG_INF("%s: - size_key=%d, size_value=%d, key_only=%d, min_hits=%d\n", __func__, - config.size_key, config.size_value, config.key_only, config.min_hits); - } +struct common_speculative_state_ngram_map_k : public common_speculative_state { + // draft ngram map for speculative decoding without draft model + common_ngram_map map; - void begin(llama_seq_id seq_id, const llama_tokens & prompt) override { - GGML_ASSERT(seq_id < (llama_seq_id) n_seq); + common_speculative_state_ngram_map_k( + enum common_speculative_type type, + common_ngram_map map) + : common_speculative_state(type), map(std::move(map)) {} - common_ngram_map_begin(config[seq_id], prompt); + void begin(const llama_tokens & prompt) override { + common_ngram_map_begin(map, prompt); } - bool process(const llama_batch & /*batch*/) override { - // TODO: implement - return true; + void draft( + const common_params_speculative & params, + const llama_tokens & prompt_tgt, + llama_token id_last, + llama_tokens & result) override { + common_ngram_map_draft(map, prompt_tgt, id_last, result); + GGML_UNUSED(params); } - void draft(common_speculative_draft_params_vec & dparams) override { - assert(dparams.size() == n_seq); - - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - auto & dp = dparams[seq_id]; - if (!dp.drafting) { - continue; - } - - common_ngram_map_draft(config[seq_id], *dp.prompt, dp.id_last, *dp.result); - } + void accept(uint16_t n_accepted) override { + common_ngram_map_accept(map, n_accepted); } +}; - void accept(llama_seq_id seq_id, uint16_t n_accepted, bool is_other) override { - GGML_ASSERT((seq_id < (llama_seq_id) config.size())); - - if (is_other) { - return; - } - - common_ngram_map_accept(config[seq_id], n_accepted); - } +struct common_speculative_state_ngram_mod : public common_speculative_state { + common_ngram_mod & mod; - bool need_embd() const override { - return false; - } -}; + // the last position in the prompt that was added to the ngram container + size_t i_last = 0; -struct common_speculative_impl_ngram_mod : public common_speculative_impl { - common_params_speculative_ngram_mod params; + // length of the last drafted n‑gram (number of tokens returned by draft) + size_t n_draft_last = 0; - // shared across all sequences - common_ngram_mod mod; + // consecutive accept rounds with low acceptance fraction (< 0.5) + int n_low = 0; // enable trace logging if LLAMA_TRACE is set const bool verbose; - struct seq_info { - // the last position in the prompt that was added to the ngram container - size_t i_last = 0; - - // length of the last drafted n-gram (number of tokens returned by draft) - size_t n_draft_last = 0; - - // consecutive accept rounds with low acceptance fraction (< 0.5) - int n_low = 0; - }; - - std::vector sinfos; - - common_speculative_impl_ngram_mod( - const common_params_speculative & params, - uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, n_seq) - , params(params.ngram_mod) - , mod(params.ngram_mod.n_match, 4*1024*1024) - , verbose(std::getenv("LLAMA_TRACE") != nullptr) { + common_speculative_state_ngram_mod(enum common_speculative_type type, common_ngram_mod & mod) + : common_speculative_state(type), mod(mod), verbose(std::getenv("LLAMA_TRACE") != nullptr) { static_assert(sizeof(llama_token) == sizeof(common_ngram_mod::entry_t)); - - LOG_INF("%s: adding speculative implementation 'ngram-mod'\n", __func__); - LOG_INF("%s: - n_match=%d, n_max=%d, n_min=%d\n", __func__, - this->params.n_match, this->params.n_max, this->params.n_min); - LOG_INF("%s: - mod size=%zu (%.3f MB)\n", __func__, - mod.size(), (float)(mod.size_bytes())/1024/1024); - - if (this->params.n_match < 16) { - LOG_WRN("%s: ngram_mod n_match=%d is too small - poor quality is possible, " - "see: https://github.com/ggml-org/llama.cpp/pull/19164\n", __func__, this->params.n_match); - } - - sinfos.resize(n_seq); } - void begin(llama_seq_id seq_id, const llama_tokens & prompt) override { - auto & sinfo = sinfos[seq_id]; + void begin(const llama_tokens & prompt) override { + i_last = 0; - sinfo.i_last = 0; - sinfo.n_draft_last = 0; + n_draft_last = 0; const size_t n = mod.get_n(); + if (prompt.size() < n) { return; } @@ -948,7 +714,7 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { mod.add(prompt.data() + i); } - sinfo.i_last = prompt.size() - n; + i_last = prompt.size() - n; const double f = (double)mod.get_used() / (double)mod.size(); LOG_INF("%s: ngram_mod occupancy = %zu/%zu (%.2f)\n", __func__, mod.get_used(), mod.size(), f); @@ -961,17 +727,16 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { } } - void draft_one( - llama_seq_id seq_id, - common_speculative_draft_params & dparams) { - auto & sinfo = sinfos[seq_id]; - auto & result = *dparams.result; - - const auto & prompt = *dparams.prompt; + void draft( + const common_params_speculative & params, + const llama_tokens & prompt_tgt, + llama_token id_last, + llama_tokens & result) override { + GGML_UNUSED(params); - sinfo.n_draft_last = 0; + n_draft_last = 0; - const size_t cur_len = prompt.size(); + const size_t cur_len = prompt_tgt.size(); if (cur_len < mod.get_n()) { return; } @@ -979,19 +744,19 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { const size_t n = mod.get_n(); // add new ngrams in chunks - if (sinfo.i_last + 32 < cur_len) { - for (size_t i = sinfo.i_last; i < cur_len - n; ++i) { - mod.add(prompt.data() + i); + if (i_last + 32 < cur_len) { + for (size_t i = i_last; i < cur_len - n; ++i) { + mod.add(prompt_tgt.data() + i); } - sinfo.i_last = cur_len - n; + i_last = cur_len - n; } result.resize(n + params.n_max); for (size_t i = 0; i < n - 1; ++i) { - result[i] = prompt.at(cur_len - n + 1 + i); + result[i] = prompt_tgt[cur_len - n + 1 + i]; } - result[n - 1] = dparams.id_last; + result[n - 1] = id_last; for (int i = 0; i < params.n_max; ++i) { const llama_token token = mod.get(result.data() + i); @@ -1013,107 +778,59 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { } result.resize(result.size() - n); - // store length of drafted n-gram for later acceptance analysis - sinfo.n_draft_last = result.size(); + // store length of drafted n‑gram for later acceptance analysis + n_draft_last = result.size(); } - bool process(const llama_batch & /*batch*/) override { - // TODO: implement - return true; - } - - void draft(common_speculative_draft_params_vec & dparams) override { - assert(dparams.size() == n_seq); - - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - auto & dp = dparams[seq_id]; - if (!dp.drafting) { - continue; - } - - draft_one(seq_id, dp); - } - } - - void accept(llama_seq_id seq_id, uint16_t n_accepted, bool is_other) override { - if (is_other) { - return; + void accept(uint16_t n_accepted) override { + if (verbose) { + LOG_INF("%s: accepted %d tokens from %zu drafted tokens\n", __func__, n_accepted, n_draft_last); } - auto & sinfo = sinfos[seq_id]; - // compute acceptance fraction if we have a recorded draft length - if (sinfo.n_draft_last > 0) { - const double f_acc = (double)n_accepted / (double)sinfo.n_draft_last; - if (f_acc < 0.25) { - sinfo.n_low++; - if (sinfo.n_low >= 5) { - if (verbose) { - LOG_WRN("%s: low acceptance streak (%d) - resetting ngram_mod\n", __func__, sinfo.n_low); - } + if (n_draft_last > 0) { + const double f_acc = (double)n_accepted / (double)n_draft_last; + if (f_acc < 0.5) { + n_low++; + if (n_low >= 3) { + LOG_WRN("%s: low acceptance streak (%d) – resetting ngram_mod\n", __func__, n_low); mod.reset(); - sinfo.n_low = 0; - sinfo.i_last = 0; + n_low = 0; } } else { - sinfo.n_low = 0; + n_low = 0; } } } - - bool need_embd() const override { - return false; - } }; -struct common_speculative_impl_ngram_cache : public common_speculative_impl { - common_params_speculative_ngram_cache params; - +struct common_speculative_state_ngram_cache : public common_speculative_state { uint16_t n_draft; - bool save_dynamic; bool save_static; - struct seq_info { - size_t cache_size = 0; // number of tokens in n-gram cache + common_ngram_cache ngram_cache_context; + common_ngram_cache ngram_cache_dynamic; + common_ngram_cache ngram_cache_static; - common_ngram_cache ngram_cache_context; - common_ngram_cache ngram_cache_dynamic; - common_ngram_cache ngram_cache_static; - }; + size_t cache_size = 0; // number of tokens in n-gram cache - std::vector sinfos; - - common_speculative_impl_ngram_cache( - const common_params_speculative & params, - uint32_t n_seq, - uint16_t n_draft, + common_speculative_state_ngram_cache( + const enum common_speculative_type type, const std::string & path_static, const std::string & path_dynamic, - bool save_dynamic, - bool save_static) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, n_seq) - , params(params.ngram_cache) + uint16_t n_draft, + bool save_dynamic, + bool save_static) + : common_speculative_state(type) , n_draft(n_draft) , save_dynamic(save_dynamic) , save_static(save_static) { - LOG_INF("%s: adding speculative implementation 'ngram-cache'\n", __func__); - LOG_INF("%s: - n_draft=%d, cache_static=%s, cache_dynamic=%s\n", __func__, - n_draft, - path_static.empty() ? "none" : path_static.c_str(), - path_dynamic.empty() ? "none" : path_dynamic.c_str()); - - sinfos.resize(n_seq); - if (!path_static.empty()) { try { - auto ngram_cache_static = common_ngram_cache_load(path_static); - - for (auto & sinfo : sinfos) { - sinfo.ngram_cache_static = ngram_cache_static; - } + ngram_cache_static = common_ngram_cache_load(path_static); } catch (...) { LOG_ERR("failed to open static lookup cache: %s", path_static.c_str()); GGML_ABORT("Couldn't read static lookup cache"); @@ -1122,11 +839,7 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { if (!path_dynamic.empty()) { try { - auto ngram_cache_dynamic = common_ngram_cache_load(path_dynamic); - - for (auto & sinfo : sinfos) { - sinfo.ngram_cache_dynamic = ngram_cache_dynamic; - } + ngram_cache_dynamic = common_ngram_cache_load(path_dynamic); } catch (...) { LOG_ERR("failed to open dynamic lookup cache: %s", path_dynamic.c_str()); GGML_ABORT("Couldn't read dynamic lookup cache"); @@ -1134,48 +847,44 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { } } - void begin(llama_seq_id /*seq_id*/, const llama_tokens & /*prompt*/) override { - // noop + void begin(const llama_tokens & prompt) override { + GGML_UNUSED(prompt); } - void draft_one( - llama_seq_id seq_id, - common_speculative_draft_params & dparams) { - auto & sinfo = sinfos[seq_id]; - auto & result = *dparams.result; - - const auto & prompt = *dparams.prompt; + void draft( + const common_params_speculative & params, + const llama_tokens & prompt_tgt, + llama_token id_last, + llama_tokens & result) override { + GGML_UNUSED(params); - if (sinfo.cache_size < prompt.size() + 1) { + if (cache_size < prompt_tgt.size() + 1) { llama_tokens tokens_new; - tokens_new.reserve(prompt.size() + 1 - sinfo.cache_size); - for (size_t j = sinfo.cache_size; j < prompt.size(); ++j) { - tokens_new.push_back(prompt[j]); + tokens_new.reserve(prompt_tgt.size() + 1 - cache_size); + for (size_t j = cache_size; j < prompt_tgt.size(); ++j) { + tokens_new.push_back(prompt_tgt[j]); } - tokens_new.push_back(dparams.id_last); // add the last token + tokens_new.push_back(id_last); // add the last token - // Update context ngram cache with new dparams.prompt: - common_ngram_cache_update( - sinfo.ngram_cache_context, - LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, + // Update context ngram cache with new prompt_tgt: + common_ngram_cache_update(ngram_cache_context, LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, tokens_new, tokens_new.size(), false); - sinfo.cache_size = prompt.size() + 1; + cache_size = prompt_tgt.size() + 1; } llama_tokens inp; - inp.reserve(prompt.size() + 1); - for (size_t j = 0; j < prompt.size(); ++j) { - inp.push_back(prompt[j]); + inp.reserve(prompt_tgt.size() + 1); + for (size_t j = 0; j < prompt_tgt.size(); ++j) { + inp.push_back(prompt_tgt[j]); } - inp.push_back(dparams.id_last); + inp.push_back(id_last); - result.push_back(dparams.id_last); + result.push_back(id_last); - common_ngram_cache_draft( - inp, result, n_draft, LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, - sinfo.ngram_cache_context, - sinfo.ngram_cache_dynamic, - sinfo.ngram_cache_static); + common_ngram_cache_draft(inp, result, n_draft, LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, + ngram_cache_context, + ngram_cache_dynamic, + ngram_cache_static); if (result.size() > 0) { // delete first token in result (which is the id_last token) @@ -1183,129 +892,67 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { } } - bool process(const llama_batch & /*batch*/) override { - // TODO: implement - return true; - } - - void draft(common_speculative_draft_params_vec & dparams) override { - assert(dparams.size() == n_seq); - - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - auto & dp = dparams[seq_id]; - if (!dp.drafting) { - continue; - } - - draft_one(seq_id, dp); - } - } - - void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { - // noop - } - - bool need_embd() const override { - return false; + void accept(uint16_t n_accepted) override { + // TODO: noop + GGML_UNUSED(n_accepted); } }; struct common_speculative { - common_speculative_draft_params_vec dparams; - - // list of implementations to use and their states - std::vector> impls; - - // which implementaion was used for a given seq_id - std::vector impl_last; + std::vector> impls; // list of implementations to use and their states + common_speculative_state * curr_impl = nullptr; // current implementation in use (for stats) }; -static common_ngram_map get_common_ngram_map( - common_speculative_type type, - const common_params_speculative_ngram_map & config) { - uint16_t size_key = config.size_n; - uint16_t size_value = config.size_m; - bool key_only = type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K; - uint16_t min_hits = config.min_hits; +static common_ngram_map get_common_ngram_map(const common_speculative_config & config) { + uint16_t size_key = config.params.ngram_size_n; + uint16_t size_value = config.params.ngram_size_m; + bool key_only = (config.type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K); + uint16_t min_hits = config.params.ngram_min_hits; return common_ngram_map(size_key, size_value, key_only, min_hits); } -static common_speculative_impl_ngram_cache create_state_ngram_cache( - const common_speculative_config & config, - uint32_t n_seq, - const std::string & path_static, - const std::string & path_dynamic) { +static common_speculative_state_ngram_cache create_state_ngram_cache( + const std::string & path_static, const std::string & path_dynamic, + const common_speculative_config & config) { uint16_t n_draft = 8; // TODO get from config? // TODO bool param in common/common.h to set save_static/save_dynamic? bool save_static = false; bool save_dynamic = false; - common_speculative_impl_ngram_cache state(config.params, n_seq, n_draft, path_static, path_dynamic, save_static, save_dynamic); + common_speculative_state_ngram_cache state(config.type, path_static, path_dynamic, n_draft, save_static, save_dynamic); return state; } -std::string common_speculative_type_name_str(const std::vector & types) { +std::string common_speculative_type_name_str() { std::string result; - - for (size_t i = 0; i < types.size(); i++) { + for (size_t i = 0; i < common_speculative_types.size(); i++) { if (i > 0) { - result += ","; + result += ", "; } - result += common_speculative_type_to_str(types[i]); + result += common_speculative_type_to_str(common_speculative_types[i]); } return result; } -const char * common_speculative_all_types_str() { - static std::string all_types_str = []() { - std::vector types; - types.reserve(COMMON_SPECULATIVE_TYPE_COUNT); - for (int i = 0; i < COMMON_SPECULATIVE_TYPE_COUNT; i++) { - types.push_back((common_speculative_type) i); - } - return common_speculative_type_name_str(types); - }(); - return all_types_str.c_str(); -} - -std::string common_speculative_type_to_str(common_speculative_type type) { +std::string common_speculative_type_to_str(enum common_speculative_type type) { switch (type) { case COMMON_SPECULATIVE_TYPE_NONE: return "none"; - case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: return "draft-simple"; - case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: return "draft-eagle3"; - case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: return "draft-mtp"; - case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram-simple"; - case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: return "ngram-map-k"; - case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram-map-k4v"; - case COMMON_SPECULATIVE_TYPE_NGRAM_MOD: return "ngram-mod"; - case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: return "ngram-cache"; + case COMMON_SPECULATIVE_TYPE_DRAFT: return "draft"; + case COMMON_SPECULATIVE_TYPE_EAGLE3: return "eagle3"; + case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram_simple"; + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: return "ngram_map_k"; + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram_map_k4v"; + case COMMON_SPECULATIVE_TYPE_NGRAM_MOD: return "ngram_mod"; + case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: return "ngram_cache"; + case COMMON_SPECULATIVE_TYPE_MTP: return "mtp"; default: return "unknown"; } } -std::vector common_speculative_types_from_names(const std::vector & names) { - std::vector types; - types.reserve(names.size()); - - for (const auto & name : names) { - auto type = common_speculative_type_from_name_map.find(name); - if (type != common_speculative_type_from_name_map.end()) { - if (type->second == COMMON_SPECULATIVE_TYPE_NONE) { - return std::vector { COMMON_SPECULATIVE_TYPE_NONE }; - } - types.push_back(type->second); - continue; - } - throw std::invalid_argument("unknown speculative type: " + name); - } - - return types; -} - -common_speculative_type common_speculative_type_from_name(const std::string & name) { +enum common_speculative_type common_speculative_type_from_name(const std::string & name) { const auto it = common_speculative_type_from_name_map.find(name); if (it == common_speculative_type_from_name_map.end()) { return COMMON_SPECULATIVE_TYPE_COUNT; @@ -1313,39 +960,90 @@ common_speculative_type common_speculative_type_from_name(const std::string & na return it->second; } -static uint32_t common_get_enabled_speculative_configs(const std::vector & configs) { - uint32_t result = 0; - for (size_t i = 0; i < configs.size(); i++) { - result |= (1u << configs[i]); +bool common_speculative_is_compat(llama_context * ctx_tgt) { + auto * mem = llama_get_memory(ctx_tgt); + if (mem == nullptr) { + return false; } - return result; + + bool res = true; + + llama_memory_clear(mem, true); + + // eval 2 tokens to check if the context is compatible + std::vector tmp; + tmp.push_back(0); + tmp.push_back(0); + + int ret = llama_decode(ctx_tgt, llama_batch_get_one(tmp.data(), tmp.size())); + if (ret != 0) { + LOG_ERR("%s: llama_decode() failed: %d\n", __func__, ret); + res = false; + goto done; + } + + // try to remove the last tokens + if (!llama_memory_seq_rm(mem, 0, 1, -1)) { + LOG_WRN("%s: the target context does not support partial sequence removal\n", __func__); + res = false; + goto done; + } + +done: + llama_memory_clear(mem, true); + llama_synchronize(ctx_tgt); + + return res; } // initialization of the speculative decoding system // -common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq) { +common_speculative * common_speculative_init( + common_params_speculative & params, + llama_context * ctx_tgt) { + llama_context * ctx_dft = nullptr; + // Gemma4 MTP loads the assistant into the target model (llama_model_load_mtp_from_file); no second context. + if (params.model_dft && params.type != COMMON_SPECULATIVE_TYPE_MTP) { + ctx_dft = llama_init_from_model(params.model_dft, params.cparams_dft); + if (ctx_dft == nullptr) { + LOG_ERR("%s", "failed to create draft context\n"); + return nullptr; + } + } + + if (params.type == COMMON_SPECULATIVE_TYPE_MTP) { + const llama_model * model_tgt = llama_get_model(ctx_tgt); + const llama_model * model_mtp = llama_model_get_mtp_assistant(model_tgt); + + if (!model_mtp) { + LOG_ERR("%s: MTP requires the assistant GGUF loaded into the target (CLI: --spec-type mtp with --mtp-head or --model-draft)\n", __func__); + return nullptr; + } + if (!common_speculative_mtp_arch_ok(model_tgt, model_mtp)) { + LOG_ERR("%s: MTP requires target arch gemma4 and assistant arch gemma4_assistant\n", __func__); + return nullptr; + } + if (!common_speculative_are_compatible_mtp(model_tgt, model_mtp)) { + LOG_ERR("%s: MTP assistant failed vocab compatibility check\n", __func__); + return nullptr; + } + } + // Compute the implementations to use based on the config and their order of preference std::vector configs = {}; // list of speculative configs to try { - uint32_t enabled_configs = common_get_enabled_speculative_configs(params.types); - - bool has_draft_model_path = !params.draft.mparams.path.empty(); - - bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE)); + bool has_draft = !params.mparams_dft.path.empty(); bool has_draft_eagle3 = false; // TODO PR-18039: if params.speculative.eagle3 - bool has_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr; - - bool has_ngram_cache = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_CACHE)); - bool has_ngram_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE)); - bool has_ngram_map_k = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K)); - bool has_ngram_map_k4v = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V)); - bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD)); - // when adding a new type - update here the logic above - static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 9); + bool has_ngram_cache = (params.type == COMMON_SPECULATIVE_TYPE_NGRAM_CACHE); + bool has_ngram_simple = (params.type == COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE); + bool has_ngram_map_k = (params.type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K); + bool has_ngram_map_k4v = (params.type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V); + bool has_ngram_mod = (params.type == COMMON_SPECULATIVE_TYPE_NGRAM_MOD); + bool has_mtp = (params.type == COMMON_SPECULATIVE_TYPE_MTP); - // this list here defines the priority of the speculators - // the one with highest priority are listed first + // In a more complex implementation we could use the same implementation but with different parameters. + // This was initially used in PR-18471 but removed to simplify the code. if (has_ngram_simple) { // This implementation can guess a lot of tokens without any draft model. configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, params)); @@ -1358,91 +1056,93 @@ common_speculative * common_speculative_init(common_params_speculative & params, configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, params)); } if (has_ngram_mod) { + // shared instance for all speculative decoding contexts + if (!params.ngram_mod) { + params.ngram_mod = std::make_shared(params.ngram_size_n, 4*1024*1024); + + LOG_INF("%s: initialized ngram_mod with n=%d, size=%zu (%.3f MB)\n", __func__, + params.ngram_size_n, params.ngram_mod->size(), + (float)(params.ngram_mod->size_bytes())/1024/1024); + + if (params.ngram_size_n < 16) { + LOG_WRN("%s: ngram_mod n=%d is too small - poor quality is possible, see: https://github.com/ggml-org/llama.cpp/pull/19164\n", __func__, params.ngram_size_n); + } + } + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, params)); } if (has_ngram_cache) { configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, params)); } - if (has_draft_simple) { - if (!has_draft_model_path) { - LOG_WRN("%s: draft model is not specified - cannot use 'draft' type\n", __func__); - has_draft_simple = false; + if (has_draft) { + if (has_mtp) { + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_MTP, params)); + } else { + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT, params)); } - } else if (has_draft_model_path && !has_mtp && !has_draft_eagle3) { - LOG_WRN("%s: draft model is specified but 'draft' speculative type is not explicitly enabled - enabling it\n", __func__); - has_draft_simple = true; - } - - if (has_draft_simple) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, params)); } if (has_draft_eagle3) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, params)); - } - if (has_mtp) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params)); + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_EAGLE3, params)); } } - std::vector> impls = {}; + std::vector> impls = {}; for (const common_speculative_config & config : configs) { + LOG_DBG("%s: adding implementation %s\n", __func__, common_speculative_type_to_str(config.type).c_str()); switch (config.type) { case COMMON_SPECULATIVE_TYPE_NONE: break; - case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: { - impls.push_back(std::make_unique(config.params, n_seq)); + case COMMON_SPECULATIVE_TYPE_DRAFT: { + impls.push_back(std::make_unique(config.type, + /* .ctx_tgt = */ ctx_tgt, + /* .ctx_dft = */ ctx_dft, + /* .replacements = */ params.replacements + )); break; } - case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: { - impls.push_back(std::make_unique(config.params, n_seq)); + case COMMON_SPECULATIVE_TYPE_EAGLE3: { + impls.push_back(std::make_unique(config.type)); break; } - case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: { - impls.push_back(std::make_unique(config.params, n_seq)); + case COMMON_SPECULATIVE_TYPE_MTP: { + impls.push_back(std::make_unique(config.type, ctx_tgt)); break; } case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: { - common_ngram_map ngram_map = get_common_ngram_map(config.type, config.params.ngram_simple); + common_ngram_map ngram_map = get_common_ngram_map(config); uint16_t ngram_size_key = ngram_map.size_key; uint16_t mgram_size_value = ngram_map.size_value; auto config_simple = common_ngram_simple_config { - /* .size_ngram = */ ngram_size_key, - /* .size_mgram = */ mgram_size_value + /* .size_ngram = */ ngram_size_key, + /* .size_mgram = */ mgram_size_value }; - auto state = std::make_unique( - /* .params = */ config.params, - /* .n_seq = */ n_seq, - /* .state = */ config_simple + auto state = std::make_unique( + /* .type = */ config.type, + /* .state = */ config_simple ); impls.push_back(std::move(state)); break; } - case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: { - impls.push_back( - std::make_unique( - get_common_ngram_map(config.type, config.params.ngram_map_k), n_seq)); - break; - } + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: { - impls.push_back( - std::make_unique( - get_common_ngram_map(config.type, config.params.ngram_map_k4v), n_seq)); + impls.push_back(std::make_unique( + (config.type), + get_common_ngram_map(config) + )); break; } case COMMON_SPECULATIVE_TYPE_NGRAM_MOD: { - impls.push_back( - std::make_unique(config.params, n_seq)); + GGML_ASSERT(config.params.ngram_mod); + impls.push_back(std::make_unique(config.type, *config.params.ngram_mod)); break; } case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: { auto state = create_state_ngram_cache( - config, n_seq, - params.ngram_cache.lookup_cache_static, - params.ngram_cache.lookup_cache_dynamic); - impls.push_back(std::make_unique(state)); + params.lookup_cache_static, params.lookup_cache_dynamic, config); + impls.push_back(std::make_unique(state)); break; } default: @@ -1451,14 +1151,12 @@ common_speculative * common_speculative_init(common_params_speculative & params, } if (impls.empty()) { - LOG_WRN("%s: no implementations specified for speculative decoding\n", __func__); + LOG_WRN("%s", "no implementations specified for speculative decoding\n"); return nullptr; } auto * result = new common_speculative { - /* .dparams = */ common_speculative_draft_params_vec(n_seq), - /* .impls = */ std::move(impls), - /* .impl_last = */ std::vector(n_seq, nullptr) + /* .impls = */ std::move(impls) }; return result; @@ -1472,152 +1170,78 @@ void common_speculative_free(common_speculative * spec) { delete spec; } -common_speculative_draft_params & common_speculative_get_draft_params( - common_speculative * spec, - llama_seq_id seq_id) { - GGML_ASSERT(spec); - GGML_ASSERT(seq_id < (llama_seq_id) spec->dparams.size()); - - return spec->dparams[seq_id]; -} - -void common_speculative_begin(common_speculative * spec, llama_seq_id seq_id, const llama_tokens & prompt) { +void common_speculative_set_seq_id(common_speculative * spec, llama_seq_id seq_id) { if (spec == nullptr) { return; } - for (auto & impl : spec->impls) { - common_time_meas tm(impl->t_begin_us, !impl->gen_perf); - impl->begin(seq_id, prompt); - impl->n_call_begin++; - } -} - -bool common_speculative_process(common_speculative * spec, const llama_batch & batch) { - bool result = true; - - if (spec == nullptr) { - return result; - } - - for (auto & impl : spec->impls) { - result = result && impl->process(batch); + if (impl->type == COMMON_SPECULATIVE_TYPE_MTP) { + static_cast(impl.get())->seq_id = seq_id; + } } - - return result; } -bool common_speculative_need_embd(common_speculative * spec) { +void common_speculative_set_h_idx(common_speculative * spec, int batch_idx) { if (spec == nullptr) { - return false; + return; } - for (auto & impl : spec->impls) { - if (impl->need_embd()) { - return true; + if (impl->type == COMMON_SPECULATIVE_TYPE_MTP) { + static_cast(impl.get())->h_idx = batch_idx; } } - - return false; } -bool common_speculative_need_embd_pre_norm(common_speculative * spec) { +void common_speculative_begin(common_speculative * spec, const llama_tokens & prompt) { if (spec == nullptr) { - return false; + return; } for (auto & impl : spec->impls) { - if (impl->need_embd_pre_norm()) { - return true; - } + common_time_meas tm(impl->t_begin_us, !impl->gen_perf); + impl->begin(prompt); + impl->n_call_begin++; } - - return false; } -void common_speculative_draft(common_speculative * spec) { - if (spec == nullptr) { - return; - } - - auto & dparams = spec->dparams; - - { - int n_drafting = 0; - - for (auto & dp : dparams) { - GGML_ASSERT(!dp.drafting || dp.result->empty()); - - if (dp.drafting) { - n_drafting++; - } - } +llama_tokens common_speculative_draft( + common_speculative * spec, + const common_params_speculative & params, + const llama_tokens & prompt_tgt, // specified in target model vocab + llama_token id_last) { + llama_tokens result; - if (n_drafting == 0) { - return; - } - } + spec->curr_impl = nullptr; // reset current implementation for (auto & impl : spec->impls) { { common_time_meas tm(impl->t_draft_us, !impl->gen_perf); - impl->draft(dparams); + impl->draft(params, prompt_tgt, id_last, result); impl->n_call_draft++; } - int n_drafting = 0; + if (!result.empty()) { + LOG_DBG("%s: called impl %s, hist size = %zu, call_count = %zu, gen = %zu\n", __func__, + common_speculative_type_to_str(impl.get()->type).c_str(), prompt_tgt.size(), + impl.get()->n_call_draft, result.size()); - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) dparams.size(); ++seq_id) { - auto & dp = dparams[seq_id]; + spec->curr_impl = impl.get(); // set current implementation for stats + impl->n_gen_drafts++; + impl->n_gen_tokens += result.size(); - auto & result = *dp.result; - - // a new draft has been sampled - if (dp.drafting && !result.empty()) { - dp.drafting = false; - - if (dp.n_max > 0) { - if (!result.empty() && (int) result.size() > dp.n_max) { - LOG_DBG("%s: truncating draft to %d tokens\n", __func__, dp.n_max); - result.resize(dp.n_max); - } - } - - if (!result.empty()) { - LOG_DBG("%s: called impl %s, hist size = %zu, call_count = %zu, gen = %zu\n", __func__, - common_speculative_type_to_str(impl.get()->type).c_str(), dp.prompt->size(), - impl.get()->n_call_draft, result.size()); - - // remember which implementation was used - spec->impl_last[seq_id] = impl.get(); - - impl->n_gen_drafts++; - impl->n_gen_tokens += result.size(); - } - } - - if (dp.drafting) { - n_drafting++; - } - } - - if (n_drafting == 0) { - break; + break; // We have a draft, so break out of the loop and return it. } } - // these sequences failed to generate a draft - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) dparams.size(); ++seq_id) { - auto & dp = dparams[seq_id]; + return result; +} - if (dp.drafting) { - dp.drafting = false; - } +void common_speculative_accept(common_speculative * spec, uint16_t n_accepted) { + if (n_accepted == 0) { + return; } -} -void common_speculative_accept(common_speculative * spec, llama_seq_id seq_id, uint16_t n_accepted) { - common_speculative_impl * impl = spec->impl_last[seq_id]; + common_speculative_state * impl = spec->curr_impl; GGML_ASSERT(impl); @@ -1628,16 +1252,9 @@ void common_speculative_accept(common_speculative * spec, llama_seq_id seq_id, u impl->n_acc_tokens += n_accepted; } - impl->accept(seq_id, n_accepted, false); + impl->accept(n_accepted); impl->n_call_accept++; } - - // accept with the rest of the implementations, using is_other == true - for (auto & impl_other : spec->impls) { - if (impl_other.get() != impl) { - impl_other->accept(seq_id, n_accepted, true); - } - } } void common_speculative_print_stats(const common_speculative * spec) { @@ -1657,7 +1274,7 @@ void common_speculative_print_stats(const common_speculative * spec) { str_perf = ""; } - LOG_INF("statistics %16s: #calls(b,g,a) = %4zu %6zu %6zu, #gen drafts = %6zu, #acc drafts = %5zu, #gen tokens = %6zu, #acc tokens = %5zu%s\n", + LOG_INF("statistics %s: #calls(b,g,a) = %zu %zu %zu, #gen drafts = %zu, #acc drafts = %zu, #gen tokens = %zu, #acc tokens = %zu%s\n", common_speculative_type_to_str(impl->type).c_str(), impl->n_call_begin, impl->n_call_draft, impl->n_call_accept, impl->n_gen_drafts, diff --git a/common/speculative.h b/common/speculative.h index f24bac79edb7..e61fa6edb473 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -5,14 +5,8 @@ struct common_speculative; -// comma separated list the provided types -std::string common_speculative_type_name_str(const std::vector & types); - // comma separated list of all types -const char * common_speculative_all_types_str(); - -// parse user provided types -std::vector common_speculative_types_from_names(const std::vector & names); +std::string common_speculative_type_name_str(); // convert string to type enum common_speculative_type common_speculative_type_from_name(const std::string & name); @@ -20,56 +14,39 @@ enum common_speculative_type common_speculative_type_from_name(const std::string // convert type to string std::string common_speculative_type_to_str(enum common_speculative_type type); -common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq); - -void common_speculative_free(common_speculative * spec); - -struct common_speculative_draft_params { - // this flag is used to chain the drafts through all the available implementations - // after the first successful draft from an implementation, we set it - // to false to prevent further drafts for that sequence - // at the end of the draft() call, all drafting flags will be reset to false - bool drafting = false; - - // overrides individual configurations (-1 disabled) - // can be used to constraint the max draft based on the remaining context size - int32_t n_max = -1; +// check if the llama_context is compatible for speculative decoding +// note: clears the memory of the context +bool common_speculative_is_compat(llama_context * ctx_tgt); - llama_pos n_past; - llama_token id_last; +common_speculative * common_speculative_init( + common_params_speculative & params, + llama_context * ctx_tgt); - // TODO: remove in the future by keeping track of the prompt from the _begin() call and the consecutive accept calls - const llama_tokens * prompt; - - // the generated draft from the last _draft() call - llama_tokens * result; -}; - -common_speculative_draft_params & common_speculative_get_draft_params(common_speculative * spec, llama_seq_id seq_id); +void common_speculative_free(common_speculative * spec); // optionally call once at the beginning of a new generation -void common_speculative_begin(common_speculative * spec, llama_seq_id seq_id, const llama_tokens & prompt); +void common_speculative_begin(common_speculative * spec, const llama_tokens & prompt); -// process the batch and update the internal state of the speculative context -bool common_speculative_process(common_speculative * spec, const llama_batch & batch); +// set target-side sequence id used by implementations that read from the target's KV memory +// (currently only used by the MTP implementation; safe no-op for others) +void common_speculative_set_seq_id(common_speculative * spec, llama_seq_id seq_id); -// true if any implementation requires target post-norm embeddings to be extracted -bool common_speculative_need_embd(common_speculative * spec); +// Set the output index in the target's most recent decode whose embeddings should be read +// as h_prev for the next MTP draft. -1 means "last output" (default). +// In speculative verification, after partial draft acceptance the last batch output corresponds +// to a rejected draft; the correct h_prev is at the last *accepted* batch index. +// Safe no-op for non-MTP implementations. +void common_speculative_set_h_idx(common_speculative * spec, int batch_idx); -// true if any implementation requires target pre-norm embeddings to be extracted -bool common_speculative_need_embd_pre_norm(common_speculative * spec); +// sample up to n_draft tokens and add them to the batch using the draft model +llama_tokens common_speculative_draft( + common_speculative * spec, + const common_params_speculative & params, + const llama_tokens & prompt, + llama_token id_last); -// generate drafts for the sequences specified with `common_speculative_get_draft_params` -void common_speculative_draft(common_speculative * spec); - -// informs the speculative context that n_accepted tokens were accepted by the target model -void common_speculative_accept(common_speculative * spec, llama_seq_id, uint16_t n_accepted); +// informs the speculative decoder that n_accepted tokens were accepted by the target model +void common_speculative_accept(common_speculative * spec, uint16_t n_accepted); // print statistics about the speculative decoding void common_speculative_print_stats(const common_speculative * spec); - -struct common_speculative_deleter { - void operator()(common_speculative * s) { common_speculative_free(s); } -}; - -typedef std::unique_ptr common_speculative_ptr; diff --git a/docs/development/gemma4-assistant-tensor-inventory.md b/docs/development/gemma4-assistant-tensor-inventory.md new file mode 100644 index 000000000000..6a844b33cea3 --- /dev/null +++ b/docs/development/gemma4-assistant-tensor-inventory.md @@ -0,0 +1,38 @@ +# Gemma 4 Assistant (MTP drafter) — tensor inventory (Stage 0) + +This document records the expected tensor layout for `Gemma4AssistantForCausalLM` / `model_type: gemma4_assistant`, cross-referenced with the MLX-VLM reference implementation (PR #1112). + +## HF config highlights (`google/gemma-4-26B-A4B-it-assistant`) + +- `architectures`: `["Gemma4AssistantForCausalLM"]` +- `model_type`: `gemma4_assistant` +- `backbone_hidden_size`: 2816 (must match paired target Gemma 4 backbone) +- `use_ordered_embeddings`: typically `false` for 26B-A4B / 31B (dense tied LM head); `true` for E2B/E4B (centroid `MaskedEmbedder`) +- `num_centroids`: 2048 (when ordered embeddings) +- `centroid_intermediate_top_k`: 32 +- Nested `text_config`: 4 layers, `layer_types` e.g. three `sliding_attention` + one `full_attention`, `attention_k_eq_v: true`, `vocab_size: 262144`, `hidden_size: 1024`, etc. + +## Expected safetensors names (HF / MLX) + +| Logical component | Typical HF / checkpoint name | GGUF name (this fork) | +|-------------------|------------------------------|------------------------| +| Pre-projection | `pre_projection.weight` | `mtp.pre_projection.weight` | +| Post-projection | `post_projection.weight` | `mtp.post_projection.weight` | +| Token embeddings (inner) | `model.embed_tokens.weight` | `token_embd.weight` | +| Final norm | `model.norm.weight` | `output_norm.weight` | +| Per-layer | `model.layers.{i}.*` | `blk.{i}.*` (via `tensor_mapping.py`) | +| Centroid head (E2B/E4B) | `masked_embedding.centroids.weight` | `mtp.centroids.weight` | +| Token ordering | `masked_embedding.token_ordering` | `mtp.token_ordering` (I32) | +| LM head (if untied) | `lm_head.weight` | `output.weight` | + +## MLX reference files + +- `mlx_vlm/speculative/drafters/gemma4_assistant/gemma4_assistant.py` — forward, `draft_block`, `sanitize` +- `mlx_vlm/speculative/drafters/gemma4_assistant/masked_embedder.py` — sparse centroid LM head +- `mlx_vlm/speculative/drafters/gemma4_assistant/masks.py` — SWA / full masks + +## Notes for llama.cpp port + +1. MTP consumes **target K/V** for the last sliding-attention and last full-attention layers; the assistant has **no independent KV** in the reference design. The initial C++ integration uses the assistant as a loadable arch with standard KV for bring-up; full KV sharing is wired through `--spec-type mtp` and remains dependent on target/draft pairing and cache layout. +2. `attention_k_eq_v: true` maps to missing `blk.*.attn_v.weight` (V taken from K), matching Gemma 4 handling in `gemma4-iswa.cpp`. +3. Greedy parity with the target requires byte-identical drafting; validate with `tests/test-speculative-mtp` when model paths are available. diff --git a/docs/speculative.md b/docs/speculative.md index 041ff58038de..f9359450511e 100644 --- a/docs/speculative.md +++ b/docs/speculative.md @@ -13,6 +13,43 @@ The `llama-server` application supports several implementations of speculative d A much smaller model (called the _draft model_) generates drafts. A draft model is the most used approach in speculative decoding. +### Gemma 4 MTP assistant (`mtp`) + +For **Gemma 4** targets with the **gemma4_assistant** (MTP head) GGUF, use `--spec-type mtp`. The assistant is **not** a second `llama_context`: weights are loaded into the target model via `llama_model_load_mtp_from_file` (done automatically when using the server/CLI init path). Cross-attention in the MTP graph reads **K/V from the target KV cache** (shared full/sliding layers). + +- Prefer **`--mtp-head /path/to/assistant.gguf`** for clarity; **`--model-draft` (`-md`)** is accepted as a backward-compatible alias (same path field). +- The draft block size \(B\) is **`--draft-block-size`** (the head proposes `B - 1` tokens per round; default 4). +- **`--gpu-layers-draft` / `-ngld`** and **`-ctkd` / `-ctvd`** still apply to how the **assistant tensors** are placed and typed when the assistant GGUF is loaded; the target uses `-ngl` and `-ctk`/`-ctv`. + +Example (paths illustrative). **TurboQuant** KV on the target: `-ctk`/`-ctv`. Assistant-side cache types follow the draft flags if you use them for offload/quant selection. + +```sh +llama-server \ + -m /path/to/gemma-4-target.gguf \ + --mtp-head /path/to/gemma-4-assistant.gguf \ + --spec-type mtp \ + --draft-block-size 4 \ + -c 16384 \ + -ngl 99 -ngld 99 \ + -ctk turbo3 -ctv turbo3 \ + -ctkd turbo3 -ctvd turbo3 \ + -fa on \ + --host 127.0.0.1 --port 8080 +``` + +Repo helper (defaults under `.scratch/`): `scripts/run-gemma4-mtp-server.sh`. + +#### Reconverting `gemma4_assistant` from Hugging Face + +Example (paths relative to repo root): + +```sh +python convert_hf_to_gguf.py .scratch/gemma-4-26B-A4B-it-assistant \ + --outfile .scratch/gemma-assistant-mtp.gguf --outtype f16 +``` + +Use the resulting GGUF as `--mtp-head` (or `-md`) with `--spec-type mtp`. Older assistant GGUFs with `token_embd.weight` first axis 2816 (backbone width) instead of 1024 will fail load; run `scripts/verify-gemma4-assistant-gguf.py` on the file to check. + ### n-gram Cache (`ngram-cache`) An n-gram is a sequence of n tokens. The n-gram cache implementation maintains statistics about short n-gram sequences. diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index d4e7ac799af2..6ca5a9531249 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -153,6 +153,12 @@ class LLM: DENSE_FEAT_IN_SIZE = "{arch}.{dense}_feat_in" DENSE_FEAT_OUT_SIZE = "{arch}.{dense}_feat_out" + N_CENTROIDS = "{arch}.n_centroids" + CENTROID_TOP_K = "{arch}.centroid_top_k" + N_EMBD_BACKBONE = "{arch}.n_embd_backbone" + USE_ORDERED_EMBEDDINGS = "{arch}.use_ordered_embeddings" + REQUIRES_TARGET_ARCH = "{arch}.requires_target_arch" + class Attention: HEAD_COUNT = "{arch}.attention.head_count" HEAD_COUNT_KV = "{arch}.attention.head_count_kv" @@ -182,6 +188,7 @@ class Attention: KEY_LENGTH_SWA = "{arch}.attention.key_length_swa" VALUE_LENGTH_SWA = "{arch}.attention.value_length_swa" SHARED_KV_LAYERS = "{arch}.attention.shared_kv_layers" + K_EQ_V = "{arch}.attention.k_eq_v" SLIDING_WINDOW_PATTERN = "{arch}.attention.sliding_window_pattern" TEMPERATURE_SCALE = "{arch}.attention.temperature_scale" @@ -430,6 +437,7 @@ class MODEL_ARCH(IntEnum): GEMMA3 = auto() GEMMA3N = auto() GEMMA4 = auto() + GEMMA4_ASSISTANT = auto() GEMMA_EMBEDDING = auto() STARCODER2 = auto() RWKV6 = auto() @@ -862,6 +870,11 @@ class MODEL_TENSOR(IntEnum): NEXTN_HNORM = auto() NEXTN_SHARED_HEAD_HEAD = auto() NEXTN_SHARED_HEAD_NORM = auto() + # Gemma 4 MTP assistant (GGUF names: mtp.*) + MTP_PRE_PROJECTION = auto() + MTP_POST_PROJECTION = auto() + MTP_CENTROIDS = auto() + MTP_TOKEN_ORDERING = auto() # lfm2 audio A_ENC_NORM_CONV = auto() A_ENC_LINEAR_POS = auto() @@ -945,6 +958,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.GEMMA3: "gemma3", MODEL_ARCH.GEMMA3N: "gemma3n", MODEL_ARCH.GEMMA4: "gemma4", + MODEL_ARCH.GEMMA4_ASSISTANT: "gemma4_assistant", MODEL_ARCH.GEMMA_EMBEDDING: "gemma-embedding", MODEL_ARCH.STARCODER2: "starcoder2", MODEL_ARCH.RWKV6: "rwkv6", @@ -1407,6 +1421,10 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.NEXTN_HNORM: "blk.{bid}.nextn.hnorm", MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD: "blk.{bid}.nextn.shared_head_head", MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM: "blk.{bid}.nextn.shared_head_norm", + MODEL_TENSOR.MTP_PRE_PROJECTION: "mtp.pre_projection", + MODEL_TENSOR.MTP_POST_PROJECTION: "mtp.post_projection", + MODEL_TENSOR.MTP_CENTROIDS: "mtp.centroids", + MODEL_TENSOR.MTP_TOKEN_ORDERING: "mtp.token_ordering", } MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { @@ -2481,6 +2499,26 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.PER_LAYER_PROJ_NORM, MODEL_TENSOR.PER_LAYER_POST_NORM, ], + MODEL_ARCH.GEMMA4_ASSISTANT: [ + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.MTP_PRE_PROJECTION, + MODEL_TENSOR.MTP_POST_PROJECTION, + MODEL_TENSOR.MTP_CENTROIDS, + MODEL_TENSOR.MTP_TOKEN_ORDERING, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_POST_NORM, + MODEL_TENSOR.FFN_PRE_NORM, + MODEL_TENSOR.FFN_POST_NORM, + MODEL_TENSOR.LAYER_OUT_SCALE, + ], MODEL_ARCH.GEMMA_EMBEDDING: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT, diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index a101382719d0..38b62d8bc4fa 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -799,6 +799,24 @@ def add_clamp_kqv(self, value: float) -> None: def add_shared_kv_layers(self, value: int) -> None: self.add_uint32(Keys.Attention.SHARED_KV_LAYERS.format(arch=self.arch), value) + def add_n_centroids(self, value: int) -> None: + self.add_uint32(Keys.LLM.N_CENTROIDS.format(arch=self.arch), value) + + def add_centroid_top_k(self, value: int) -> None: + self.add_uint32(Keys.LLM.CENTROID_TOP_K.format(arch=self.arch), value) + + def add_n_embd_backbone(self, value: int) -> None: + self.add_uint32(Keys.LLM.N_EMBD_BACKBONE.format(arch=self.arch), value) + + def add_use_ordered_embeddings(self, value: bool) -> None: + self.add_bool(Keys.LLM.USE_ORDERED_EMBEDDINGS.format(arch=self.arch), value) + + def add_attention_k_eq_v(self, value: bool) -> None: + self.add_bool(Keys.Attention.K_EQ_V.format(arch=self.arch), value) + + def add_requires_target_arch(self, value: str) -> None: + self.add_string(Keys.LLM.REQUIRES_TARGET_ARCH.format(arch=self.arch), value) + # if input is array, true means SWA and false means full_attention for each layer def add_sliding_window_pattern(self, value: int | Sequence[bool]) -> None: key = Keys.Attention.SLIDING_WINDOW_PATTERN.format(arch=self.arch) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index c2235cb3b614..0ae11396a7d6 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -2268,6 +2268,19 @@ class TensorNameMap: MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM: ( "model.layers.{bid}.shared_head.norm", ), + + MODEL_TENSOR.MTP_PRE_PROJECTION: ( + "pre_projection", + ), + MODEL_TENSOR.MTP_POST_PROJECTION: ( + "post_projection", + ), + MODEL_TENSOR.MTP_CENTROIDS: ( + "masked_embedding.centroids", + ), + MODEL_TENSOR.MTP_TOKEN_ORDERING: ( + "masked_embedding.token_ordering", + ), } # architecture-specific block mappings diff --git a/include/llama.h b/include/llama.h index 52eb212466b5..82301687ea7d 100644 --- a/include/llama.h +++ b/include/llama.h @@ -499,6 +499,20 @@ extern "C" { size_t n_paths, struct llama_model_params params); + // Gemma 4 MTP: load gemma4_assistant GGUF into a gemma4 target (call after llama_model_load_from_file, before llama_init_from_model). + // Returns 0 on success. + LLAMA_API int llama_model_load_mtp_from_file( + struct llama_model * model, + const char * path_mtp, + struct llama_model_params params); + + LLAMA_API const struct llama_model * llama_model_get_mtp_assistant(const struct llama_model * model); + + LLAMA_API bool llama_model_has_mtp_assistant(const struct llama_model * model); + + // Backbone hidden size for MTP input (0 if no MTP assistant is loaded). + LLAMA_API uint32_t llama_model_mtp_n_embd_backbone(const struct llama_model * model); + LLAMA_API void llama_model_save_to_file( const struct llama_model * model, const char * path_model); @@ -574,6 +588,16 @@ extern "C" { // Returns label of classifier output by index (