From 71abe2bc2b444cec764f465ecd56e5dfbf87dfb2 Mon Sep 17 00:00:00 2001 From: Alexandre Demasi Date: Thu, 23 Apr 2026 16:27:10 +0200 Subject: [PATCH 1/5] fix(server): forward multipart audio fields to transcription handler The multipart dispatcher for POST /v1/audio/transcriptions was only forwarding `model` and `file` from the parsed multipart body to `handle_openai_audio_transcriptions`, silently dropping every other OpenAI-spec field: `response_format`, `language`, `prompt`, `stream`, and `temperature`. This meant callers who sent, e.g., `response_format=verbose_json` via multipart (which is the canonical way per the OpenAI audio API) would always get the default `json` response regardless of what they asked for, and similar for language hints, prompts, streaming, and temperature. Forward all of these from the parsed `parts` map into the request JSON before dispatching. `stream` is parsed as a boolean and `temperature` as a float to match the JSON-body code path in the handler. Fields not present in the multipart body are simply omitted, preserving the existing defaults in the handler. This is a bug fix independent of any response-format features. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/server/server.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/server/server.cpp b/src/server/server.cpp index b7bf6683..31533f14 100644 --- a/src/server/server.cpp +++ b/src/server/server.cpp @@ -1034,6 +1034,11 @@ std::unique_ptr create_lm_server(model_list& models, ModelDownloader& json request_json; request_json["model"] = parts["model"].content; request_json["file"] = parts["file"].content; + if (parts.count("response_format")) request_json["response_format"] = parts["response_format"].content; + if (parts.count("language")) request_json["language"] = parts["language"].content; + if (parts.count("prompt")) request_json["prompt"] = parts["prompt"].content; + if (parts.count("stream")) request_json["stream"] = (parts["stream"].content == "true"); + if (parts.count("temperature")) request_json["temperature"] = std::stof(parts["temperature"].content); rest_handler->handle_openai_audio_transcriptions(request_json, send_response, send_streaming_response, cancellation_token); }); From f4672a8c42ddca05aa73a4cafb802b79bcce5298 Mon Sep 17 00:00:00 2001 From: Alexandre Demasi Date: Thu, 23 Apr 2026 16:27:27 +0200 Subject: [PATCH 2/5] feat(server): support OpenAI verbose_json for audio transcriptions When a client calls `POST /v1/audio/transcriptions` with `response_format=verbose_json`, the handler now returns an OpenAI-compatible payload of the form: { "task": "transcribe", "language": "", "duration": , "text": "", "segments": [ {"id", "seek", "start", "end", "text", "tokens", "temperature", "avg_logprob", "compression_ratio", "no_speech_prob"}, ... ], "model": "" } Per-segment `start` / `end` timestamps are parsed from the Whisper engine's internal `<|X.XX|>` markers by enabling the timestamp flag on `whisper_engine->generate(...)` when verbose output is requested, then splitting the raw output on a `<\|(\d+\.\d+)\|>` regex. Consecutive markers bound each segment; whitespace-only or empty segments are dropped. `duration` is the maximum end-time across emitted segments. The plain `json` format is unchanged on the wire and remains backward-compatible: the handler still returns `{model, text}`, but the `text` field now has any stray timestamp markers stripped (the underlying generator is only asked for timestamps when `verbose_json` is requested, so the strip is a belt-and-suspenders pass). Motivation: enables downstream tools (diarization pipelines, subtitle generators, alignment tools) to align transcript text with audio timecodes via the standard OpenAI API surface, instead of requiring a FastFlowLM-specific endpoint. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/server/rest_handler.cpp | 99 +++++++++++++++++++++++++++++-------- 1 file changed, 79 insertions(+), 20 deletions(-) diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 6920f9e1..6a028eca 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include "server.hpp" ///@brief Normalize messages by merging consecutive user messages (like Ollama does) @@ -1259,35 +1260,93 @@ void RestHandler::handle_openai_audio_transcriptions(const json& request, bool stream = request.value("stream", false); json response; if (this->asr) { + std::string response_format = request.value("response_format", std::string("json")); + bool want_verbose = (response_format == "verbose_json"); #ifndef FASTFLOWLM_LINUX_LIMITED_MODELS this->whisper_engine->load_audio(audio_raw); header_print("FLM", "Transforming audio to text..."); - // Show text std::cout << "Audio content: " << std::flush; - std::pair audio_result = this->whisper_engine->generate(Whisper::whisper_task_type_t::e_transcribe, true, false, std::cout); - std::string audio_context = audio_result.first; + std::pair audio_result = this->whisper_engine->generate( + Whisper::whisper_task_type_t::e_transcribe, + true, + want_verbose, + std::cout); + std::string raw_output = audio_result.first; + std::string language = audio_result.second; std::cout << std::endl; #else throw std::runtime_error("ASR models are not supported in this build"); - std::string audio_context; + std::string raw_output; + std::string language; #endif - response = { - {"model", model}, - {"text", audio_context} - //{"usage", { - // {"type", "tokens"}, - // {"input_tokens", 0}, - // {"input_tokens_details", json::array({ - // { - // {"text_tokens", 0}, - // {"audio_tokens", 0} - // } - // })}, - // {"output_tokens", 0}, - // {"total_tokens", 0} - //}} - }; + static const std::regex ts_regex(R"(<\|(\d+\.\d+)\|>)"); + + if (want_verbose) { + std::vector> markers; + auto it = std::sregex_iterator(raw_output.begin(), raw_output.end(), ts_regex); + auto end = std::sregex_iterator(); + for (; it != end; ++it) { + markers.emplace_back( + std::stof((*it)[1].str()), + static_cast(it->position()), + static_cast(it->position() + it->length())); + } + + json segments = json::array(); + std::string plain_text; + float max_end = 0.0f; + for (size_t i = 0; i + 1 < markers.size(); ++i) { + float seg_start = std::get<0>(markers[i]); + float seg_end = std::get<0>(markers[i + 1]); + size_t text_begin = std::get<2>(markers[i]); + size_t text_end = std::get<1>(markers[i + 1]); + if (text_end < text_begin) continue; + std::string seg_text = raw_output.substr(text_begin, text_end - text_begin); + size_t a = seg_text.find_first_not_of(" \t\n\r"); + if (a == std::string::npos) continue; + size_t b = seg_text.find_last_not_of(" \t\n\r"); + seg_text = seg_text.substr(a, b - a + 1); + if (seg_text.empty()) continue; + + segments.push_back({ + {"id", (int)segments.size()}, + {"seek", 0}, + {"start", seg_start}, + {"end", seg_end}, + {"text", std::string(" ") + seg_text}, + {"tokens", json::array()}, + {"temperature", 0.0}, + {"avg_logprob", 0.0}, + {"compression_ratio", 0.0}, + {"no_speech_prob", 0.0} + }); + if (!plain_text.empty()) plain_text += " "; + plain_text += seg_text; + if (seg_end > max_end) max_end = seg_end; + } + + response = { + {"task", "transcribe"}, + {"language", language}, + {"duration", max_end}, + {"text", plain_text}, + {"segments", segments}, + {"model", model} + }; + } else { + std::string plain_text = std::regex_replace(raw_output, ts_regex, ""); + size_t a = plain_text.find_first_not_of(" \t\n\r"); + if (a == std::string::npos) plain_text.clear(); + else { + size_t b = plain_text.find_last_not_of(" \t\n\r"); + plain_text = plain_text.substr(a, b - a + 1); + } + response = { + {"model", model}, + {"text", plain_text} + }; + } } else { header_print("Warning", "No asr model loaded, cannot load audio file"); From e1c2516a04864131994f8cf59af8ae2061a33f0b Mon Sep 17 00:00:00 2001 From: Alexandre Demasi Date: Sat, 25 Apr 2026 19:26:39 +0200 Subject: [PATCH 3/5] fix(server): defensive temperature parse in multipart dispatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, `std::stof(parts["temperature"].content)` was called unconditionally when the multipart body contained a `temperature` field. If the value was empty, non-numeric, or in a locale-specific format, this threw `std::invalid_argument` / `std::out_of_range`, which was caught by the generic exception handler downstream and surfaced to the client as a 500 — masking what was really a malformed client request. Wrap the parse in a try/catch and drop the field on parse failure, mirroring the way `language` and `prompt` silently fall through to the handler's defaults when absent. The handler then applies its default `temperature`, the request succeeds, and we no longer 500 on malformed input. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/server/server.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/server/server.cpp b/src/server/server.cpp index 31533f14..9be5c099 100644 --- a/src/server/server.cpp +++ b/src/server/server.cpp @@ -1038,7 +1038,13 @@ std::unique_ptr create_lm_server(model_list& models, ModelDownloader& if (parts.count("language")) request_json["language"] = parts["language"].content; if (parts.count("prompt")) request_json["prompt"] = parts["prompt"].content; if (parts.count("stream")) request_json["stream"] = (parts["stream"].content == "true"); - if (parts.count("temperature")) request_json["temperature"] = std::stof(parts["temperature"].content); + if (parts.count("temperature")) { + try { + request_json["temperature"] = std::stof(parts["temperature"].content); + } catch (const std::exception&) { + // malformed temperature: drop it and let the handler apply its default + } + } rest_handler->handle_openai_audio_transcriptions(request_json, send_response, send_streaming_response, cancellation_token); }); From 90c6a519c05b99759a809b039c91a3a917166ab2 Mon Sep 17 00:00:00 2001 From: Alexandre Demasi Date: Sat, 25 Apr 2026 19:26:56 +0200 Subject: [PATCH 4/5] refactor(server): hoist raw_output/language above transcription #ifdef The `FASTFLOWLM_LINUX_LIMITED_MODELS` branch had `throw std::runtime_error(...)` followed by declarations of `raw_output` and `language`. Those declarations were unreachable: `throw` exits the function before they execute, and they existed only to satisfy the post-`#endif` code that referenced the variables. Move the declarations above the `#ifdef` so both branches share the same single declaration. The non-LIMITED branch assigns into the hoisted variables; the LIMITED branch throws unconditionally. No behavioral change, removes the unreachable-code smell flagged by review. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/server/rest_handler.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 6a028eca..69aa1fe4 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -1262,6 +1262,8 @@ void RestHandler::handle_openai_audio_transcriptions(const json& request, if (this->asr) { std::string response_format = request.value("response_format", std::string("json")); bool want_verbose = (response_format == "verbose_json"); + std::string raw_output; + std::string language; #ifndef FASTFLOWLM_LINUX_LIMITED_MODELS this->whisper_engine->load_audio(audio_raw); header_print("FLM", "Transforming audio to text..."); @@ -1271,13 +1273,11 @@ void RestHandler::handle_openai_audio_transcriptions(const json& request, true, want_verbose, std::cout); - std::string raw_output = audio_result.first; - std::string language = audio_result.second; + raw_output = audio_result.first; + language = audio_result.second; std::cout << std::endl; #else throw std::runtime_error("ASR models are not supported in this build"); - std::string raw_output; - std::string language; #endif static const std::regex ts_regex(R"(<\|(\d+\.\d+)\|>)"); From 17ad979b107952e252806c8826521a6b095a501d Mon Sep 17 00:00:00 2001 From: Alexandre Demasi Date: Sat, 25 Apr 2026 19:27:28 +0200 Subject: [PATCH 5/5] fix(server): fallback segment for verbose_json with <2 markers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original verbose_json builder iterates over consecutive marker pairs to bound segments, so it requires at least two `<|X.XX|>` markers in the Whisper output. Very short clips (or any case where the model emits only the initial timestamp before EOS) produce fewer than two markers, and previously this returned an empty `segments: []`, an empty `text`, and `duration: 0` — silently dropping the transcript even though `raw_output` had content. After the segmentation loop, if `segments` is still empty, emit a single fallback segment containing the full stripped transcript. Its `start` defaults to the first available marker (or 0 when no markers exist) and `end` defaults to the last available marker (or 0). `duration` is updated accordingly. If the trimmed transcript is empty as well, no fallback is emitted — `segments` and `text` genuinely have nothing to surface. This guarantees verbose_json never returns less information than the plain `json` path on the same input. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/server/rest_handler.cpp | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 69aa1fe4..b5120013 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -1326,6 +1326,36 @@ void RestHandler::handle_openai_audio_transcriptions(const json& request, if (seg_end > max_end) max_end = seg_end; } + // Fallback: if no segments were emitted (e.g. fewer than two + // timestamp markers were produced for very short clips, or every + // candidate segment trimmed to empty), surface the raw transcript + // as a single segment so verbose_json never drops text the model + // actually produced. + if (segments.empty()) { + std::string fallback_text = std::regex_replace(raw_output, ts_regex, ""); + size_t fa = fallback_text.find_first_not_of(" \t\n\r"); + if (fa != std::string::npos) { + size_t fb = fallback_text.find_last_not_of(" \t\n\r"); + fallback_text = fallback_text.substr(fa, fb - fa + 1); + float fb_start = markers.empty() ? 0.0f : std::get<0>(markers.front()); + float fb_end = markers.empty() ? 0.0f : std::get<0>(markers.back()); + segments.push_back({ + {"id", 0}, + {"seek", 0}, + {"start", fb_start}, + {"end", fb_end}, + {"text", std::string(" ") + fallback_text}, + {"tokens", json::array()}, + {"temperature", 0.0}, + {"avg_logprob", 0.0}, + {"compression_ratio", 0.0}, + {"no_speech_prob", 0.0} + }); + plain_text = fallback_text; + if (fb_end > max_end) max_end = fb_end; + } + } + response = { {"task", "transcribe"}, {"language", language},