From 6640d9ff8d1ad074440d2d93b593bfe0f8d4b54e Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 21 Jun 2026 09:05:39 +0800 Subject: [PATCH 1/2] gpu device selection + new header --- .../python/src/transcribe_cpp/__init__.py | 53 +++++++-- .../python/src/transcribe_cpp/_generated.py | 12 +- bindings/rust/sys/src/transcribe_sys.rs | 33 +++++- bindings/rust/transcribe-cpp/src/backend.rs | 67 +++++++++-- bindings/rust/transcribe-cpp/src/lib.rs | 1 + bindings/rust/transcribe-cpp/src/model.rs | 13 +++ .../swift/Sources/TranscribeCpp/ABIHash.swift | 2 +- .../swift/Sources/TranscribeCpp/Backend.swift | 44 +++++++- .../swift/Sources/TranscribeCpp/Model.swift | 14 +++ .../Sources/TranscribeCpp/TranscribeCpp.swift | 5 +- bindings/typescript/src/_generated.ts | 11 +- bindings/typescript/src/ffi.ts | 4 + bindings/typescript/src/index.ts | 35 +++++- bindings/typescript/src/types.ts | 14 +++ examples/cli/main.cpp | 72 ++++++++++++ include/transcribe.abihash | 2 +- include/transcribe.h | 100 ++++++++++++++--- src/arch/canary/model.cpp | 3 +- src/arch/canary_qwen/model.cpp | 3 +- src/arch/cohere/model.cpp | 3 +- src/arch/funasr_nano/model.cpp | 3 +- src/arch/gigaam/model.cpp | 3 +- src/arch/granite/model.cpp | 3 +- src/arch/granite_nar/model.cpp | 3 +- src/arch/medasr/model.cpp | 3 +- src/arch/moonshine/model.cpp | 3 +- src/arch/moonshine_streaming/model.cpp | 3 +- src/arch/parakeet/model.cpp | 3 +- src/arch/qwen3_asr/model.cpp | 3 +- src/arch/sensevoice/model.cpp | 3 +- src/arch/voxtral/model.cpp | 3 +- src/arch/voxtral_realtime/model.cpp | 3 +- src/arch/whisper/bin_load.cpp | 3 +- src/arch/whisper/model.cpp | 3 +- src/transcribe-load-common.cpp | 104 ++++++++++++++++++ src/transcribe-load-common.h | 15 +++ src/transcribe-model.h | 13 +++ src/transcribe.cpp | 88 +++++++++++---- tests/api_smoke.c | 9 +- tools/transcribe-bench/main.cpp | 9 ++ 40 files changed, 685 insertions(+), 86 deletions(-) diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index ea8cc3cb..97999502 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -25,7 +25,7 @@ import threading import weakref from dataclasses import dataclass -from typing import Literal, Sequence, Union +from typing import Literal, Optional, Sequence, Union from . import _abi, _generated from ._library import _base_version, artifact_dir, load_library, selected_provider @@ -240,6 +240,14 @@ def native_provider() -> str | None: return selected_provider() +_DEVICE_TYPE_NAMES = { + _generated.TRANSCRIBE_DEVICE_TYPE_CPU: "cpu", + _generated.TRANSCRIBE_DEVICE_TYPE_GPU: "gpu", + _generated.TRANSCRIBE_DEVICE_TYPE_IGPU: "igpu", + _generated.TRANSCRIBE_DEVICE_TYPE_ACCEL: "accel", +} + + @dataclass(frozen=True) class BackendDevice: """One registered compute device (owned copies of the C strings).""" @@ -247,23 +255,42 @@ class BackendDevice: name: str description: str kind: str # "cpu" | "accel" | "metal" | "vulkan" | "cuda" | "sycl" | "gpu" | "unknown" + device_type: str # vendor-agnostic class: "cpu" | "gpu" | "igpu" | "accel" + device_id: Optional[str] # stable hw id (PCI bus id), or None (e.g. Metal) + memory_total: int # reported capacity in bytes, or 0 if unreported + # Available bytes — a SNAPSHOT at query time, or 0 if unreported. Re-query + # (via backends() or Model.device) to refresh; backend-defined and not + # comparable across device kinds. + memory_free: int + + +def _backend_device_from_raw(dev) -> BackendDevice: + """Build a BackendDevice from a library-filled transcribe_backend_device.""" + return BackendDevice( + name=_decode(dev.name), + description=_decode(dev.description), + kind=_decode(dev.kind), + device_type=_DEVICE_TYPE_NAMES.get(dev.device_type, "gpu"), + device_id=_decode(dev.device_id) if dev.device_id else None, + memory_total=int(dev.memory_total), + memory_free=int(dev.memory_free), + ) def backends() -> list[BackendDevice]: """The compute devices registered with the native runtime — what the process can actually run on, after backend-module loading and graceful - degradation (e.g. a Vulkan module skipped on a machine without Vulkan).""" + degradation (e.g. a Vulkan module skipped on a machine without Vulkan). + + Each device's ``memory_free`` is live as of the call; call again to poll + a device's available memory over time.""" devices = [] for i in range(_lib.transcribe_backend_device_count()): dev = _generated.transcribe_backend_device() _lib.transcribe_backend_device_init(_byref(dev)) _check(_lib.transcribe_get_backend_device(i, _byref(dev)), f"reading backend device {i}") - devices.append(BackendDevice( - name=_decode(dev.name), - description=_decode(dev.description), - kind=_decode(dev.kind), - )) + devices.append(_backend_device_from_raw(dev)) return devices @@ -781,6 +808,18 @@ def variant(self) -> str: def backend(self) -> str: return _decode(_lib.transcribe_model_backend(self._h)) + @property + def device(self) -> BackendDevice: + """The compute device this model is running on. ``memory_free`` is a + live snapshot, so read this again to poll how much device memory is + left after the model loaded. Raises if the model has no resolved + compute device.""" + dev = _generated.transcribe_backend_device() + _lib.transcribe_backend_device_init(_byref(dev)) + _check(_lib.transcribe_model_get_device(self._h, _byref(dev)), + "model_get_device") + return _backend_device_from_raw(dev) + @property def capabilities(self) -> Capabilities: caps = _Capabilities() diff --git a/bindings/python/src/transcribe_cpp/_generated.py b/bindings/python/src/transcribe_cpp/_generated.py index 59cb7ec3..2c84289d 100644 --- a/bindings/python/src/transcribe_cpp/_generated.py +++ b/bindings/python/src/transcribe_cpp/_generated.py @@ -13,7 +13,7 @@ # Stable digest of the ABI surface below (structs, enums, macros, layout, # prototypes). A native provider package echoes this back so the API # package can reject an ABI-mismatched provider before dlopen. -PUBLIC_HEADER_HASH = "2273744299e5aa65" +PUBLIC_HEADER_HASH = "ebe6a6816e34a24e" # === enum constants === TRANSCRIBE_OK = 0 @@ -79,6 +79,10 @@ TRANSCRIBE_BACKEND_VULKAN = 3 TRANSCRIBE_BACKEND_CPU_ACCEL = 4 TRANSCRIBE_BACKEND_CUDA = 5 +TRANSCRIBE_DEVICE_TYPE_CPU = 0 +TRANSCRIBE_DEVICE_TYPE_GPU = 1 +TRANSCRIBE_DEVICE_TYPE_IGPU = 2 +TRANSCRIBE_DEVICE_TYPE_ACCEL = 3 TRANSCRIBE_FEATURE_INITIAL_PROMPT = 0 TRANSCRIBE_FEATURE_TEMPERATURE_FALLBACK = 1 TRANSCRIBE_FEATURE_LONG_FORM = 2 @@ -145,7 +149,7 @@ class transcribe_whisper_chunk_trace(_c.Structure): pass transcribe_ext._fields_ = [("size", _c.c_uint64), ("kind", _c.c_uint32)] -transcribe_backend_device._fields_ = [("struct_size", _c.c_uint64), ("name", _c.c_char_p), ("description", _c.c_char_p), ("kind", _c.c_char_p)] +transcribe_backend_device._fields_ = [("struct_size", _c.c_uint64), ("name", _c.c_char_p), ("description", _c.c_char_p), ("kind", _c.c_char_p), ("device_id", _c.c_char_p), ("memory_total", _c.c_uint64), ("memory_free", _c.c_uint64), ("device_type", _c.c_int)] transcribe_model_load_params._fields_ = [("struct_size", _c.c_uint64), ("backend", _c.c_int), ("gpu_device", _c.c_int)] transcribe_session_params._fields_ = [("struct_size", _c.c_uint64), ("n_threads", _c.c_int), ("kv_type", _c.c_int), ("n_ctx", _c.c_int32)] transcribe_run_params._fields_ = [("struct_size", _c.c_uint64), ("task", _c.c_int), ("timestamps", _c.c_int), ("pnc", _c.c_int), ("itn", _c.c_int), ("language", _c.c_char_p), ("target_language", _c.c_char_p), ("keep_special_tags", _c.c_bool), ("family", _c.POINTER(transcribe_ext)), ("spec_k_drafts", _c.c_int32)] @@ -187,7 +191,7 @@ class transcribe_whisper_chunk_trace(_c.Structure): # C-compiler layout captured at generation (for offset self-check). STRUCT_LAYOUT = { 'transcribe_ext': {'size': 16, 'align': 8, 'offsets': {'size': 0, 'kind': 8}}, - 'transcribe_backend_device': {'size': 32, 'align': 8, 'offsets': {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24}}, + 'transcribe_backend_device': {'size': 64, 'align': 8, 'offsets': {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56}}, 'transcribe_model_load_params': {'size': 16, 'align': 8, 'offsets': {'struct_size': 0, 'backend': 8, 'gpu_device': 12}}, 'transcribe_session_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'n_threads': 8, 'kv_type': 12, 'n_ctx': 16}}, 'transcribe_run_params': {'size': 64, 'align': 8, 'offsets': {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'language': 24, 'target_language': 32, 'keep_special_tags': 40, 'family': 48, 'spec_k_drafts': 56}}, @@ -287,6 +291,8 @@ def configure(lib): lib.transcribe_model_free.argtypes = [_c.c_void_p] lib.transcribe_model_get_capabilities.restype = _c.c_int lib.transcribe_model_get_capabilities.argtypes = [_c.c_void_p, _c.POINTER(transcribe_capabilities)] + lib.transcribe_model_get_device.restype = _c.c_int + lib.transcribe_model_get_device.argtypes = [_c.c_void_p, _c.POINTER(transcribe_backend_device)] lib.transcribe_model_load_file.restype = _c.c_int lib.transcribe_model_load_file.argtypes = [_c.c_char_p, _c.POINTER(transcribe_model_load_params), _c.POINTER(_c.c_void_p)] lib.transcribe_model_load_params_init.restype = None diff --git a/bindings/rust/sys/src/transcribe_sys.rs b/bindings/rust/sys/src/transcribe_sys.rs index 2cb22204..f2aea5a2 100644 --- a/bindings/rust/sys/src/transcribe_sys.rs +++ b/bindings/rust/sys/src/transcribe_sys.rs @@ -1,11 +1,11 @@ // @generated by `cargo xtask bindgen` from include/transcribe/extensions.h // DO NOT EDIT BY HAND. Regenerate: `cargo xtask bindgen`. -// Pinned to include/transcribe.abihash = 2273744299e5aa65 +// Pinned to include/transcribe.abihash = ebe6a6816e34a24e /// The public-ABI digest these bindings were generated against /// (sha256/16 over the normalized FFI surface). The load-time version /// gate and the CI drift check both anchor on this value. -pub const PUBLIC_HEADER_HASH: &str = "2273744299e5aa65"; +pub const PUBLIC_HEADER_HASH: &str = "ebe6a6816e34a24e"; /* automatically generated by rust-bindgen 0.72.1 */ @@ -205,6 +205,15 @@ unsafe extern "C" { unsafe extern "C" { pub fn transcribe_backend_device_count() -> ::std::os::raw::c_int; } +impl transcribe_device_type { + pub const TRANSCRIBE_DEVICE_TYPE_CPU: transcribe_device_type = transcribe_device_type(0); + pub const TRANSCRIBE_DEVICE_TYPE_GPU: transcribe_device_type = transcribe_device_type(1); + pub const TRANSCRIBE_DEVICE_TYPE_IGPU: transcribe_device_type = transcribe_device_type(2); + pub const TRANSCRIBE_DEVICE_TYPE_ACCEL: transcribe_device_type = transcribe_device_type(3); +} +#[repr(transparent)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct transcribe_device_type(pub ::std::os::raw::c_uint); #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct transcribe_backend_device { @@ -212,11 +221,15 @@ pub struct transcribe_backend_device { pub name: *const ::std::os::raw::c_char, pub description: *const ::std::os::raw::c_char, pub kind: *const ::std::os::raw::c_char, + pub device_id: *const ::std::os::raw::c_char, + pub memory_total: u64, + pub memory_free: u64, + pub device_type: transcribe_device_type, } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { ["Size of transcribe_backend_device"] - [::std::mem::size_of::() - 32usize]; + [::std::mem::size_of::() - 64usize]; ["Alignment of transcribe_backend_device"] [::std::mem::align_of::() - 8usize]; ["Offset of field: transcribe_backend_device::struct_size"] @@ -227,6 +240,14 @@ const _: () = { [::std::mem::offset_of!(transcribe_backend_device, description) - 16usize]; ["Offset of field: transcribe_backend_device::kind"] [::std::mem::offset_of!(transcribe_backend_device, kind) - 24usize]; + ["Offset of field: transcribe_backend_device::device_id"] + [::std::mem::offset_of!(transcribe_backend_device, device_id) - 32usize]; + ["Offset of field: transcribe_backend_device::memory_total"] + [::std::mem::offset_of!(transcribe_backend_device, memory_total) - 40usize]; + ["Offset of field: transcribe_backend_device::memory_free"] + [::std::mem::offset_of!(transcribe_backend_device, memory_free) - 48usize]; + ["Offset of field: transcribe_backend_device::device_type"] + [::std::mem::offset_of!(transcribe_backend_device, device_type) - 56usize]; }; unsafe extern "C" { pub fn transcribe_backend_device_init(p: *mut transcribe_backend_device); @@ -240,6 +261,12 @@ unsafe extern "C" { unsafe extern "C" { pub fn transcribe_backend_available(kind: transcribe_backend_request) -> bool; } +unsafe extern "C" { + pub fn transcribe_model_get_device( + model: *const transcribe_model, + out: *mut transcribe_backend_device, + ) -> transcribe_status; +} #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct transcribe_model_load_params { diff --git a/bindings/rust/transcribe-cpp/src/backend.rs b/bindings/rust/transcribe-cpp/src/backend.rs index c9ebe00e..46d190fd 100644 --- a/bindings/rust/transcribe-cpp/src/backend.rs +++ b/bindings/rust/transcribe-cpp/src/backend.rs @@ -14,9 +14,37 @@ use std::path::Path; use transcribe_cpp_sys as sys; use crate::error::{check, Result}; -use crate::result::owned_str; +use crate::result::{owned_opt_str, owned_str}; use crate::types::Backend; +/// The vendor-agnostic class of a compute device, orthogonal to +/// [`Device::kind`] (which carries the vendor). Distinguishes a discrete GPU +/// from an integrated one, and a host-memory accelerator from the CPU. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeviceType { + /// CPU using system memory. + Cpu, + /// Discrete GPU with dedicated memory. + Gpu, + /// Integrated GPU using host memory. + Igpu, + /// Host-memory accelerator (BLAS/AMX/...). + Accel, +} + +impl DeviceType { + fn from_raw(raw: sys::transcribe_device_type) -> Self { + use sys::transcribe_device_type as T; + match raw { + T::TRANSCRIBE_DEVICE_TYPE_CPU => DeviceType::Cpu, + T::TRANSCRIBE_DEVICE_TYPE_IGPU => DeviceType::Igpu, + T::TRANSCRIBE_DEVICE_TYPE_ACCEL => DeviceType::Accel, + // includes TRANSCRIBE_DEVICE_TYPE_GPU and any unknown value + _ => DeviceType::Gpu, + } + } +} + /// One registered compute device. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Device { @@ -24,9 +52,36 @@ pub struct Device { pub name: String, /// Human-readable description, e.g. "Apple M4 Max". pub description: String, - /// Classified kind: "cpu", "accel", "metal", "vulkan", "cuda", "sycl", - /// "gpu", or "unknown". + /// Classified vendor kind: "cpu", "accel", "metal", "vulkan", "cuda", + /// "sycl", "gpu", or "unknown". pub kind: String, + /// The CPU/GPU/IGPU/ACCEL axis, orthogonal to [`Device::kind`]. + pub device_type: DeviceType, + /// Stable hardware id when the backend reports one (PCI bus id for PCI + /// devices), or `None` (e.g. Metal). + pub device_id: Option, + /// Reported device memory capacity in bytes, or 0 if unreported. + pub memory_total: u64, + /// Available device memory in bytes — a snapshot at the time this was + /// queried, or 0 if unreported. Re-query (via [`devices`] or + /// [`crate::Model::device`]) to refresh it; the value is backend-defined + /// and not comparable across device kinds. + pub memory_free: u64, +} + +impl Device { + /// Build a [`Device`] from the raw FFI struct filled by the library. + pub(crate) fn from_raw(raw: &sys::transcribe_backend_device) -> Device { + Device { + name: owned_str(raw.name), + description: owned_str(raw.description), + kind: owned_str(raw.kind), + device_type: DeviceType::from_raw(raw.device_type), + device_id: owned_opt_str(raw.device_id), + memory_total: raw.memory_total, + memory_free: raw.memory_free, + } + } } /// Load backend modules from `dir` (dynamic builds) and register their @@ -76,11 +131,7 @@ pub fn devices() -> Vec { unsafe { sys::transcribe_backend_device_init(&mut raw) }; let status = unsafe { sys::transcribe_get_backend_device(i, &mut raw) }; if status == sys::transcribe_status::TRANSCRIBE_OK { - out.push(Device { - name: owned_str(raw.name), - description: owned_str(raw.description), - kind: owned_str(raw.kind), - }); + out.push(Device::from_raw(&raw)); } } out diff --git a/bindings/rust/transcribe-cpp/src/lib.rs b/bindings/rust/transcribe-cpp/src/lib.rs index 5a39406a..4d7f1eb1 100644 --- a/bindings/rust/transcribe-cpp/src/lib.rs +++ b/bindings/rust/transcribe-cpp/src/lib.rs @@ -59,6 +59,7 @@ mod version; pub use backend::{ backend_available, device_count, devices, init_backends, init_backends_default, Device, + DeviceType, }; pub use cancel::CancelToken; pub use error::{Error, Result}; diff --git a/bindings/rust/transcribe-cpp/src/model.rs b/bindings/rust/transcribe-cpp/src/model.rs index cdac12b8..b95945b1 100644 --- a/bindings/rust/transcribe-cpp/src/model.rs +++ b/bindings/rust/transcribe-cpp/src/model.rs @@ -20,6 +20,7 @@ use std::sync::{Arc, Mutex}; use transcribe_cpp_sys as sys; +use crate::backend::Device; use crate::error::{check, Result}; use crate::result::owned_str; use crate::session::Session; @@ -197,6 +198,18 @@ impl Model { owned_str(unsafe { sys::transcribe_model_backend(self.inner.ptr) }) } + /// The compute [`Device`] this model is running on — the one that owns its + /// weights. Its `memory_free` is a live snapshot, so re-call to poll how + /// much memory is left on the device after the model loaded. Errors with + /// [`Error::Backend`](crate::Error) if the model has no resolved device. + pub fn device(&self) -> Result { + let mut raw: sys::transcribe_backend_device = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_backend_device_init(&mut raw) }; + let status = unsafe { sys::transcribe_model_get_device(self.inner.ptr, &mut raw) }; + check(status, "model_get_device")?; + Ok(Device::from_raw(&raw)) + } + /// Tokenize plain UTF-8 text into the model's vocabulary (no BOS/EOS, no /// special tags). Errors with [`Error::NotImplemented`](crate::Error) for /// families whose tokenizer has no encode path (e.g. SentencePiece today). diff --git a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift index 9353d417..346391eb 100644 --- a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift +++ b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift @@ -13,7 +13,7 @@ import CTranscribe extension Transcribe { /// sha256/16 of the normalized public FFI surface, pinned to the value in /// include/transcribe.abihash at the time this binding was last reviewed. - public static let pinnedHeaderHash = "2273744299e5aa65" + public static let pinnedHeaderHash = "ebe6a6816e34a24e" /// The public-ABI digest this binding was reviewed against (16 hex chars). public static func headerHash() -> String { pinnedHeaderHash } diff --git a/bindings/swift/Sources/TranscribeCpp/Backend.swift b/bindings/swift/Sources/TranscribeCpp/Backend.swift index 52a10754..fcdb5423 100644 --- a/bindings/swift/Sources/TranscribeCpp/Backend.swift +++ b/bindings/swift/Sources/TranscribeCpp/Backend.swift @@ -22,14 +22,56 @@ public enum Backend: Sendable, Equatable { } } +/// The vendor-agnostic class of a compute device, orthogonal to `Device.kind` +/// (which carries the vendor). Distinguishes a discrete GPU from an integrated +/// one, and a host-memory accelerator from the CPU. +public enum DeviceType: Sendable, Equatable { + case cpu + case gpu + case igpu + case accel + + init(_ c: transcribe_device_type) { + switch c.rawValue { + case TRANSCRIBE_DEVICE_TYPE_CPU.rawValue: self = .cpu + case TRANSCRIBE_DEVICE_TYPE_IGPU.rawValue: self = .igpu + case TRANSCRIBE_DEVICE_TYPE_ACCEL.rawValue: self = .accel + // includes TRANSCRIBE_DEVICE_TYPE_GPU and any unknown value + default: self = .gpu + } + } +} + /// A registered compute device. public struct Device: Sendable, Equatable { /// ggml device name, e.g. "Metal". public let name: String /// Human-readable description, e.g. "Apple M4 Max". public let description: String - /// Classified kind string, e.g. "cpu", "metal", "vulkan", "cuda". + /// Classified vendor kind string, e.g. "cpu", "metal", "vulkan", "cuda". public let kind: String + /// The CPU/GPU/IGPU/ACCEL axis, orthogonal to `kind`. + public let deviceType: DeviceType + /// Stable hardware id (PCI bus id) when the backend reports one, else nil + /// (e.g. Metal). + public let deviceId: String? + /// Reported device memory capacity in bytes, or 0 if unreported. + public let memoryTotal: UInt64 + /// Available device memory in bytes — a snapshot at query time, or 0 if + /// unreported. Re-query (`TranscribeCpp.devices()` or `Model.device`) to + /// refresh; backend-defined and not comparable across device kinds. + public let memoryFree: UInt64 + + /// Build from the raw C struct the library filled. + init(_ raw: transcribe_backend_device) { + name = raw.name.map { String(cString: $0) } ?? "" + description = raw.description.map { String(cString: $0) } ?? "" + kind = raw.kind.map { String(cString: $0) } ?? "" + deviceType = DeviceType(raw.device_type) + deviceId = raw.device_id.map { String(cString: $0) } + memoryTotal = raw.memory_total + memoryFree = raw.memory_free + } } /// A public ABI struct, for the no-model layout-liveness check. Mirrors the diff --git a/bindings/swift/Sources/TranscribeCpp/Model.swift b/bindings/swift/Sources/TranscribeCpp/Model.swift index 08ae61cd..506a00c5 100644 --- a/bindings/swift/Sources/TranscribeCpp/Model.swift +++ b/bindings/swift/Sources/TranscribeCpp/Model.swift @@ -73,6 +73,20 @@ public final class Model: @unchecked Sendable { /// The runtime backend bound to this model, e.g. "metal" / "cpu". public var backend: String { String(cString: transcribe_model_backend(ptr)) } + /// The compute `Device` this model is running on — the one that owns its + /// weights. `memoryFree` is a live snapshot, so read this again to poll how + /// much device memory is left after the model loaded. Throws if the model + /// has no resolved compute device. + public var device: Device { + get throws { + var raw = transcribe_backend_device() + transcribe_backend_device_init(&raw) + try TranscribeError.check( + transcribe_model_get_device(ptr, &raw), context: "model_get_device") + return Device(raw) + } + } + /// Tokenize plain UTF-8 text into the model's vocabulary (no special /// tokens). Throws `.notImplemented` for vocabularies without an encoder. public func tokenize(_ text: String) throws -> [Int32] { diff --git a/bindings/swift/Sources/TranscribeCpp/TranscribeCpp.swift b/bindings/swift/Sources/TranscribeCpp/TranscribeCpp.swift index 30ecc058..9d126ac3 100644 --- a/bindings/swift/Sources/TranscribeCpp/TranscribeCpp.swift +++ b/bindings/swift/Sources/TranscribeCpp/TranscribeCpp.swift @@ -73,10 +73,7 @@ public enum Transcribe { var raw = transcribe_backend_device() transcribe_backend_device_init(&raw) guard transcribe_get_backend_device(index, &raw) == TRANSCRIBE_OK else { continue } - devices.append(Device( - name: raw.name.map { String(cString: $0) } ?? "", - description: raw.description.map { String(cString: $0) } ?? "", - kind: raw.kind.map { String(cString: $0) } ?? "")) + devices.append(Device(raw)) } return devices } diff --git a/bindings/typescript/src/_generated.ts b/bindings/typescript/src/_generated.ts index 947a9a0f..3aa8679d 100644 --- a/bindings/typescript/src/_generated.ts +++ b/bindings/typescript/src/_generated.ts @@ -11,7 +11,7 @@ // Stable digest of the ABI surface (structs, enums, macros, layout, // prototypes), computed by the Python oracle and pinned here so a header // ABI change turns this binding's drift check red for conscious review. -export const PUBLIC_HEADER_HASH = "2273744299e5aa65"; +export const PUBLIC_HEADER_HASH = "ebe6a6816e34a24e"; // === enum constants === export const TRANSCRIBE_OK = 0; @@ -77,6 +77,10 @@ export const TRANSCRIBE_BACKEND_METAL = 2; export const TRANSCRIBE_BACKEND_VULKAN = 3; export const TRANSCRIBE_BACKEND_CPU_ACCEL = 4; export const TRANSCRIBE_BACKEND_CUDA = 5; +export const TRANSCRIBE_DEVICE_TYPE_CPU = 0; +export const TRANSCRIBE_DEVICE_TYPE_GPU = 1; +export const TRANSCRIBE_DEVICE_TYPE_IGPU = 2; +export const TRANSCRIBE_DEVICE_TYPE_ACCEL = 3; export const TRANSCRIBE_FEATURE_INITIAL_PROMPT = 0; export const TRANSCRIBE_FEATURE_TEMPERATURE_FALLBACK = 1; export const TRANSCRIBE_FEATURE_LONG_FORM = 2; @@ -103,7 +107,7 @@ export const TRANSCRIBE_EXT_KIND_WHISPER_RUN = 1314015319; export interface StructLayout { size: number; align: number; offsets: Record; } export const STRUCT_LAYOUT: Record = { 'transcribe_ext': { size: 16, align: 8, offsets: {'size': 0, 'kind': 8} }, - 'transcribe_backend_device': { size: 32, align: 8, offsets: {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24} }, + 'transcribe_backend_device': { size: 64, align: 8, offsets: {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56} }, 'transcribe_model_load_params': { size: 16, align: 8, offsets: {'struct_size': 0, 'backend': 8, 'gpu_device': 12} }, 'transcribe_session_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'n_threads': 8, 'kv_type': 12, 'n_ctx': 16} }, 'transcribe_run_params': { size: 64, align: 8, offsets: {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'language': 24, 'target_language': 32, 'keep_special_tags': 40, 'family': 48, 'spec_k_drafts': 56} }, @@ -145,7 +149,7 @@ export const ABI_STRUCT_IDS: Record = { export function defineTypes(koffi: any): Record { const T: Record = {}; T['transcribe_ext'] = koffi.struct({ size: 'uint64_t', kind: 'uint32_t' }); - T['transcribe_backend_device'] = koffi.struct({ struct_size: 'uint64_t', name: 'char *', description: 'char *', kind: 'char *' }); + T['transcribe_backend_device'] = koffi.struct({ struct_size: 'uint64_t', name: 'char *', description: 'char *', kind: 'char *', device_id: 'char *', memory_total: 'uint64_t', memory_free: 'uint64_t', device_type: 'int' }); T['transcribe_model_load_params'] = koffi.struct({ struct_size: 'uint64_t', backend: 'int', gpu_device: 'int' }); T['transcribe_session_params'] = koffi.struct({ struct_size: 'uint64_t', n_threads: 'int', kv_type: 'int', n_ctx: 'int32_t' }); T['transcribe_run_params'] = koffi.struct({ struct_size: 'uint64_t', task: 'int', timestamps: 'int', pnc: 'int', itn: 'int', language: 'char *', target_language: 'char *', keep_special_tags: 'bool', family: 'void *', spec_k_drafts: 'int32_t' }); @@ -207,6 +211,7 @@ export const FUNCTION_SIGNATURES: Record = { 'transcribe_model_backend': { ret: 'const char *', args: ['const struct transcribe_model *'] }, 'transcribe_model_free': { ret: 'void', args: ['struct transcribe_model *'] }, 'transcribe_model_get_capabilities': { ret: 'transcribe_status', args: ['const struct transcribe_model *', 'struct transcribe_capabilities *'] }, + 'transcribe_model_get_device': { ret: 'transcribe_status', args: ['const struct transcribe_model *', 'struct transcribe_backend_device *'] }, 'transcribe_model_load_file': { ret: 'transcribe_status', args: ['const char *', 'const struct transcribe_model_load_params *', 'struct transcribe_model **'] }, 'transcribe_model_load_params_init': { ret: 'void', args: ['struct transcribe_model_load_params *'] }, 'transcribe_model_supports': { ret: '_Bool', args: ['const struct transcribe_model *', 'transcribe_feature'] }, diff --git a/bindings/typescript/src/ffi.ts b/bindings/typescript/src/ffi.ts index 03529709..ad2db9d6 100644 --- a/bindings/typescript/src/ffi.ts +++ b/bindings/typescript/src/ffi.ts @@ -65,6 +65,10 @@ export function bindLibrary(libraryPath: string): Bound { modelArch: lib.func("transcribe_model_arch_string", "str", ["void *"]), modelVariant: lib.func("transcribe_model_variant_string", "str", ["void *"]), modelBackend: lib.func("transcribe_model_backend", "str", ["void *"]), + modelGetDevice: lib.func("transcribe_model_get_device", "int", [ + "void *", + iop(T.transcribe_backend_device), + ]), modelSupports: lib.func("transcribe_model_supports", "bool", ["void *", "int"]), tokenize: lib.func("transcribe_tokenize", "int", ["void *", "str", "int32_t *", "size_t"]), capabilitiesInit: lib.func("transcribe_capabilities_init", "void", [ diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index a68771b2..e98b143a 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -25,6 +25,7 @@ import type { BatchItem, Capabilities, CommitPolicy, + DeviceType, ExtSlot, FamilyExtension, Feature, @@ -268,6 +269,28 @@ export function libraryPath(): string { return native().libraryPath; } +const DEVICE_TYPE_NAMES: Record = { + [g.TRANSCRIBE_DEVICE_TYPE_CPU]: "cpu", + [g.TRANSCRIBE_DEVICE_TYPE_GPU]: "gpu", + [g.TRANSCRIBE_DEVICE_TYPE_IGPU]: "igpu", + [g.TRANSCRIBE_DEVICE_TYPE_ACCEL]: "accel", +}; + +// Decode a koffi-filled transcribe_backend_device struct into a BackendInfo. +// memory_* are uint64 (bigint from koffi) but stay well under 2^53 for any +// real device, so num() narrows them losslessly. +function deviceFromRaw(dev: any): BackendInfo { + return { + name: dev.name ?? "", + description: dev.description ?? "", + kind: dev.kind ?? "", + deviceType: DEVICE_TYPE_NAMES[dev.device_type] ?? "gpu", + deviceId: dev.device_id ?? null, + memoryTotal: num(dev.memory_total), + memoryFree: num(dev.memory_free), + }; +} + export function getAvailableBackends(): BackendInfo[] { const n = native(); const count = n.F.backendDeviceCount(); @@ -276,7 +299,7 @@ export function getAvailableBackends(): BackendInfo[] { const dev: any = {}; n.F.backendDeviceInit(dev); check(n, n.F.getBackendDevice(i, dev), `reading backend device ${i}`); - out.push({ name: dev.name ?? "", description: dev.description ?? "", kind: dev.kind ?? "" }); + out.push(deviceFromRaw(dev)); } return out; } @@ -1172,6 +1195,16 @@ export class TranscribeModel { return this.#n.F.modelBackend(this.handle) ?? ""; } + /** The compute device this model is running on. `memoryFree` is a live + * snapshot, so read this again to poll how much device memory is left + * after the model loaded. */ + get device(): BackendInfo { + const dev: any = {}; + this.#n.F.backendDeviceInit(dev); + check(this.#n, this.#n.F.modelGetDevice(this.handle, dev), "reading model device"); + return deviceFromRaw(dev); + } + dispose(): void { if (this.#disposed) return; this.#disposed = true; diff --git a/bindings/typescript/src/types.ts b/bindings/typescript/src/types.ts index efa7923f..56f425e8 100644 --- a/bindings/typescript/src/types.ts +++ b/bindings/typescript/src/types.ts @@ -82,10 +82,24 @@ export interface TranscriptionResult { truncated: boolean; } +/** Vendor-agnostic device class, orthogonal to {@link BackendInfo.kind}. */ +export type DeviceType = "cpu" | "gpu" | "igpu" | "accel"; + export interface BackendInfo { name: string; description: string; kind: string; + /** The CPU/GPU/IGPU/ACCEL axis, orthogonal to `kind`. */ + deviceType: DeviceType; + /** Stable hardware id (PCI bus id) when the backend reports one, else null + * (e.g. Metal). */ + deviceId: string | null; + /** Reported device memory capacity in bytes, or 0 if unreported. */ + memoryTotal: number; + /** Available device memory in bytes — a snapshot at query time, or 0 if + * unreported. Re-query (via {@link getAvailableBackends} or `model.device`) + * to refresh; backend-defined and not comparable across device kinds. */ + memoryFree: number; } export interface ModelOptions { diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 7be177c5..19a2f23c 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -124,12 +124,14 @@ struct cli_args { // 0/1 keeps the per-file serial loop. bool translate = false; bool quiet = false; + bool list_devices = false; // --list-devices: print devices and exit bool batch_jsonl = false; // --batch-jsonl: output JSONL int repeat = 1; int n_threads = 0; // 0 = library default (all cores) int n_ctx = 0; // 0 = model's true max; >0 lowers the cap transcribe_kv_type kv_type = TRANSCRIBE_KV_TYPE_AUTO; transcribe_backend_request backend = TRANSCRIBE_BACKEND_AUTO; + int gpu_device = 0; // --device N: 0 = auto, >0 = registry index transcribe_timestamp_kind timestamps = TRANSCRIBE_TIMESTAMPS_NONE; // Whisper-family knobs. Ignored for non-Whisper models. @@ -204,6 +206,8 @@ void print_usage(const char * argv0) { " max audio.\n" " --kv-type TYPE flash-attn KV type: auto, f32, f16 (default: auto)\n" " --backend TYPE compute backend: auto, cpu, cpu_accel, metal, vulkan (default: auto)\n" + " --device N GPU device index from --list-devices: 0 = auto\n" + " (first of kind), >0 selects that registry index\n" " --timestamps TYPE timestamps: auto, none, segment, word, token (default: none)\n" " --batch FILE batch mode: FILE has one wav path per line\n" " --batch-jsonl output one JSON line per file (for batch)\n" @@ -240,10 +244,52 @@ void print_usage(const char * argv0) { " supports_spec_decode. -1 = family default,\n" " 0 = off, > 0 = explicit K. Silently ignored\n" " by families without spec support.\n" + " --list-devices list registered compute devices (with memory)\n" + " and exit; ignores all other options\n" " -h, --help show this help\n", argv0, argv0); } +// Print every registered compute device and its live memory, then return an +// exit code. Used by --list-devices. Calls transcribe_init_backends_default() +// first so dynamic-backend builds register their modules (no-op when the +// backends are compiled in). +int list_devices_main() { + const transcribe_status st = transcribe_init_backends_default(); + if (st != TRANSCRIBE_OK) { + std::fprintf(stderr, + "warning: transcribe_init_backends_default() returned %d; " + "listing whatever registered\n", (int) st); + } + const int n = transcribe_backend_device_count(); + if (n <= 0) { + std::fprintf(stderr, "no compute devices registered\n"); + return EXIT_FAILURE; + } + std::printf("%d compute device(s):\n", n); + for (int i = 0; i < n; ++i) { + struct transcribe_backend_device d; + transcribe_backend_device_init(&d); + if (transcribe_get_backend_device(i, &d) != TRANSCRIBE_OK) { + continue; + } + const char * type_str = + d.device_type == TRANSCRIBE_DEVICE_TYPE_CPU ? "cpu" : + d.device_type == TRANSCRIBE_DEVICE_TYPE_GPU ? "gpu" : + d.device_type == TRANSCRIBE_DEVICE_TYPE_IGPU ? "igpu" : + d.device_type == TRANSCRIBE_DEVICE_TYPE_ACCEL ? "accel" : "?"; + const double gib = 1024.0 * 1024.0 * 1024.0; + std::printf(" [%d] %s\n", i, + (d.description && *d.description) ? d.description : d.name); + std::printf(" name=%s kind=%s type=%s id=%s\n", + d.name ? d.name : "?", d.kind ? d.kind : "?", type_str, + (d.device_id && *d.device_id) ? d.device_id : "(none)"); + std::printf(" memory: %.2f GiB total, %.2f GiB free\n", + (double) d.memory_total / gib, (double) d.memory_free / gib); + } + return EXIT_SUCCESS; +} + bool parse_args(int argc, char ** argv, cli_args & out) { for (int i = 1; i < argc; ++i) { const std::string a = argv[i]; @@ -258,6 +304,8 @@ bool parse_args(int argc, char ** argv, cli_args & out) { if (a == "-h" || a == "--help") { print_usage(argv[0]); std::exit(0); + } else if (a == "--list-devices") { + out.list_devices = true; } else if (a == "-m" || a == "--model") { const char * v = take_value(a.c_str()); if (!v) return false; @@ -318,6 +366,14 @@ bool parse_args(int argc, char ** argv, cli_args & out) { std::fprintf(stderr, "error: --backend must be auto, cpu, cpu_accel, metal, vulkan, or cuda\n"); return false; } + } else if (a == "--device") { + const char * v = take_value(a.c_str()); + if (!v) return false; + out.gpu_device = std::atoi(v); + if (out.gpu_device < 0) { + std::fprintf(stderr, "error: --device must be >= 0 (0 = auto)\n"); + return false; + } } else if (a == "--timestamps") { const char * v = take_value(a.c_str()); if (!v) return false; @@ -437,6 +493,11 @@ bool parse_args(int argc, char ** argv, cli_args & out) { out.wav_path = a; } } + // --list-devices is a standalone query handled before any audio is + // needed, so skip the audio-input requirement for it. + if (out.list_devices) { + return true; + } if (out.wav_path.empty() && out.batch_file.empty()) { std::fprintf(stderr, "error: missing audio.wav or --batch\n"); return false; @@ -477,6 +538,15 @@ int main(int argc, char ** argv) { return EXIT_FAILURE; } + // Device listing is a standalone query: no model, no audio. Honor it + // before any other setup so `--list-devices` works on its own. + if (args.list_devices) { + if (!args.quiet) { + transcribe_log_set(log_cb, nullptr); + } + return list_devices_main(); + } + // Install the log sink ONCE at startup, before any models or contexts // exist. This is the only supported usage model in 0.x; see the // threading contract in transcribe.h. @@ -529,6 +599,7 @@ int main(int argc, char ** argv) { // Load model once. struct transcribe_model_load_params mp; transcribe_model_load_params_init(&mp); mp.backend = args.backend; + mp.gpu_device = args.gpu_device; struct transcribe_model * model = nullptr; const transcribe_status load_st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); @@ -888,6 +959,7 @@ int main(int argc, char ** argv) { if (!args.model_path.empty()) { struct transcribe_model_load_params mp; transcribe_model_load_params_init(&mp); mp.backend = args.backend; + mp.gpu_device = args.gpu_device; struct transcribe_model * model = nullptr; const transcribe_status st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); diff --git a/include/transcribe.abihash b/include/transcribe.abihash index 5778a476..3d27bb16 100644 --- a/include/transcribe.abihash +++ b/include/transcribe.abihash @@ -1 +1 @@ -2273744299e5aa65 +ebe6a6816e34a24e diff --git a/include/transcribe.h b/include/transcribe.h index efac6a73..278fc46a 100644 --- a/include/transcribe.h +++ b/include/transcribe.h @@ -774,21 +774,54 @@ TRANSCRIBE_API transcribe_status transcribe_init_backends_default(void); */ TRANSCRIBE_API int transcribe_backend_device_count(void); +/* + * Device type: the vendor-agnostic ggml classification of a device, + * orthogonal to `kind` below (which carries the vendor: metal/vulkan/cuda/ + * ...). Use this to tell a discrete GPU from an integrated one, or a + * host-memory accelerator from the CPU. The numeric values mirror ggml's + * device-type enum. + */ +typedef enum { + TRANSCRIBE_DEVICE_TYPE_CPU = 0, /* CPU using system memory */ + TRANSCRIBE_DEVICE_TYPE_GPU = 1, /* discrete GPU with dedicated memory */ + TRANSCRIBE_DEVICE_TYPE_IGPU = 2, /* integrated GPU using host memory */ + TRANSCRIBE_DEVICE_TYPE_ACCEL = 3, /* host-memory accelerator (BLAS/AMX) */ +} transcribe_device_type; + /* * One registered compute device. * - * name / description / kind are borrowed pointers into runtime-owned - * storage: valid for the life of the process, never freed by the caller. + * name / description / kind / device_id are borrowed pointers into + * runtime-owned storage: valid for the life of the process, never freed by + * the caller. * - * kind is the library's classification, one of: "cpu", "accel" (a + * kind is the library's vendor classification, one of: "cpu", "accel" (a * host-memory accelerator such as BLAS/AMX), "metal", "vulkan", "cuda", - * "sycl", "gpu" (an unrecognized GPU), or "unknown". + * "sycl", "gpu" (an unrecognized GPU), or "unknown". device_type is the + * orthogonal CPU/GPU/IGPU/ACCEL axis. + * + * device_id is a stable hardware identifier when the backend reports one + * (for PCI devices the lower-case bus id "domain:bus:device.function", e.g. + * "0000:c1:00.0"), or NULL when unknown (e.g. Metal). + * + * memory_total is the device's reported capacity in bytes. memory_free is a + * SNAPSHOT of available bytes at the moment this struct was filled; it goes + * stale the instant anything allocates — re-call the accessor to refresh it, + * as every fill re-queries the driver live. Both numbers are backend-defined + * and NOT comparable across kinds: on Apple unified memory `total` is the + * recommended max working-set size (not system RAM) and `free` nets out only + * this process's allocations; on a discrete GPU they are device-global; on + * the CPU they are system RAM. 0 means the backend does not report it. */ struct transcribe_backend_device { - uint64_t struct_size; /* sizeof(*this); set by _init() */ - const char * name; /* ggml device name, e.g. "Metal" */ - const char * description; /* human-readable, e.g. "Apple M4 Max" */ - const char * kind; /* classified kind string; see above */ + uint64_t struct_size; /* sizeof(*this); set by _init() */ + const char * name; /* ggml device name, e.g. "Metal" */ + const char * description; /* human-readable, e.g. "Apple M4 Max" */ + const char * kind; /* vendor kind string; see above */ + const char * device_id; /* stable hw id (PCI bus id) or NULL */ + uint64_t memory_total; /* reported capacity in bytes, or 0 */ + uint64_t memory_free; /* available bytes snapshot, or 0 */ + transcribe_device_type device_type; /* CPU/GPU/IGPU/ACCEL axis */ }; TRANSCRIBE_API void transcribe_backend_device_init( @@ -797,6 +830,11 @@ TRANSCRIBE_API void transcribe_backend_device_init( /* * Fill *out (initialized via transcribe_backend_device_init) with device * `index` in [0, transcribe_backend_device_count()). + * + * memory_free is live as of this call; re-invoke to refresh it (e.g. to + * poll a device's available memory over time). The device handles are + * stable for the life of the process, so the same index always names the + * same device. */ TRANSCRIBE_API transcribe_status transcribe_get_backend_device( int index, @@ -813,6 +851,22 @@ TRANSCRIBE_API transcribe_status transcribe_get_backend_device( TRANSCRIBE_API bool transcribe_backend_available( transcribe_backend_request kind); +/* + * Fill *out (initialized via transcribe_backend_device_init) with the + * compute device this loaded model is running on — the device that owns its + * weights and runs most of its graph. Same struct and same live-snapshot + * semantics as transcribe_get_backend_device: memory_free is current as of + * the call, so re-invoke to ask "how much memory is left on the device my + * model landed on" at any time after load. + * + * Returns TRANSCRIBE_ERR_INVALID_ARG if model or out is NULL (or out fails + * the struct-size check), or TRANSCRIBE_ERR_BACKEND if the model has no + * resolved compute device. + */ +TRANSCRIBE_API transcribe_status transcribe_model_get_device( + const struct transcribe_model * model, + struct transcribe_backend_device * out); + /* * Initialization of caller-owned params structs. * @@ -843,13 +897,29 @@ TRANSCRIBE_API bool transcribe_backend_available( * backend: which backend to request. See transcribe_backend_request * for the semantics of each value. Default is AUTO. * - * gpu_device: Reserved for future multi-device selection. 0 means - * "auto / the first device of the chosen kind" and is the - * default; any other value returns TRANSCRIBE_ERR_INVALID_ARG - * in 0.x. AUTO always picks the first device of the chosen - * kind in ggml's registry order; explicit METAL/VULKAN - * requests likewise pick the first matching device. There is - * no per-device selection in the current release. + * gpu_device: Multi-GPU selector. 0 (the default) means "auto / the first + * device of the chosen kind": AUTO picks the first GPU/IGPU in + * ggml's registry order, and explicit METAL/VULKAN/CUDA requests + * pick the first matching device, as before. + * + * A value > 0 selects the GPU/IGPU device at that global ggml + * registry index — the same index space transcribe_get_backend_device() + * enumerates, so enumerate first to choose one. The selected + * device becomes the model's primary backend, validated against + * `backend`: it must be a GPU/IGPU, and for an explicit + * METAL/VULKAN/CUDA request it must be that vendor. The index is + * order-dependent — ggml's registry order can shift across driver + * updates or hosts, so treat it as a runtime selection, not a + * stable identifier; correlate via the enumerated device's name / + * device_id when you need stability. + * + * gpu_device is rejected with TRANSCRIBE_ERR_INVALID_ARG when it + * is negative, out of range, names a non-GPU device, names a + * device whose vendor doesn't match an explicit GPU request, or + * is non-zero alongside a CPU / CPU_ACCEL request (there is no + * GPU to select). Note there is no way to explicitly select the + * device at registry index 0; that is exactly what AUTO / + * first-of-kind already picks. */ struct transcribe_model_load_params { uint64_t struct_size; diff --git a/src/arch/canary/model.cpp b/src/arch/canary/model.cpp index ddeb81db..4f69dc75 100644 --- a/src/arch/canary/model.cpp +++ b/src/arch/canary/model.cpp @@ -548,7 +548,7 @@ transcribe_status load( (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, "canary", m->plan); + backend_req, (params != nullptr) ? params->gpu_device : 0, "canary", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); @@ -556,6 +556,7 @@ transcribe_status load( } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/arch/canary_qwen/model.cpp b/src/arch/canary_qwen/model.cpp index 07b1d166..4e81ffab 100644 --- a/src/arch/canary_qwen/model.cpp +++ b/src/arch/canary_qwen/model.cpp @@ -707,13 +707,14 @@ transcribe_status load( // ---- Backend plan + alloc + stream tensor data ---- const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; - if (auto st = load_common::init_backends(backend_req, "canary_qwen", m->plan); + if (auto st = load_common::init_backends(backend_req, (params != nullptr) ? params->gpu_device : 0, "canary_qwen", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/arch/cohere/model.cpp b/src/arch/cohere/model.cpp index b6c95b68..5259c4fe 100644 --- a/src/arch/cohere/model.cpp +++ b/src/arch/cohere/model.cpp @@ -674,7 +674,7 @@ transcribe_status load( (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, "cohere", m->plan); + backend_req, (params != nullptr) ? params->gpu_device : 0, "cohere", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); @@ -682,6 +682,7 @@ transcribe_status load( } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/arch/funasr_nano/model.cpp b/src/arch/funasr_nano/model.cpp index ad91b60d..fd18d5ab 100644 --- a/src/arch/funasr_nano/model.cpp +++ b/src/arch/funasr_nano/model.cpp @@ -400,12 +400,13 @@ transcribe_status load( const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (auto st = transcribe::load_common::init_backends( - backend_req, "funasr_nano", m->plan); st != TRANSCRIBE_OK) + backend_req, (params != nullptr) ? params->gpu_device : 0, "funasr_nano", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/arch/gigaam/model.cpp b/src/arch/gigaam/model.cpp index b31716ab..60853af3 100644 --- a/src/arch/gigaam/model.cpp +++ b/src/arch/gigaam/model.cpp @@ -157,13 +157,14 @@ transcribe_status load(Loader & loader, const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (auto st = transcribe::load_common::init_backends( - backend_req, "gigaam", m->plan); + backend_req, (params != nullptr) ? params->gpu_device : 0, "gigaam", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/arch/granite/model.cpp b/src/arch/granite/model.cpp index 67d43479..2662c65b 100644 --- a/src/arch/granite/model.cpp +++ b/src/arch/granite/model.cpp @@ -432,13 +432,14 @@ transcribe_status load( const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, "granite", m->plan); + backend_req, (params != nullptr) ? params->gpu_device : 0, "granite", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/arch/granite_nar/model.cpp b/src/arch/granite_nar/model.cpp index 500eab57..bd180f7c 100644 --- a/src/arch/granite_nar/model.cpp +++ b/src/arch/granite_nar/model.cpp @@ -366,13 +366,14 @@ transcribe_status load( const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, "granite_nar", m->plan); + backend_req, (params != nullptr) ? params->gpu_device : 0, "granite_nar", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/arch/medasr/model.cpp b/src/arch/medasr/model.cpp index 47fb1df3..33dc350d 100644 --- a/src/arch/medasr/model.cpp +++ b/src/arch/medasr/model.cpp @@ -192,13 +192,14 @@ transcribe_status load(Loader & loader, const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (auto st = transcribe::load_common::init_backends( - backend_req, "medasr", m->plan); + backend_req, (params != nullptr) ? params->gpu_device : 0, "medasr", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/arch/moonshine/model.cpp b/src/arch/moonshine/model.cpp index 9b6414a2..3578884a 100644 --- a/src/arch/moonshine/model.cpp +++ b/src/arch/moonshine/model.cpp @@ -270,12 +270,13 @@ transcribe_status load( const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (auto st = transcribe::load_common::init_backends( - backend_req, "moonshine", m->plan); st != TRANSCRIBE_OK) + backend_req, (params != nullptr) ? params->gpu_device : 0, "moonshine", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/arch/moonshine_streaming/model.cpp b/src/arch/moonshine_streaming/model.cpp index 19e40615..f1853739 100644 --- a/src/arch/moonshine_streaming/model.cpp +++ b/src/arch/moonshine_streaming/model.cpp @@ -287,12 +287,13 @@ transcribe_status load( const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (auto st = transcribe::load_common::init_backends( - backend_req, "moonshine_streaming", m->plan); st != TRANSCRIBE_OK) + backend_req, (params != nullptr) ? params->gpu_device : 0, "moonshine_streaming", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/arch/parakeet/model.cpp b/src/arch/parakeet/model.cpp index d00af67b..87f0ec92 100644 --- a/src/arch/parakeet/model.cpp +++ b/src/arch/parakeet/model.cpp @@ -689,7 +689,7 @@ transcribe_status load( (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, "parakeet", m->plan); + backend_req, (params != nullptr) ? params->gpu_device : 0, "parakeet", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); @@ -698,6 +698,7 @@ transcribe_status load( // Label for the public API: report the primary backend. m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; // Allocate a backend buffer for every tensor in ctx_meta on the // primary backend. After this returns, each ggml_tensor in diff --git a/src/arch/qwen3_asr/model.cpp b/src/arch/qwen3_asr/model.cpp index 217d75af..67ee8647 100644 --- a/src/arch/qwen3_asr/model.cpp +++ b/src/arch/qwen3_asr/model.cpp @@ -287,13 +287,14 @@ transcribe_status load( const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, "qwen3_asr", m->plan); + backend_req, (params != nullptr) ? params->gpu_device : 0, "qwen3_asr", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/arch/sensevoice/model.cpp b/src/arch/sensevoice/model.cpp index 96e5b2a3..f12532bf 100644 --- a/src/arch/sensevoice/model.cpp +++ b/src/arch/sensevoice/model.cpp @@ -153,12 +153,13 @@ transcribe_status load( const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (auto st = transcribe::load_common::init_backends( - backend_req, "sensevoice", m->plan); st != TRANSCRIBE_OK) + backend_req, (params != nullptr) ? params->gpu_device : 0, "sensevoice", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/arch/voxtral/model.cpp b/src/arch/voxtral/model.cpp index 912f17d9..6dcd9f0b 100644 --- a/src/arch/voxtral/model.cpp +++ b/src/arch/voxtral/model.cpp @@ -422,13 +422,14 @@ transcribe_status load( const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, "voxtral", m->plan); + backend_req, (params != nullptr) ? params->gpu_device : 0, "voxtral", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/arch/voxtral_realtime/model.cpp b/src/arch/voxtral_realtime/model.cpp index 56bf63ed..e6c9c980 100644 --- a/src/arch/voxtral_realtime/model.cpp +++ b/src/arch/voxtral_realtime/model.cpp @@ -276,12 +276,13 @@ transcribe_status load(Loader & loader, const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; - if (auto st = transcribe::load_common::init_backends(backend_req, "voxtral_realtime", m->plan); + if (auto st = transcribe::load_common::init_backends(backend_req, (params != nullptr) ? params->gpu_device : 0, "voxtral_realtime", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/arch/whisper/bin_load.cpp b/src/arch/whisper/bin_load.cpp index 73de5d98..30c90bd8 100644 --- a/src/arch/whisper/bin_load.cpp +++ b/src/arch/whisper/bin_load.cpp @@ -686,12 +686,13 @@ transcribe_status load_from_bin(const char * path, // ---- Backend plan ---- if (auto st = transcribe::load_common::init_backends( - params->backend, "whisper", m->plan); + params->backend, params->gpu_device, "whisper", m->plan); st != TRANSCRIBE_OK) { return st; } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; // ---- Allocate backend buffer ---- ggml_backend_buffer_t buf = diff --git a/src/arch/whisper/model.cpp b/src/arch/whisper/model.cpp index 40b8d6ce..172d6b90 100644 --- a/src/arch/whisper/model.cpp +++ b/src/arch/whisper/model.cpp @@ -532,13 +532,14 @@ transcribe_status whisper_load( const transcribe_backend_request backend_req = (params != nullptr) ? params->backend : TRANSCRIBE_BACKEND_AUTO; if (const transcribe_status st = transcribe::load_common::init_backends( - backend_req, "whisper", m->plan); + backend_req, (params != nullptr) ? params->gpu_device : 0, "whisper", m->plan); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; } m->backend = ggml_backend_name(m->plan.primary); + m->primary_backend = m->plan.primary; ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); diff --git a/src/transcribe-load-common.cpp b/src/transcribe-load-common.cpp index 8e1914c1..fe4cf5ad 100644 --- a/src/transcribe-load-common.cpp +++ b/src/transcribe-load-common.cpp @@ -121,9 +121,84 @@ bool valid_backend_request(int raw) { return false; } +// Resolve a BackendPlan for an explicit device selection (gpu_device > 0). +// `dev_index` is a global ggml registry index. The selected device becomes +// the primary; `requested` constrains what kind it must be. Only GPU/IGPU +// devices are selectable this way — strict/accel CPU requests reject a +// non-zero gpu_device before reaching here. The assembled plan mirrors the +// specific-GPU path: primary GPU, then ACCEL, then CPU last as the fallback. +transcribe_status init_backends_explicit_index(transcribe_backend_request requested, + int dev_index, + const char * error_tag, + BackendPlan & out) +{ + const size_t n = ggml_backend_dev_count(); + if (dev_index < 0 || static_cast(dev_index) >= n) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "%s: gpu_device %d out of range [0, %zu)", + error_tag, dev_index, n); + return TRANSCRIBE_ERR_INVALID_ARG; + } + + ggml_backend_dev_t dev = ggml_backend_dev_get(static_cast(dev_index)); + const auto dev_type = ggml_backend_dev_type(dev); + if (dev_type != GGML_BACKEND_DEVICE_TYPE_GPU && + dev_type != GGML_BACKEND_DEVICE_TYPE_IGPU) + { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "%s: gpu_device %d (%s) is not a GPU device", + error_tag, dev_index, ggml_backend_dev_name(dev)); + return TRANSCRIBE_ERR_INVALID_ARG; + } + + const BackendKind got = classify_device(dev); + + // A specific vendor request pins the kind; AUTO accepts any GPU. + BackendKind wanted = BackendKind::Unknown; // Unknown == "any GPU" (AUTO) + switch (requested) { + case TRANSCRIBE_BACKEND_METAL: wanted = BackendKind::Metal; break; + case TRANSCRIBE_BACKEND_VULKAN: wanted = BackendKind::Vulkan; break; + case TRANSCRIBE_BACKEND_CUDA: wanted = BackendKind::Cuda; break; + case TRANSCRIBE_BACKEND_AUTO: break; + default: + // CPU / CPU_ACCEL never reach here (caller rejects nonzero + // gpu_device for them); anything else is a programming error. + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (wanted != BackendKind::Unknown && got != wanted) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "%s: gpu_device %d is a %s device but %s was requested", + error_tag, dev_index, kind_name(got), kind_name(wanted)); + return TRANSCRIBE_ERR_INVALID_ARG; + } + + ggml_backend_t gpu_be = ggml_backend_dev_init(dev, nullptr); + if (gpu_be == nullptr) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "%s: failed to initialize gpu_device %d (%s)", + error_tag, dev_index, ggml_backend_dev_name(dev)); + return TRANSCRIBE_ERR_BACKEND; + } + log_msg(TRANSCRIBE_LOG_LEVEL_INFO, + "%s: using %s backend (gpu_device %d): %s", + error_tag, kind_name(got), dev_index, ggml_backend_dev_name(dev)); + + out.primary = gpu_be; + out.primary_kind = got; + out.scheduler_list.push_back(gpu_be); + + append_accel_backends(out.scheduler_list, error_tag); + + ggml_backend_t cpu_be = init_cpu_backend(error_tag); + if (cpu_be == nullptr) return TRANSCRIBE_ERR_BACKEND; + out.scheduler_list.push_back(cpu_be); + return TRANSCRIBE_OK; +} + } // namespace transcribe_status init_backends(transcribe_backend_request requested, + int gpu_device, const char * error_tag, BackendPlan & out) { @@ -139,6 +214,35 @@ transcribe_status init_backends(transcribe_backend_request requested, valid_backend_request(requested_raw) ? requested_raw : TRANSCRIBE_BACKEND_AUTO); + // Explicit device selection. 0 is "auto / first of kind" and falls + // through to the per-request logic below; a negative index is always + // invalid; a positive index pins a specific GPU/IGPU device. + if (gpu_device < 0) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "%s: gpu_device must be >= 0 (got %d)", + error_tag, gpu_device); + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (gpu_device > 0) { + if (!valid_backend_request(requested_raw)) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "%s: invalid transcribe_backend_request value %d", + error_tag, requested_raw); + return TRANSCRIBE_ERR_INVALID_ARG; + } + // gpu_device names a GPU; a CPU-only request has nothing to select. + if (requested_raw == TRANSCRIBE_BACKEND_CPU || + requested_raw == TRANSCRIBE_BACKEND_CPU_ACCEL) + { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "%s: gpu_device %d is invalid for a CPU backend request", + error_tag, gpu_device); + return TRANSCRIBE_ERR_INVALID_ARG; + } + return init_backends_explicit_index(out.requested, gpu_device, + error_tag, out); + } + // Explicit switch over the enum so an unknown / garbage value // from a C caller never silently collapses into AUTO. Unknown // values are a programming error on the caller's side, not a diff --git a/src/transcribe-load-common.h b/src/transcribe-load-common.h index 8f1a4832..c8ebf81b 100644 --- a/src/transcribe-load-common.h +++ b/src/transcribe-load-common.h @@ -48,6 +48,16 @@ namespace transcribe::load_common { // Library-internal classification of what each value // actually picks up is documented in // transcribe-backend.h. +// gpu_device: multi-GPU selector. 0 (the default) means "auto / the +// first device of the chosen kind" — the existing +// first-of-kind behavior. A value > 0 selects the GPU/IGPU +// device at that global ggml registry index (the same index +// space transcribe_get_backend_device() enumerates) as the +// primary, validated against `requested`: the device must be +// a GPU/IGPU, and for a specific METAL/VULKAN/CUDA request it +// must be that vendor. gpu_device is not valid for a +// CPU / CPU_ACCEL request (there is no GPU to pick) nor +// negative; both return TRANSCRIBE_ERR_INVALID_ARG. // error_tag: log prefix, e.g. "parakeet". // out: populated on success. out.primary is the backend // that owns the weight buffer; out.scheduler_list @@ -57,12 +67,17 @@ namespace transcribe::load_common { // // Returns: // TRANSCRIBE_OK on success. +// TRANSCRIBE_ERR_INVALID_ARG if gpu_device is negative, out of range, +// names a non-GPU device, names a device whose +// vendor doesn't match a specific GPU request, or +// is non-zero for a CPU request. // TRANSCRIBE_ERR_BACKEND if the caller asked for a specific // backend (METAL / VULKAN) that could not // be initialized, or if the CPU backend // itself fails to initialize (there is no // fallback past CPU). transcribe_status init_backends(transcribe_backend_request requested, + int gpu_device, const char * error_tag, BackendPlan & out); diff --git a/src/transcribe-model.h b/src/transcribe-model.h index 25af4b87..185cfd89 100644 --- a/src/transcribe-model.h +++ b/src/transcribe-model.h @@ -37,6 +37,11 @@ struct Arch; class Tokenizer; } // namespace transcribe +// Forward declaration for the resolved primary backend handle stored below. +// The full type comes from ggml-backend.h in the .cpp that fills it. +struct ggml_backend; +typedef struct ggml_backend * ggml_backend_t; + // The public C ABI forward-declares this as `struct transcribe_model;`, // so the real definition stays in the global namespace and uses the // `struct` keyword for ABI compatibility with C callers. @@ -58,6 +63,14 @@ struct transcribe_model { // this empty. std::string backend; + // The resolved primary compute backend this model runs on (the handle + // that owns the weight buffer). Set by per-family load() right where it + // sets `backend`, from BackendPlan::primary. Used by the public + // transcribe_model_get_device() accessor to recover the device — and its + // live memory — without exposing the per-family BackendPlan. nullptr + // until a family binds it. + ggml_backend_t primary_backend = nullptr; + // Public capabilities. Per-family load() fills this in directly, // calling set_languages() for the languages chain. Immutable after // a successful load. Zero-initialized here; the capabilities diff --git a/src/transcribe.cpp b/src/transcribe.cpp index 4b151384..55d619f3 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -891,6 +891,44 @@ extern "C" transcribe_status transcribe_init_backends_default(void) { #endif } +namespace { + +// Map ggml's device-type enum onto the public transcribe_device_type. GPU +// and any unexpected type (e.g. the META tensor-parallel aggregate, which we +// never construct) collapse to GPU; the dedicated CPU/IGPU/ACCEL cases are +// reported faithfully. +transcribe_device_type to_device_type(enum ggml_backend_dev_type t) { + switch (t) { + case GGML_BACKEND_DEVICE_TYPE_CPU: return TRANSCRIBE_DEVICE_TYPE_CPU; + case GGML_BACKEND_DEVICE_TYPE_IGPU: return TRANSCRIBE_DEVICE_TYPE_IGPU; + case GGML_BACKEND_DEVICE_TYPE_ACCEL: return TRANSCRIBE_DEVICE_TYPE_ACCEL; + case GGML_BACKEND_DEVICE_TYPE_GPU: + default: return TRANSCRIBE_DEVICE_TYPE_GPU; + } +} + +// Fill a caller-owned transcribe_backend_device from a ggml device, honoring +// the caller's declared struct_size via copy_out_prefix. ggml_backend_dev_get_props +// queries memory live, so every call observes a fresh memory_free snapshot. +void fill_backend_device(ggml_backend_dev_t dev, uint64_t caller_size, + struct transcribe_backend_device * out) { + ggml_backend_dev_props props{}; + ggml_backend_dev_get_props(dev, &props); + + struct transcribe_backend_device staged{}; + staged.struct_size = caller_size; + staged.name = ggml_backend_dev_name(dev); + staged.description = ggml_backend_dev_description(dev); + staged.kind = transcribe::kind_name(transcribe::classify_device(dev)); + staged.device_id = props.device_id; + staged.memory_total = props.memory_total; + staged.memory_free = props.memory_free; + staged.device_type = to_device_type(props.type); + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); +} + +} // namespace + extern "C" int transcribe_backend_device_count(void) { return static_cast(ggml_backend_dev_count()); } @@ -911,9 +949,7 @@ extern "C" transcribe_status transcribe_get_backend_device( return TRANSCRIBE_ERR_INVALID_ARG; } ggml_backend_dev_t dev = ggml_backend_dev_get(static_cast(index)); - out->name = ggml_backend_dev_name(dev); - out->description = ggml_backend_dev_description(dev); - out->kind = transcribe::kind_name(transcribe::classify_device(dev)); + fill_backend_device(dev, out->struct_size, out); return TRANSCRIBE_OK; } @@ -1371,22 +1407,11 @@ extern "C" transcribe_status transcribe_model_load_file( return st; } - // Reserved-field validation. gpu_device is documented in the public - // header as 0 = auto/default in 0.x, reserved for future multi-device - // selection. 0 is the zero value so a {0} / default-initialized struct - // passes; reject anything else now so that callers who pass a stale - // explicit device index (or garbage) get a clean error today rather - // than a silent success followed by surprise behavior when we actually - // wire device selection up. A stderr line is included because "invalid - // argument" alone doesn't tell the caller which field tripped. When - // multi-device support lands this check relaxes to `< 0 || >= n_devices`. - if (params->gpu_device != 0) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, - "transcribe_model_load_file: gpu_device must be 0 (auto) in 0.x " - "(got %d); multi-device selection is reserved for a " - "future release", params->gpu_device); - return TRANSCRIBE_ERR_INVALID_ARG; - } + // gpu_device is validated where the device registry is available — in + // load_common::init_backends, which each family calls. 0 means auto/first + // of kind; a positive index selects a specific GPU; negative / out of + // range / kind-mismatched values return TRANSCRIBE_ERR_INVALID_ARG from + // there. See the public header and transcribe-load-common.h. // Raw-validate the backend request before the families' first // enum-typed load of it (see enum_field_raw). init_backends re-checks @@ -2662,6 +2687,31 @@ extern "C" const char * transcribe_model_backend(const struct transcribe_model * return model->backend.c_str(); } +extern "C" transcribe_status transcribe_model_get_device( + const struct transcribe_model * model, + struct transcribe_backend_device * out) +{ + if (model == nullptr || out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_backend_device_size); + st != TRANSCRIBE_OK) + { + return st; + } + // The model's primary backend is bound by per-family load(); a model + // that never resolved one (or a 2B build with no real backend) has none. + if (model->primary_backend == nullptr) { + return TRANSCRIBE_ERR_BACKEND; + } + ggml_backend_dev_t dev = ggml_backend_get_device(model->primary_backend); + if (dev == nullptr) { + return TRANSCRIBE_ERR_BACKEND; + } + fill_backend_device(dev, out->struct_size, out); + return TRANSCRIBE_OK; +} + // --------------------------------------------------------------------------- // Timings // --------------------------------------------------------------------------- diff --git a/tests/api_smoke.c b/tests/api_smoke.c index 510c4e09..0a3566a4 100644 --- a/tests/api_smoke.c +++ b/tests/api_smoke.c @@ -430,15 +430,18 @@ static void test_load_invalid(void) { == TRANSCRIBE_ERR_BAD_STRUCT_SIZE); CHECK(m == NULL); - /* gpu_device must be 0 (auto) in 0.x; any other value -> INVALID_ARG, - * checked before the file is touched. */ + /* gpu_device selection is validated during load against the live device + * registry (in load_common::init_backends), not as an upfront reserved- + * field check — so a nonzero gpu_device no longer short-circuits to + * INVALID_ARG before the file is even opened. A missing file still + * surfaces as FILE_NOT_FOUND regardless of gpu_device. */ struct transcribe_model_load_params mp_dev; transcribe_model_load_params_init(&mp_dev); mp_dev.gpu_device = 1; m = (struct transcribe_model *)0xdeadbeef; CHECK(transcribe_model_load_file("/__transcribe_smoke_does_not_exist__.gguf", &mp_dev, &m) - == TRANSCRIBE_ERR_INVALID_ARG); + == TRANSCRIBE_ERR_FILE_NOT_FOUND); CHECK(m == NULL); /* Otherwise-valid call against a path that does not exist on disk diff --git a/tools/transcribe-bench/main.cpp b/tools/transcribe-bench/main.cpp index 6bbe87c5..b9453e44 100644 --- a/tools/transcribe-bench/main.cpp +++ b/tools/transcribe-bench/main.cpp @@ -37,6 +37,7 @@ struct bench_args { int n_threads = 0; bool quiet = false; transcribe_backend_request backend = TRANSCRIBE_BACKEND_AUTO; + int gpu_device = 0; // --device N: 0 = auto, >0 = index // Passed through to transcribe_run_params::spec_k_drafts. -1 = family // default, 0 = spec decode off, > 0 = explicit draft length. Silently // ignored by families without supports_spec_decode. Set by @@ -60,6 +61,8 @@ void print_usage(const char * argv0) { " cpu is strict CPU (no GPU, no BLAS/AMX).\n" " cpu_accel is CPU + host-memory accelerators\n" " (BLAS/AMX) when the build includes them.\n" + " --device N GPU device index: 0 = auto (first of kind),\n" + " >0 selects that ggml registry index\n" " --spec-k-drafts N speculative-decode draft length on the offline\n" " path: -1 = family default, 0 = off, > 0 = K.\n" " Ignored by families without spec support.\n" @@ -109,6 +112,11 @@ bool parse_args(int argc, char ** argv, bench_args & out) { auto v = need_val(i, "--backend"); if (!v) return false; if (!parse_backend_kind(v, out.backend)) return false; } + else if (a == "--device") { + auto v = need_val(i, "--device"); if (!v) return false; + out.gpu_device = std::atoi(v); + if (out.gpu_device < 0) { std::fprintf(stderr, "error: --device must be >= 0 (0 = auto)\n"); return false; } + } else { std::fprintf(stderr, "error: unknown option '%s'\n", a.c_str()); return false; @@ -206,6 +214,7 @@ int main(int argc, char ** argv) { if (!quiet) std::fprintf(stderr, "loading model %s\n", args.model_path.c_str()); struct transcribe_model_load_params mp; transcribe_model_load_params_init(&mp); mp.backend = args.backend; + mp.gpu_device = args.gpu_device; struct transcribe_model * model = nullptr; if (const transcribe_status st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); From 42e94896b8778ddf026faa582f1020be3b5c6487 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 21 Jun 2026 16:41:46 +0800 Subject: [PATCH 2/2] clean up --- bindings/rust/transcribe-cpp/src/backend.rs | 11 +-- bindings/rust/transcribe-cpp/src/model.rs | 2 +- .../swift/Sources/TranscribeCpp/Backend.swift | 7 +- .../swift/Sources/TranscribeCpp/Options.swift | 2 +- bindings/typescript/src/types.ts | 2 +- examples/cli/main.cpp | 3 +- include/transcribe.h | 13 +-- src/arch/cohere/model.cpp | 5 +- src/arch/parakeet/model.cpp | 5 +- src/transcribe-backend.h | 2 +- src/transcribe.cpp | 18 ++++- tests/backend_init_unit.cpp | 79 +++++++++++++++++-- tools/transcribe-bench/main.cpp | 2 +- 13 files changed, 113 insertions(+), 38 deletions(-) diff --git a/bindings/rust/transcribe-cpp/src/backend.rs b/bindings/rust/transcribe-cpp/src/backend.rs index 46d190fd..01707d80 100644 --- a/bindings/rust/transcribe-cpp/src/backend.rs +++ b/bindings/rust/transcribe-cpp/src/backend.rs @@ -17,16 +17,17 @@ use crate::error::{check, Result}; use crate::result::{owned_opt_str, owned_str}; use crate::types::Backend; -/// The vendor-agnostic class of a compute device, orthogonal to -/// [`Device::kind`] (which carries the vendor). Distinguishes a discrete GPU -/// from an integrated one, and a host-memory accelerator from the CPU. +/// ggml's vendor-agnostic class for a compute device, orthogonal to +/// [`Device::kind`] (which carries the vendor). Backends report this +/// classification themselves, so use it as a runtime hint rather than a +/// portable hardware-memory taxonomy. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DeviceType { /// CPU using system memory. Cpu, - /// Discrete GPU with dedicated memory. + /// Backend-reported GPU. Gpu, - /// Integrated GPU using host memory. + /// Backend-reported integrated GPU. Igpu, /// Host-memory accelerator (BLAS/AMX/...). Accel, diff --git a/bindings/rust/transcribe-cpp/src/model.rs b/bindings/rust/transcribe-cpp/src/model.rs index b95945b1..3116787d 100644 --- a/bindings/rust/transcribe-cpp/src/model.rs +++ b/bindings/rust/transcribe-cpp/src/model.rs @@ -32,7 +32,7 @@ use crate::version; pub struct ModelOptions { /// Which backend to request. Default [`Backend::Auto`]. pub backend: Backend, - /// Reserved for multi-device selection; must be 0 in 0.x. + /// GPU device registry index. 0 means auto / first matching device. pub gpu_device: i32, } diff --git a/bindings/swift/Sources/TranscribeCpp/Backend.swift b/bindings/swift/Sources/TranscribeCpp/Backend.swift index fcdb5423..b4ce362a 100644 --- a/bindings/swift/Sources/TranscribeCpp/Backend.swift +++ b/bindings/swift/Sources/TranscribeCpp/Backend.swift @@ -22,9 +22,10 @@ public enum Backend: Sendable, Equatable { } } -/// The vendor-agnostic class of a compute device, orthogonal to `Device.kind` -/// (which carries the vendor). Distinguishes a discrete GPU from an integrated -/// one, and a host-memory accelerator from the CPU. +/// ggml's vendor-agnostic class for a compute device, orthogonal to +/// `Device.kind` (which carries the vendor). Backends report this classification +/// themselves, so use it as a runtime hint rather than a portable +/// hardware-memory taxonomy. public enum DeviceType: Sendable, Equatable { case cpu case gpu diff --git a/bindings/swift/Sources/TranscribeCpp/Options.swift b/bindings/swift/Sources/TranscribeCpp/Options.swift index a76e64cf..95b325ff 100644 --- a/bindings/swift/Sources/TranscribeCpp/Options.swift +++ b/bindings/swift/Sources/TranscribeCpp/Options.swift @@ -87,7 +87,7 @@ public enum Feature: Sendable { public struct ModelOptions: Sendable { public var backend: Backend - /// GPU device ordinal; must be 0 in 0.x. + /// GPU device registry index. 0 means auto / first matching device. public var gpuDevice: Int32 public init(backend: Backend = .auto, gpuDevice: Int32 = 0) { self.backend = backend diff --git a/bindings/typescript/src/types.ts b/bindings/typescript/src/types.ts index 56f425e8..43abddd5 100644 --- a/bindings/typescript/src/types.ts +++ b/bindings/typescript/src/types.ts @@ -105,7 +105,7 @@ export interface BackendInfo { export interface ModelOptions { /** "auto" (default), or an explicit backend. */ backend?: Backend; - /** GPU device ordinal for multi-GPU hosts. */ + /** GPU device registry index. 0 means auto / first matching device. */ gpuDevice?: number; } diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 19a2f23c..da5acc36 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -205,7 +205,8 @@ void print_usage(const char * argv0) { " max, >max is clamped down. Lowers the effective\n" " max audio.\n" " --kv-type TYPE flash-attn KV type: auto, f32, f16 (default: auto)\n" - " --backend TYPE compute backend: auto, cpu, cpu_accel, metal, vulkan (default: auto)\n" + " --backend TYPE compute backend: auto, cpu, cpu_accel, metal, vulkan, cuda\n" + " (default: auto)\n" " --device N GPU device index from --list-devices: 0 = auto\n" " (first of kind), >0 selects that registry index\n" " --timestamps TYPE timestamps: auto, none, segment, word, token (default: none)\n" diff --git a/include/transcribe.h b/include/transcribe.h index 278fc46a..55f91547 100644 --- a/include/transcribe.h +++ b/include/transcribe.h @@ -775,16 +775,17 @@ TRANSCRIBE_API transcribe_status transcribe_init_backends_default(void); TRANSCRIBE_API int transcribe_backend_device_count(void); /* - * Device type: the vendor-agnostic ggml classification of a device, + * Device type: ggml's vendor-agnostic classification of a device, * orthogonal to `kind` below (which carries the vendor: metal/vulkan/cuda/ - * ...). Use this to tell a discrete GPU from an integrated one, or a - * host-memory accelerator from the CPU. The numeric values mirror ggml's - * device-type enum. + * ...). Backends report this classification themselves, so treat it as a + * runtime hint about CPU/GPU/IGPU/ACCEL placement rather than a portable + * hardware-memory taxonomy. The numeric values mirror ggml's device-type + * enum. */ typedef enum { TRANSCRIBE_DEVICE_TYPE_CPU = 0, /* CPU using system memory */ - TRANSCRIBE_DEVICE_TYPE_GPU = 1, /* discrete GPU with dedicated memory */ - TRANSCRIBE_DEVICE_TYPE_IGPU = 2, /* integrated GPU using host memory */ + TRANSCRIBE_DEVICE_TYPE_GPU = 1, /* backend-reported GPU */ + TRANSCRIBE_DEVICE_TYPE_IGPU = 2, /* backend-reported integrated GPU */ TRANSCRIBE_DEVICE_TYPE_ACCEL = 3, /* host-memory accelerator (BLAS/AMX) */ } transcribe_device_type; diff --git a/src/arch/cohere/model.cpp b/src/arch/cohere/model.cpp index 5259c4fe..38cf26a2 100644 --- a/src/arch/cohere/model.cpp +++ b/src/arch/cohere/model.cpp @@ -482,10 +482,7 @@ transcribe_status load( const transcribe_model_load_params * params, transcribe_model ** out_model) { - // params->backend is consumed below in the backend init block; - // params->gpu_device is reserved per the public header contract - // and is not yet honored (multi-device selection is a future - // release). + // Backend and device selection are resolved below via load_common. const int64_t t_load_start = ggml_time_us(); diff --git a/src/arch/parakeet/model.cpp b/src/arch/parakeet/model.cpp index 87f0ec92..6601b245 100644 --- a/src/arch/parakeet/model.cpp +++ b/src/arch/parakeet/model.cpp @@ -487,10 +487,7 @@ transcribe_status load( // The dispatcher has already verified out_model is non-null and // the loader has a valid gguf_context with general.architecture // set. *out_model is currently null (the dispatcher cleared it). - // params->backend is consumed below in the backend init block; - // params->gpu_device is reserved per the public header contract - // and is not yet honored (multi-device selection is a future - // release). + // Backend and device selection are resolved below via load_common. const int64_t t_load_start = ggml_time_us(); diff --git a/src/transcribe-backend.h b/src/transcribe-backend.h index 1b41b6c2..1e4f9095 100644 --- a/src/transcribe-backend.h +++ b/src/transcribe-backend.h @@ -6,7 +6,7 @@ // Rationale // --------- // The public API exposes a small `transcribe_backend_request` enum -// (auto|cpu|cpu_accel|metal|vulkan). Internally the library needs two related +// (auto|cpu|cpu_accel|metal|vulkan|cuda). Internally the library needs two related // things that don't belong in the public header: // // 1. A typed classification of whatever ggml backend we actually diff --git a/src/transcribe.cpp b/src/transcribe.cpp index 55d619f3..b03038a3 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -915,11 +915,23 @@ void fill_backend_device(ggml_backend_dev_t dev, uint64_t caller_size, ggml_backend_dev_props props{}; ggml_backend_dev_get_props(dev, &props); + const transcribe::BackendKind kind = transcribe::classify_device(dev); + const char * kind_name = transcribe::kind_name(kind); + const char * name = ggml_backend_dev_name(dev); + if (name == nullptr || name[0] == '\0') { + name = (props.name != nullptr && props.name[0] != '\0') + ? props.name : kind_name; + } + const char * description = ggml_backend_dev_description(dev); + if (description == nullptr) { + description = props.description != nullptr ? props.description : ""; + } + struct transcribe_backend_device staged{}; staged.struct_size = caller_size; - staged.name = ggml_backend_dev_name(dev); - staged.description = ggml_backend_dev_description(dev); - staged.kind = transcribe::kind_name(transcribe::classify_device(dev)); + staged.name = name; + staged.description = description; + staged.kind = kind_name; staged.device_id = props.device_id; staged.memory_total = props.memory_total; staged.memory_free = props.memory_free; diff --git a/tests/backend_init_unit.cpp b/tests/backend_init_unit.cpp index 071ea661..8f60c426 100644 --- a/tests/backend_init_unit.cpp +++ b/tests/backend_init_unit.cpp @@ -10,6 +10,8 @@ // actually returns, not on registry probes — a device can be // registered but fail initialization. // - AUTO: always succeeds; asserts based on the returned primary_kind. +// - Explicit gpu_device: rejects invalid selectors and, when a +// nonzero GPU index exists, binds that exact registry device. // - Invalid enum: returns TRANSCRIBE_ERR_INVALID_ARG. #include "transcribe-load-common.h" @@ -18,6 +20,7 @@ #include "ggml.h" #include "ggml-backend.h" +#include #include #include @@ -70,6 +73,12 @@ void free_plan(transcribe::BackendPlan & plan) { plan.primary_kind = transcribe::BackendKind::Unknown; } +bool is_gpu_device(ggml_backend_dev_t dev) { + const auto type = ggml_backend_dev_type(dev); + return type == GGML_BACKEND_DEVICE_TYPE_GPU || + type == GGML_BACKEND_DEVICE_TYPE_IGPU; +} + } // namespace int main() { @@ -82,7 +91,7 @@ int main() { { BackendPlan plan; transcribe_status st = init_backends( - TRANSCRIBE_BACKEND_CPU, "test-cpu", plan); + TRANSCRIBE_BACKEND_CPU, 0, "test-cpu", plan); REQUIRE(st == TRANSCRIBE_OK); CHECK_EQ(plan.primary_kind, BackendKind::Cpu); REQUIRE(plan.primary != nullptr); @@ -105,7 +114,7 @@ int main() { { BackendPlan plan; transcribe_status st = init_backends( - TRANSCRIBE_BACKEND_CPU_ACCEL, "test-cpu-accel", plan); + TRANSCRIBE_BACKEND_CPU_ACCEL, 0, "test-cpu-accel", plan); REQUIRE(st == TRANSCRIBE_OK); CHECK_EQ(plan.primary_kind, BackendKind::Cpu); REQUIRE(plan.primary != nullptr); @@ -132,7 +141,7 @@ int main() { { BackendPlan plan; transcribe_status st = init_backends( - TRANSCRIBE_BACKEND_METAL, "test-metal", plan); + TRANSCRIBE_BACKEND_METAL, 0, "test-metal", plan); if (st == TRANSCRIBE_OK) { CHECK_EQ(plan.primary_kind, BackendKind::Metal); @@ -150,7 +159,7 @@ int main() { { BackendPlan plan; transcribe_status st = init_backends( - TRANSCRIBE_BACKEND_VULKAN, "test-vulkan", plan); + TRANSCRIBE_BACKEND_VULKAN, 0, "test-vulkan", plan); if (st == TRANSCRIBE_OK) { CHECK_EQ(plan.primary_kind, BackendKind::Vulkan); @@ -168,7 +177,7 @@ int main() { { BackendPlan plan; transcribe_status st = init_backends( - TRANSCRIBE_BACKEND_CUDA, "test-cuda", plan); + TRANSCRIBE_BACKEND_CUDA, 0, "test-cuda", plan); if (st == TRANSCRIBE_OK) { CHECK_EQ(plan.primary_kind, BackendKind::Cuda); @@ -191,7 +200,7 @@ int main() { { BackendPlan plan; transcribe_status st = init_backends( - TRANSCRIBE_BACKEND_AUTO, "test-auto", plan); + TRANSCRIBE_BACKEND_AUTO, 0, "test-auto", plan); REQUIRE(st == TRANSCRIBE_OK); CHECK(plan.primary != nullptr); CHECK(plan.primary_kind != BackendKind::Unknown); @@ -215,10 +224,66 @@ int main() { BackendPlan plan; transcribe_status st = init_backends( static_cast(999), - "test-invalid", plan); + 0, "test-invalid", plan); + CHECK_EQ(st, TRANSCRIBE_ERR_INVALID_ARG); + } + + // --------------------------------------------------------------- + // 7. Explicit gpu_device validation + // --------------------------------------------------------------- + { + BackendPlan plan; + transcribe_status st = init_backends( + TRANSCRIBE_BACKEND_AUTO, -1, "test-gpu-negative", plan); + CHECK_EQ(st, TRANSCRIBE_ERR_INVALID_ARG); + } + { + BackendPlan plan; + transcribe_status st = init_backends( + TRANSCRIBE_BACKEND_CPU, 1, "test-gpu-cpu-request", plan); + CHECK_EQ(st, TRANSCRIBE_ERR_INVALID_ARG); + } + { + BackendPlan plan; + const int out_of_range = + static_cast(ggml_backend_dev_count()) + 1; + transcribe_status st = init_backends( + TRANSCRIBE_BACKEND_AUTO, out_of_range, + "test-gpu-out-of-range", plan); CHECK_EQ(st, TRANSCRIBE_ERR_INVALID_ARG); } + const size_t n_dev = ggml_backend_dev_count(); + for (size_t i = 1; i < n_dev; ++i) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); + if (dev != nullptr && !is_gpu_device(dev)) { + BackendPlan plan; + transcribe_status st = init_backends( + TRANSCRIBE_BACKEND_AUTO, static_cast(i), + "test-gpu-non-gpu", plan); + CHECK_EQ(st, TRANSCRIBE_ERR_INVALID_ARG); + break; + } + } + + for (size_t i = 1; i < n_dev; ++i) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); + if (dev != nullptr && is_gpu_device(dev)) { + BackendPlan plan; + transcribe_status st = init_backends( + TRANSCRIBE_BACKEND_AUTO, static_cast(i), + "test-gpu-explicit", plan); + if (st == TRANSCRIBE_OK) { + CHECK(plan.primary != nullptr); + CHECK(ggml_backend_get_device(plan.primary) == dev); + free_plan(plan); + } else { + CHECK_EQ(st, TRANSCRIBE_ERR_BACKEND); + } + break; + } + } + // --------------------------------------------------------------- // Summary // --------------------------------------------------------------- diff --git a/tools/transcribe-bench/main.cpp b/tools/transcribe-bench/main.cpp index b9453e44..c3af8f66 100644 --- a/tools/transcribe-bench/main.cpp +++ b/tools/transcribe-bench/main.cpp @@ -57,7 +57,7 @@ void print_usage(const char * argv0) { " --quiet suppress progress lines on stderr\n" " --threads N CPU threads (default 0 = library default)\n" " --backend KIND request a specific backend:\n" - " auto|cpu|cpu_accel|metal|vulkan (default auto)\n" + " auto|cpu|cpu_accel|metal|vulkan|cuda (default auto)\n" " cpu is strict CPU (no GPU, no BLAS/AMX).\n" " cpu_accel is CPU + host-memory accelerators\n" " (BLAS/AMX) when the build includes them.\n"