diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8c424ef6..5cb77efb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -90,6 +90,7 @@ jobs: cp c/openai_server.py dist/ cp c/resource_plan.py dist/ cp c/doctor.py dist/ + cp c/autotune.py dist/ cp LICENSE dist/ # web/dist sits NEXT TO coli in the archive; both coli and openai_server.py # probe that layout as well as the source checkout's one-level-up form. diff --git a/README.md b/README.md index db057fa6..06317be9 100644 --- a/README.md +++ b/README.md @@ -362,6 +362,7 @@ COLI_MODEL=/nvme/glm52_i4 ./coli chat # RAM budget, cache and MTP auto-detec COLI_MODEL=/nvme/glm52_i4 ./coli plan # inspect the planned VRAM/RAM/disk placement COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only readiness check COLI_MODEL=/nvme/glm52_i4 ./coli doctor --deep # strict tensors/shards/index/mirror preflight +COLI_MODEL=/nvme/glm52_i4 ./coli tune # measure and save this machine's fastest safe execution profile ./coli web --model /nvme/glm52_i4 # API + web dashboard on one port ./coli serve --model /nvme/glm52_i4 # OpenAI-compatible API only ``` diff --git a/c/Makefile b/c/Makefile index 10b1fa46..44559651 100644 --- a/c/Makefile +++ b/c/Makefile @@ -633,7 +633,7 @@ install: colibri$(EXE) olmoe$(EXE) $(INSTALL) -m 755 coli $(DESTDIR)$(BINDIR)/coli $(INSTALL) -m 755 colibri$(EXE) $(DESTDIR)$(LIBEXECDIR)/colibri$(EXE) $(INSTALL) -m 755 olmoe$(EXE) $(DESTDIR)$(LIBEXECDIR)/olmoe$(EXE) - $(INSTALL) -m 644 resource_plan.py doctor.py openai_server.py version.py $(DESTDIR)$(LIBEXECDIR)/ + $(INSTALL) -m 644 resource_plan.py doctor.py autotune.py openai_server.py version.py $(DESTDIR)$(LIBEXECDIR)/ $(INSTALL) -m 644 tools/*.py $(DESTDIR)$(LIBEXECDIR)/tools/ @# The dashboard is an optional build artifact (cd web && npm run build), so install @# it only when it exists. It goes NEXT TO openai_server.py, which probes ./web/dist. diff --git a/c/autotune.py b/c/autotune.py new file mode 100644 index 00000000..40d3f7f9 --- /dev/null +++ b/c/autotune.py @@ -0,0 +1,299 @@ +"""Measured, quality-preserving execution tuning for colibri. + +The tuner changes execution scheduling only. It never sweeps quantization, +expert selection, sampling, or model weights. A calibration continuation is +generated once and then teacher-forced for every candidate so every run sees +the same token and routing inputs. +""" +from __future__ import annotations + +import hashlib +import json +import os +import platform +import re +import statistics +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +SCHEMA_VERSION = 1 +PROMPT_RE = re.compile(r"\[PROMPT_TOKENS\]\s+\d+:\s*([0-9 ]+)") +TOKENS_RE = re.compile(r"\[TOKENS\]\s+\d+\s+generated:\s*([0-9 ]+)") +SPEED_RE = re.compile(r"REPLAY decode:\s+\d+\s+tokens.*?\|\s*([0-9.]+)\s+tok/s") +HIT_RE = re.compile(r"expert hit\s+([0-9.]+)%") +LATENCY_RE = re.compile(r"latency p50\s+([0-9.]+)\s*ms.*?p99\s+([0-9.]+)\s*ms") + +# Only scheduling / placement knobs belong here. Quality-affecting knobs such +# as TOPK, TOPP, DRAFT, quantization and CACHE_ROUTE are intentionally excluded. +TUNABLE_KEYS = frozenset({ + "OMP_NUM_THREADS", "COLI_NUMA", "PIPE", "DIRECT", + "COLI_CUDA_PIPE", "COLI_CUDA_ASYNC", +}) + + +def _config_path(profile_dir: str | None, fingerprint: str) -> Path: + if profile_dir: + root = Path(profile_dir).expanduser() + elif os.name == "nt": + root = Path(os.environ.get("LOCALAPPDATA", "~/AppData/Local")).expanduser() / "colibri" / "tuning" + else: + root = Path(os.environ.get("XDG_CONFIG_HOME", "~/.config")).expanduser() / "colibri" / "tuning" + return root / f"{fingerprint}.json" + + +def machine_fingerprint(plan: dict, model: str, engine: str) -> str: + """Stable identity for execution-relevant hardware, model, and engine.""" + model_path = Path(model) + files = [] + paths = [model_path / name for name in ("config.json", "model.safetensors.index.json")] + paths.extend(sorted(model_path.glob("*.safetensors"))) + for path in paths: + name = path.name + try: + stat = path.stat() + files.append((name, stat.st_size, stat.st_mtime_ns)) + except OSError: + files.append((name, None, None)) + try: + engine_stat = Path(engine).stat() + engine_id = (engine_stat.st_size, engine_stat.st_mtime_ns) + except OSError: + engine_id = (None, None) + cpu_model = platform.processor() + if sys_platform := getattr(platform, "system", lambda: "")(): + cpu_model = f"{sys_platform}:{platform.machine()}:{cpu_model}" + try: + for line in Path("/proc/cpuinfo").read_text(errors="replace").splitlines(): + if line.lower().startswith("model name"): + cpu_model += ":" + line.split(":", 1)[1].strip() + break + except OSError: + pass + payload = { + "schema": SCHEMA_VERSION, + "cpu_model": cpu_model, + "cpu": plan.get("cpu"), + "gpus": [ + {"name": gpu.get("name"), "total_bytes": gpu.get("total_bytes")} + for gpu in plan.get("tiers", {}).get("vram", {}).get("devices", []) + ], + "model": str(model_path.resolve()), + "model_files": files, + "engine": engine_id, + } + raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(raw).hexdigest()[:20] + + +def load_profile(plan: dict, model: str, engine: str, profile_dir: str | None = None) -> dict | None: + fingerprint = machine_fingerprint(plan, model, engine) + path = _config_path(profile_dir, fingerprint) + try: + profile = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if (profile.get("schema_version") != SCHEMA_VERSION + or profile.get("fingerprint") != fingerprint + or not profile.get("accepted")): + return None + env = profile.get("winner", {}).get("env", {}) + if not isinstance(env, dict) or any(key not in TUNABLE_KEYS for key in env): + return None + return profile + + +def apply_profile(env: dict, profile: dict, explicit_keys=()) -> dict: + """Apply a validated profile without overriding the caller's environment.""" + result = dict(env) + explicit = set(explicit_keys) + for key, value in profile["winner"]["env"].items(): + if key not in explicit: + result[key] = str(value) + return result + + +def save_profile(profile: dict, profile_dir: str | None = None) -> Path: + path = _config_path(profile_dir, profile["fingerprint"]) + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix=path.name + ".", dir=path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(profile, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + return path + + +def candidate_steps(plan: dict, base_env: dict) -> list[tuple[str, dict]]: + """Return a bounded coordinate-descent sweep for this topology.""" + steps = [] + cores = max(1, int(plan.get("cpu", {}).get("physical_cores", os.cpu_count() or 1))) + current_threads = int(base_env.get("OMP_NUM_THREADS", cores)) + for threads in dict.fromkeys((cores, max(1, cores // 2))): + if threads != current_threads: + steps.append((f"omp-{threads}", {"OMP_NUM_THREADS": str(threads)})) + sockets = int(plan.get("cpu", {}).get("sockets", 1)) + if sockets > 1 and base_env.get("COLI_NUMA") != "1": + steps.append(("numa-on", {"COLI_NUMA": "1"})) + has_gpu = bool(plan.get("tiers", {}).get("vram", {}).get("devices")) + if has_gpu: + pipe = int(base_env.get("COLI_CUDA_PIPE", "0")) + for value in (1, 2): + if value != pipe: + steps.append((f"cuda-pipe-{value}", {"COLI_CUDA_PIPE": str(value)})) + steps.append(("cuda-sync", {"COLI_CUDA_ASYNC": "0"})) + if plan.get("tiers", {}).get("disk", {}).get("cold_expert_bytes", 0) > 0: + direct = int(base_env.get("DIRECT", "0")) + if direct != 1: + steps.append(("direct-on", {"DIRECT": "1"})) + pipe = int(base_env.get("PIPE", "0")) + if pipe != 1: + steps.append(("io-pipe-on", {"PIPE": "1"})) + return steps + + +def parse_replay(output: str) -> dict: + speed = SPEED_RE.search(output) + if not speed: + raise ValueError("engine did not emit REPLAY throughput") + hit = HIT_RE.search(output) + latency = LATENCY_RE.search(output) + return { + "tok_s": float(speed.group(1)), + "hit_pct": float(hit.group(1)) if hit else None, + "p50_ms": float(latency.group(1)) if latency else None, + "p99_ms": float(latency.group(2)) if latency else None, + } + + +def _run(command: list[str], env: dict, timeout: int) -> subprocess.CompletedProcess: + if Path(command[0]).suffix.lower() == ".py": + command = [sys.executable, *command] + return subprocess.run(command, env=env, text=True, capture_output=True, timeout=timeout) + + +def create_replay(engine: str, cap: int, env: dict, prompt: str, tokens: int, + timeout: int) -> tuple[dict, str]: + calibration = dict(env, PROMPT=prompt, NGEN=str(tokens), TOKENS="1", PROF="1") + proc = _run([engine, str(cap)], calibration, timeout) + output = proc.stdout + "\n" + proc.stderr + if proc.returncode: + raise RuntimeError(f"calibration failed ({proc.returncode})\n{output[-2000:]}") + prompt_match, token_match = PROMPT_RE.search(output), TOKENS_RE.search(output) + if not prompt_match or not token_match: + raise RuntimeError("engine did not emit calibration token trace; rebuild the engine") + prompt_ids = [int(value) for value in prompt_match.group(1).split()] + continuation = [int(value) for value in token_match.group(1).split()] + if len(prompt_ids) < 2 or not continuation: + raise RuntimeError("calibration produced an empty token trace") + return {"prompt_ids": prompt_ids, "full_ids": prompt_ids + continuation}, output + + +def run_tune(engine: str, cap: int, base_env: dict, plan: dict, model: str, + prompt: str, tokens: int = 16, repeats: int = 2, timeout: int = 900, + min_gain: float = 0.03, profile_dir: str | None = None, + progress=None) -> tuple[dict, Path]: + """Coordinate-descent tuning with a reverse-order confirmation gate.""" + progress = progress or (lambda _message: None) + replay, _ = create_replay(engine, cap, base_env, prompt, tokens, timeout) + with tempfile.TemporaryDirectory(prefix="coli-tune-") as directory: + ref = Path(directory) / "replay.json" + ref.write_text(json.dumps(replay), encoding="utf-8") + + def measure(name, overlay): + samples = [] + for repeat in range(repeats): + progress(f"{name} ({repeat + 1}/{repeats})") + env = dict(base_env) + env.update(overlay) + env.update({"REF": str(ref), "REF_FORCE": "1", "REPLAY": "1", "PROF": "1"}) + env.pop("PROMPT", None); env.pop("TOKENS", None) + proc = _run([engine, str(cap)], env, timeout) + output = proc.stdout + "\n" + proc.stderr + if proc.returncode: + raise RuntimeError(f"{name} failed ({proc.returncode})\n{output[-2000:]}") + samples.append(parse_replay(output)) + return { + "name": name, + "env": dict(overlay), + "tok_s": statistics.median(sample["tok_s"] for sample in samples), + "hit_pct": statistics.median( + sample["hit_pct"] for sample in samples if sample["hit_pct"] is not None + ) if any(sample["hit_pct"] is not None for sample in samples) else None, + "p99_ms": statistics.median( + sample["p99_ms"] for sample in samples if sample["p99_ms"] is not None + ) if any(sample["p99_ms"] is not None for sample in samples) else None, + "samples": samples, + } + + baseline = measure("baseline", {}) + winner = baseline + accumulated = {} + candidates = [baseline] + for name, change in candidate_steps(plan, base_env): + trial_env = dict(accumulated) + trial_env.update(change) + trial = measure(name, trial_env) + candidates.append(trial) + hit_ok = (baseline["hit_pct"] is None or trial["hit_pct"] is None + or trial["hit_pct"] >= baseline["hit_pct"] - 0.5) + tail_ok = (baseline["p99_ms"] is None or trial["p99_ms"] is None + or trial["p99_ms"] <= baseline["p99_ms"] * 1.20) + if hit_ok and tail_ok and trial["tok_s"] > winner["tok_s"] * (1.0 + min_gain): + winner = trial + accumulated = trial_env + + validation = None + accepted = False + gain = 0.0 + if winner is not baseline: + # Candidate trials necessarily run after the first baseline and can + # benefit from thermal/page-cache drift. Confirm in the opposite + # order: winner first, baseline last. This is deliberately + # conservative — a later baseline gets any remaining warm-cache + # advantage. A profile is accepted only if it still wins. + confirmed_winner = measure("confirm-winner", winner["env"]) + confirmed_winner["name"] = winner["name"] + confirmed_baseline = measure("confirm-baseline", {}) + hit_ok = (confirmed_baseline["hit_pct"] is None + or confirmed_winner["hit_pct"] is None + or confirmed_winner["hit_pct"] >= confirmed_baseline["hit_pct"] - 0.5) + tail_ok = (confirmed_baseline["p99_ms"] is None + or confirmed_winner["p99_ms"] is None + or confirmed_winner["p99_ms"] <= confirmed_baseline["p99_ms"] * 1.20) + gain = confirmed_winner["tok_s"] / confirmed_baseline["tok_s"] - 1.0 + accepted = hit_ok and tail_ok and gain >= min_gain + validation = { + "winner": confirmed_winner, + "baseline": confirmed_baseline, + "hit_gate": hit_ok, + "tail_gate": tail_ok, + } + if accepted: + winner = confirmed_winner + fingerprint = machine_fingerprint(plan, model, engine) + profile = { + "schema_version": SCHEMA_VERSION, + "fingerprint": fingerprint, + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "model": str(Path(model).resolve()), + "accepted": accepted, + "minimum_gain": min_gain, + "gain": gain if accepted else 0.0, + "baseline": baseline, + "winner": winner if accepted else baseline, + "candidates": candidates, + "validation": validation, + "plan": plan, + } + return profile, save_profile(profile, profile_dir) diff --git a/c/coli b/c/coli index d31f98ac..a37a3067 100755 --- a/c/coli +++ b/c/coli @@ -58,7 +58,7 @@ except ModuleNotFoundError: _version = "unknown" # Run-in-place (source checkout, "cd c && ./coli ..."): the engine, the -# support modules (resource_plan.py, doctor.py, openai_server.py) and +# support modules (resource_plan.py, doctor.py, autotune.py, openai_server.py) and # tools/ all live next to this script — unchanged from before. # # Installed layout ("make install"): this script is $(PREFIX)/bin/coli, @@ -227,6 +227,7 @@ def resource_request(a, env): return ram,ctx,devices,vram def env_for(a): + explicit_env = set(os.environ) e = dict(os.environ, SNAP=a.model) if sys.platform == "win32": # COLI_NO_OMP_TUNE spegne SOLO il blocco OMP (stesso perimetro del @@ -283,6 +284,14 @@ def env_for(a): sys.exit(f"{C.yel}invalid resource plan:{C.r} {error}") has_cuda=cuda_binary() e=environment_for_plan(plan,e,has_cuda) + if not getattr(a, "no_tune_profile", False): + from autotune import apply_profile, load_profile + profile=load_profile(plan,a.model,GLM) + if profile: + e=apply_profile(e,profile,explicit_env) + gain=100.0*profile["gain"] + print(f" {C.dim}[TUNE] applied measured profile · +{gain:.1f}% calibration throughput{C.r}", + file=sys.stderr) rt=plan["tiers"]["ram"]; vt=plan["tiers"]["vram"] gpu=f" · VRAM {format_bytes(vt['budget_bytes'])}" if has_cuda and vt["devices"] else " · CPU" print(f" {C.dim}[PLAN] RAM {format_bytes(rt['budget_bytes'])} · cap {rt['cache_slots_per_layer']}/layer{gpu}{C.r}",file=sys.stderr) @@ -597,6 +606,50 @@ def cmd_doctor(a): print(json.dumps(report,indent=2) if a.json else format_doctor(report)) return exit_code(report) +def cmd_tune(a): + """Measure this exact model/hardware pair and persist the fastest safe path.""" + need_model(a.model) + if a.tokens < 4: + sys.exit(f"{C.yel}--tokens must be at least 4{C.r}") + if a.timeout < 1: + sys.exit(f"{C.yel}--timeout must be positive{C.r}") + if not 0.0 <= a.min_gain <= 1.0: + sys.exit(f"{C.yel}--min-gain must be between 0 and 1{C.r}") + from autotune import run_tune + from resource_plan import build_plan + try: + ram,ctx,devices,vram=resource_request(a,os.environ) + plan=build_plan(a.model,ram,ctx,devices,vram,policy=a.policy) + except (OSError,ValueError,json.JSONDecodeError) as error: + sys.exit(f"{C.yel}cannot create tuning plan:{C.r} {error}") + a.auto_tier=True + a.no_tune_profile=True + base_env=env_for(a) + prompt=f"[gMASK]<|user|>{a.prompt}<|assistant|>" + banner("tune · measured execution profile") + print(f" fixed replay: {a.tokens} tokens · {a.repeats} run(s) per candidate") + print(" only quality-preserving scheduling knobs are eligible\n") + try: + profile,path=run_tune( + GLM,a.cap,base_env,plan,a.model,prompt,tokens=a.tokens, + repeats=a.repeats,timeout=a.timeout,min_gain=a.min_gain, + profile_dir=a.profile_dir, + progress=lambda message: print(f" {C.dim}· {message}{C.r}",flush=True), + ) + except (OSError,ValueError,RuntimeError,subprocess.SubprocessError) as error: + sys.exit(f"{C.yel}tuning failed:{C.r} {error}") + if profile["accepted"]: + winner=profile["winner"] + baseline=profile["validation"]["baseline"]["tok_s"] + print(f"\n {C.grn}✓ accepted{C.r} {winner['name']}: " + f"{baseline:.2f} → {winner['tok_s']:.2f} tok/s " + f"(+{100*profile['gain']:.1f}%)") + print(f" env: {' '.join(f'{key}={value}' for key,value in winner['env'].items())}") + else: + print(f"\n {C.grn}✓ baseline retained{C.r}: no candidate cleared the " + f"{100*a.min_gain:.1f}% gain and safety gates") + print(f" profile: {path}\n") + def cmd_run(a): need_model(a.model) prompt=" ".join(a.prompt) if a.prompt else sys.exit('usage: coli run "your prompt"') @@ -1043,6 +1096,8 @@ def main(): common=argparse.ArgumentParser(add_help=False) common.add_argument("--model", default=DEF_MODEL); common.add_argument("--ram", type=int, default=0) # 0 = auto (il motore usa l'88% della RAM disponibile) common.add_argument("--auto-tier",action="store_true",help="automatically apply the RAM/VRAM plan") + common.add_argument("--no-tune-profile",action="store_true", + help="ignore a saved measured tuning profile") common.add_argument("--ctx",type=int,default=0) common.add_argument("--gpu",default=None,help="auto, none, or a device list such as 0,1") common.add_argument("--vram",type=float,default=0,help="total VRAM budget in GB (0=auto)") @@ -1063,6 +1118,16 @@ def main(): pd.add_argument("--json",action="store_true",help="emit a versioned JSON report") pd.add_argument("--deep",action="store_true", help="strictly check every tensor layout, model index, and configured mirror") + pt=sub.add_parser("tune",parents=[common], + description="measure and save the fastest quality-preserving execution profile", + help="measure and save the fastest quality-preserving execution profile") + pt.add_argument("--prompt",default="Explain why fixed-token replay makes a benchmark reproducible.") + pt.add_argument("--tokens",type=int,default=16,help="calibration continuation length") + pt.add_argument("--repeats",type=int,choices=range(1,6),default=2) + pt.add_argument("--timeout",type=int,default=900,help="seconds allowed for each engine run") + pt.add_argument("--min-gain",type=float,default=0.03, + help="minimum fractional throughput gain required to accept a candidate") + pt.add_argument("--profile-dir",default=None,help=argparse.SUPPRESS) pr=sub.add_parser("run", parents=[common]); pr.add_argument("prompt", nargs="*") pc=sub.add_parser("chat", parents=[common]) pc.add_argument("--attach", nargs="?", const="http://127.0.0.1:8000", default=None, @@ -1112,7 +1177,7 @@ def main(): help="int4 scale group size: 64 (default, group-scaled quality) or 0 (legacy per-row)") pc.add_argument("--no-mtp",action="store_true",help="skip the MTP head (no speculative drafts)") a=ap.parse_args() - handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"doctor":cmd_doctor, + handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"doctor":cmd_doctor,"tune":cmd_tune, "run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"stop":cmd_stop,"bench":cmd_bench, "convert":cmd_convert,"web":cmd_web}.get(a.cmd) if handler: sys.exit(handler(a) or 0) diff --git a/c/colibri.c b/c/colibri.c index 65acbf30..5cb96b8b 100644 --- a/c/colibri.c +++ b/c/colibri.c @@ -5924,6 +5924,9 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ * for exact A/B comparison across decode paths (e.g. resident vs CPU). * The ids are all[np .. np+produced-1]. */ if(getenv("TOKENS") && atoi(getenv("TOKENS"))){ + fprintf(stderr,"[PROMPT_TOKENS] %d:",np); + for(int i=0;i"` | One-shot generation for the given prompt (positional, may be multi-word). | | `chat` | Interactive REPL chat. | | `serve` | Start the OpenAI-compatible HTTP server. | @@ -45,6 +46,7 @@ Flags may also be given **after** the subcommand. Most flags map onto an engine | `--gpu` | `None` | `COLI_GPU(S)` | `auto`, `none`, or a device list like `0,1`. | | `--vram` | `0` (auto) | CUDA plan | Total VRAM budget in GB. | | `--auto-tier` | off | resource plan | Automatically apply the RAM/VRAM placement plan. | +| `--no-tune-profile` | off | profile loader | Ignore a saved measured profile. | ### Subcommand-specific flags @@ -75,6 +77,10 @@ Flags may also be given **after** the subcommand. Most flags map onto an engine **`bench`**: `[tasks...]` (positional), `--limit 40`, `--data `. **`plan` / `doctor`**: `--json`. +**`tune`**: `--prompt `, `--tokens 16`, `--repeats 2`, +`--timeout 900`, `--min-gain 0.03`. The command uses fixed-token replay and +only tests quality-preserving execution scheduling. + **`doctor`**: `--deep` strictly checks every safetensors header and tensor layout, filename-declared shard completeness, required core tensors, an optional model index, and runtime-equivalent size/header admission for diff --git a/docs/quickstart.md b/docs/quickstart.md index f67853f1..bf2d1fa3 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -79,7 +79,7 @@ Inside you'll find: |---|---| | `colibri.exe` | **the engine** — the C program that actually runs the model | | `coli` | the command-line launcher (`chat`, `serve`, `convert`, `doctor`, …) | -| `openai_server.py`, `resource_plan.py`, `doctor.py` | Python support for the API server and placement planner | +| `openai_server.py`, `resource_plan.py`, `doctor.py`, `autotune.py` | Python support for the API server, placement planner, diagnostics, and measured tuning | One setup step: **install Python 3** from [python.org](https://www.python.org/downloads/) — the `coli` launcher and the diff --git a/docs/tuning.md b/docs/tuning.md index ff1e7850..c4a2fb29 100644 --- a/docs/tuning.md +++ b/docs/tuning.md @@ -54,6 +54,37 @@ coli run --auto-tier --policy quality "Explain MoE offloading" coli run --policy experimental-fast --topk 4 "Benchmark prompt" ``` +## Measured machine profiles + +`coli plan` chooses a safe starting point from capacity and topology. +`coli tune` measures the remaining scheduling choices on the actual model and +machine, then saves a hardware/model/engine-specific profile: + +```bash +coli tune --model /models/glm52_i4 +coli run --model /models/glm52_i4 --auto-tier "Explain MoE offloading" +``` + +The calibration prompt is generated once. Every candidate then teacher-forces +the same continuation, so answer length and sampling do not contaminate the +comparison. The bounded sweep only includes execution knobs such as OpenMP +thread count, NUMA placement, I/O overlap, direct I/O, and CUDA pipelining. It +never changes weights, quantization, router decisions, `TOPK`, `TOPP`, or +sampling. + +A candidate is saved only when median throughput improves by at least 3% while +expert hit rate remains within 0.5 percentage points and p99 latency stays +within 20% of the baseline. The winner is then rerun before a final baseline; +this reverse-order gate gives the baseline any remaining warm-cache advantage +and rejects startup drift. Otherwise the baseline is recorded and no override +is applied. Saved profiles are loaded by `--auto-tier`; explicit environment +variables always win. Use `--no-tune-profile` to bypass a saved profile. + +Profiles live under `$XDG_CONFIG_HOME/colibri/tuning` (normally +`~/.config/colibri/tuning`) or `%LOCALAPPDATA%\colibri\tuning` on Windows. A +change to the engine binary, model metadata, CPU topology, or GPU inventory +produces a new fingerprint instead of reusing stale measurements. + Disk is an immutable recovery source, not a normal decode target. If the plan leaves cold expert bytes on disk, speed depends on cache hit rate; output quality does not. diff --git a/flake.nix b/flake.nix index 10a61293..fa8c4b62 100644 --- a/flake.nix +++ b/flake.nix @@ -71,7 +71,7 @@ cp c/colibri $out/lib/colibri/colibri cp c/coli $out/lib/colibri/coli chmod +x $out/lib/colibri/coli - cp c/openai_server.py c/resource_plan.py c/doctor.py c/version.py \ + cp c/openai_server.py c/resource_plan.py c/doctor.py c/autotune.py c/version.py \ $out/lib/colibri/ cp -r c/tools/* $out/lib/colibri/tools/