From 72d3d37231e922a6fa9afca16e08fa45842d5eb4 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Thu, 16 Jul 2026 19:39:40 +0200 Subject: [PATCH 1/2] ci: run the checks on main too (#144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @ZacharyZcR's workflow, already green on dev (78c77bf): engine + C suite, web build + vitest, python suite, and a real CUDA syntax check. Only the workflow file is cherry-picked here — main's engine code stays at 54cfe56 and nothing else from dev comes along. main is where the checks are worth the most and where they were missing entirely. Includes the two fixes the first run exposed: the pinned action's version table stops at 12.6.2 (12.6.3 failed the install), and `nvcc ... | head -40` took its exit status from head, so the job could never fail. Co-Authored-By: ZacharyZcR <#144> Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 72 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..e18f1b3e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,72 @@ +name: CI + +on: + pull_request: + branches: [dev, main] + push: + branches: [dev, main] + +jobs: + engine: + name: Engine (Linux, CPU) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build glm + run: cd c && make glm + - name: C test suite + run: cd c && make test-c + + engine-cuda-syntax: + name: CUDA syntax check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install CUDA toolkit (compiler only) + uses: Jimver/cuda-toolkit@v0.2.19 + with: + # v0.2.19's version table stops at 12.6.2 — 12.6.3 fails the install + # step with "Version not available" before nvcc is ever reached. + cuda: '12.6.2' + method: network + sub-packages: '["nvcc"]' + - name: Compile backend_cuda.cu (syntax only, no GPU) + run: | + cd c + # NOT `nvcc ... | head -40`: a pipeline exits with the status of its + # LAST command, so head's 0 masked every nvcc error and the job could + # only ever be green. A check that cannot fail is worse than no check — + # it buys false confidence in the one file nothing else compiles. + nvcc -O2 -std=c++17 -arch=sm_80 -c backend_cuda.cu -o /dev/null \ + -Xcompiler=-Wall,-Wextra + echo "CUDA syntax check passed" + + web: + name: Web UI + runs-on: ubuntu-latest + defaults: + run: + working-directory: web + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: web/package-lock.json + - run: npm ci + - name: Build + run: npm run build + - name: Test + run: npx vitest run + + python: + name: Python tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Python test suite + run: cd c && python3 -m unittest discover -s tests -p 'test_*.py' From dc5b9e1d3cd545d5b1efb4de1867ecb48b444089 Mon Sep 17 00:00:00 2001 From: David Patton Date: Wed, 15 Jul 2026 22:09:28 -0400 Subject: [PATCH 2/2] Windows/CUDA port: thermal gate, storage probe, correct dashboard stats Engine (c/glm.c), all guarded by #ifdef COLI_CUDA and cross-platform #ifdefs: - NVML thermal heartbeat + thermal_gate() to bleed GPU heat per forward step (SAFE_MODE=0 disables); no-op when NVML is absent. - probe_storage_drive()/storage_bus_is_slow(): Win32 IOCTL + POSIX statvfs disk bus/free-space probe (FORCE_SLOW_DRIVE=1 to bypass). - Win32 hwinfo_emit branch (GetSystemInfo / GlobalMemoryStatusEx / __get_cpuid) so the dashboard works without /proc. Dashboard correctness: - Report RAM and VRAM in binary GiB so 128 GiB / 16 GiB read as 128 / 16 (not decimal 137 / 17). - Show the real GPU model via NVML nvmlDeviceGetName (e.g. RTX 4070 Ti SUPER) instead of a generic label; web panel shows the name and labels the logical-processor count as threads. Server (c/openai_server.py): COLI_EBITS/COLI_DBITS env to pick quant bit-width for 16 GB GPUs; absent preserves upstream 8/8 defaults. --- c/glm.c | 265 +++++++++++++++++++++++++++++++++++++++++++++ c/openai_server.py | 11 +- web/src/App.tsx | 4 +- 3 files changed, 277 insertions(+), 3 deletions(-) diff --git a/c/glm.c b/c/glm.c index 00c70768..e984f3ea 100644 --- a/c/glm.c +++ b/c/glm.c @@ -49,6 +49,14 @@ #include "grammar.h" /* metodo F: draft grammaticali (#48) */ #include "schema_gbnf.h" /* SCHEMA=: JSON-Schema -> GBNF for method F */ #include "decode_batch.h" +#if defined(_WIN32) +#include /* PORTING: IOCTL_STORAGE_QUERY_PROPERTY / STORAGE_BUS_TYPE (disk bus probe) */ +#else +#include /* PORTING: statvfs (free disk space, POSIX storage probe) */ +#endif +#if defined(_WIN32) && (defined(__x86_64__) || defined(__i386__)) +#include /* PORTING: __get_cpuid for the CPU brand string in hwinfo_emit */ +#endif #ifdef _OPENMP #include /* scratch per-thread nell'attention */ #else @@ -57,6 +65,9 @@ static inline int omp_get_thread_num(void){ return 0; } #endif #ifdef COLI_CUDA #include "backend_cuda.h" +#if !defined(_WIN32) +#include /* PORTING: dlopen(libnvidia-ml.so) dynamic NVML binding for thermal heartbeat */ +#endif #endif #ifdef COLI_METAL #include "backend_metal.h" @@ -212,6 +223,9 @@ static void ehit_mark(Model *m, int layer, int eid); static void emap_emit(Model *m); static void hits_emit(Model *m); static void hwinfo_emit(Model *m); +#ifdef COLI_CUDA +static int nvml_gpu_name(char *buf,int len); /* real GPU model for the dashboard */ +#endif static int g_repin; static uint64_t g_last_repin; #ifdef COLI_CUDA @@ -3832,7 +3846,21 @@ static void kv_bind(Model *m, KVState *k){ } static void mtp_absorb(Model *m, const int *next_ids, const float *x, int S, int pos_base); +/* PORTING: GPU thermal gate. NVML heartbeat thread sets g_thermal_pause_ms; step() + * calls thermal_gate() once per forward to bleed heat when the GPU is hot. In a + * CPU-only build (no COLI_CUDA) this is a total no-op, so the base engine is unchanged. */ +#ifdef COLI_CUDA +static volatile int g_thermal_pause_ms; /* set by the NVML thread (0 = no pause) */ +static inline void thermal_gate(void){ + int ms=g_thermal_pause_ms; + if(ms>0){ struct timespec ts={ms/1000,(long)(ms%1000)*1000000L}; nanosleep(&ts,NULL); } +} +#else +static inline void thermal_gate(void){} +#endif + static float *step(Model *m, const int *ids, int S, int pos_base){ + thermal_gate(); Cfg *c=&m->c; int D=c->hidden; float *x=falloc((int64_t)S*D); for(int s=0;s skip the interactive prompt (headless / Web UI start). + * ==========================================================================*/ +#ifndef STORAGE_MIN_FREE_GB +#define STORAGE_MIN_FREE_GB 400ULL +#endif + +#if defined(_WIN32) +/* -1 = unknown, 0 = fast (NVMe/SAS/RAID), 1 = slow (SATA/USB/other) */ +static int storage_bus_is_slow(const char *root, const char **bus_name){ + *bus_name="unknown"; + if(!root[0] || root[1]!=':') return -1; + char vol[8]; snprintf(vol,sizeof vol,"\\\\.\\%c:",root[0]); + HANDLE h=CreateFileA(vol,0,FILE_SHARE_READ|FILE_SHARE_WRITE,NULL,OPEN_EXISTING,0,NULL); + if(h==INVALID_HANDLE_VALUE) return -1; + STORAGE_PROPERTY_QUERY q; memset(&q,0,sizeof q); + q.PropertyId=StorageDeviceProperty; q.QueryType=PropertyStandardQuery; + STORAGE_DEVICE_DESCRIPTOR d; memset(&d,0,sizeof d); + DWORD br=0; int slow=-1; + if(DeviceIoControl(h,IOCTL_STORAGE_QUERY_PROPERTY,&q,sizeof q,&d,sizeof d,&br,NULL)){ + switch(d.BusType){ + case BusTypeNvme: *bus_name="NVMe"; slow=0; break; + case BusTypeSas: *bus_name="SAS"; slow=0; break; + case BusTypeRAID: *bus_name="RAID"; slow=0; break; + case BusTypeSata: *bus_name="SATA"; slow=1; break; + case BusTypeUsb: *bus_name="USB"; slow=1; break; + default: *bus_name="other/slow"; slow=1; break; + } + } + CloseHandle(h); + return slow; +} + +static int probe_storage_drive(const char *model_path){ + char root[MAX_PATH]; + if(!GetVolumePathNameA(model_path,root,sizeof root)){ + fprintf(stderr,"[STORAGE] cannot resolve the volume of '%s' (err %lu): proceeding.\n", + model_path,(unsigned long)GetLastError()); + return 1; + } + ULARGE_INTEGER freeAvail={0}, total={0}, totalFree={0}; + if(GetDiskFreeSpaceExA(root,&freeAvail,&total,&totalFree)){ + double free_gb=(double)freeAvail.QuadPart/1e9; + if(freeAvail.QuadPart < (ULONGLONG)STORAGE_MIN_FREE_GB*1000000000ULL){ + fprintf(stderr,"[STORAGE] FATAL: only %.1f GB free on %s (minimum %llu GB).\n", + free_gb,root,(unsigned long long)STORAGE_MIN_FREE_GB); + return 0; + } + fprintf(stderr,"[STORAGE] %s: %.1f GB free (OK).\n",root,free_gb); + } + const char *bus="unknown"; + int slow=storage_bus_is_slow(root,&bus); + if(slow==0){ fprintf(stderr,"[STORAGE] bus %s: NVMe/fast, streaming at full speed.\n",bus); return 1; } + if(slow<0){ fprintf(stderr,"[STORAGE] bus type undeterminable (%s): proceeding.\n",bus); return 1; } + fprintf(stderr, + "[WARNING] Model is residing on a SATA/Slow drive (%s). Mixture-of-Experts " + "streaming will be severely bottlenecked. It is highly recommended to move " + "the checkpoint to an NVMe SSD.\n",bus); + if(getenv("FORCE_SLOW_DRIVE") && atoi(getenv("FORCE_SLOW_DRIVE"))){ + fprintf(stderr,"[STORAGE] FORCE_SLOW_DRIVE=1: proceeding without confirmation.\n"); + return 1; + } + fprintf(stderr,"Continue anyway? (y/n): "); fflush(stderr); + int ch=getchar(); + if(ch=='y'||ch=='Y') return 1; + fprintf(stderr,"[STORAGE] startup aborted by user.\n"); + return 0; +} +#else /* POSIX: statvfs for space; bus detection is not portable -> proceed */ +static int probe_storage_drive(const char *model_path){ + struct statvfs vfs; + if(statvfs(model_path,&vfs)==0){ + unsigned long long freeb=(unsigned long long)vfs.f_bavail*(unsigned long long)vfs.f_frsize; + double free_gb=(double)freeb/1e9; + if(freeb < (unsigned long long)STORAGE_MIN_FREE_GB*1000000000ULL){ + fprintf(stderr,"[STORAGE] FATAL: only %.1f GB free (minimum %llu GB).\n", + free_gb,(unsigned long long)STORAGE_MIN_FREE_GB); + return 0; + } + fprintf(stderr,"[STORAGE] %.1f GB free (OK). Bus detection unavailable on this platform.\n",free_gb); + } + return 1; +} +#endif + +#ifdef COLI_CUDA +/* ---------------------------------------------------------------------------- + * PORTING: NVML thermal heartbeat - background thread that samples GPU temperature + * and applies a step-down (proportional pause via g_thermal_pause_ms, read by + * thermal_gate() every forward step) when it overheats. NVML is loaded DYNAMICALLY + * (nvml.dll / libnvidia-ml.so): if absent, safety does not start and the engine + * proceeds. All isolated under #ifdef COLI_CUDA. + * SAFE_MODE=0 -> disable the monitor entirely. + * -------------------------------------------------------------------------- */ +enum { TH_NORMAL=0, TH_WARN=1, TH_THROTTLE=2, TH_CRITICAL=3 }; +#ifndef TH_WARN_C +#define TH_WARN_C 83 /* deg C */ +#define TH_THROT_C 88 +#define TH_CRIT_C 93 +#define TH_CLEAR_C 80 /* hysteresis: return to NORMAL below this */ +#endif +static volatile int g_thermal_state=TH_NORMAL; +static volatile int g_thermal_run=0; +static volatile unsigned g_thermal_temp=0; +static pthread_t g_thermal_thr; + +/* dynamic NVML bindings (minimal signatures needed) */ +typedef int (*nvmlInit_t)(void); +typedef int (*nvmlShutdown_t)(void); +typedef int (*nvmlHandleByIndex_t)(unsigned,void**); +typedef int (*nvmlTemp_t)(void*,int,unsigned*); +typedef int (*nvmlName_t)(void*,char*,unsigned); +static void *g_nvml_lib=NULL, *g_nvml_dev=NULL; +static nvmlInit_t p_nvmlInit=NULL; +static nvmlShutdown_t p_nvmlShutdown=NULL; +static nvmlHandleByIndex_t p_nvmlHandle=NULL; +static nvmlTemp_t p_nvmlTemp=NULL; +static nvmlName_t p_nvmlName=NULL; + +static int nvml_load(void){ +#if defined(_WIN32) + g_nvml_lib=(void*)LoadLibraryA("nvml.dll"); + if(!g_nvml_lib){ + char p[MAX_PATH]; const char *pf=getenv("ProgramFiles"); + snprintf(p,sizeof p,"%s\\NVIDIA Corporation\\NVSMI\\nvml.dll",pf?pf:"C:\\Program Files"); + g_nvml_lib=(void*)LoadLibraryA(p); + } + if(!g_nvml_lib) return 0; + #define NVSYM(n) ((void*)GetProcAddress((HMODULE)g_nvml_lib,(n))) +#else + g_nvml_lib=dlopen("libnvidia-ml.so.1",RTLD_NOW|RTLD_GLOBAL); + if(!g_nvml_lib) g_nvml_lib=dlopen("libnvidia-ml.so",RTLD_NOW|RTLD_GLOBAL); + if(!g_nvml_lib) return 0; + #define NVSYM(n) dlsym(g_nvml_lib,(n)) +#endif + p_nvmlInit=(nvmlInit_t)NVSYM("nvmlInit_v2"); + if(!p_nvmlInit) p_nvmlInit=(nvmlInit_t)NVSYM("nvmlInit"); + p_nvmlShutdown=(nvmlShutdown_t)NVSYM("nvmlShutdown"); + p_nvmlHandle=(nvmlHandleByIndex_t)NVSYM("nvmlDeviceGetHandleByIndex_v2"); + if(!p_nvmlHandle) p_nvmlHandle=(nvmlHandleByIndex_t)NVSYM("nvmlDeviceGetHandleByIndex"); + p_nvmlTemp=(nvmlTemp_t)NVSYM("nvmlDeviceGetTemperature"); + p_nvmlName=(nvmlName_t)NVSYM("nvmlDeviceGetName"); + #undef NVSYM + return p_nvmlInit && p_nvmlHandle && p_nvmlTemp; +} + +/* Fetch the GPU model string (e.g. "NVIDIA GeForce RTX 4070 Ti SUPER") via NVML. + * Reuses the handle opened by the thermal monitor; if that isn't running (e.g. + * SAFE_MODE=0) it opens NVML on demand. Returns 1 on success. */ +static int nvml_gpu_name(char *buf,int len){ + if(!buf || len<=0) return 0; + buf[0]=0; + if(!g_nvml_lib && !nvml_load()) return 0; + if(!p_nvmlName) return 0; + void *dev=g_nvml_dev; + if(!dev){ + if(p_nvmlInit) p_nvmlInit(); /* refcounted — safe if already initialised */ + if(!p_nvmlHandle) return 0; + unsigned idx=(g_cuda_ndev>0)?(unsigned)g_cuda_devices[0]:0; + if(p_nvmlHandle(idx,&dev)!=0 || !dev) return 0; + } + return p_nvmlName(dev,buf,(unsigned)len)==0 && buf[0]; +} + +static void *thermal_worker(void *arg){ + (void)arg; + while(g_thermal_run){ + unsigned t=0; + if(p_nvmlTemp && g_nvml_dev && p_nvmlTemp(g_nvml_dev,0 /*NVML_TEMPERATURE_GPU*/,&t)==0){ + g_thermal_temp=t; + int st=g_thermal_state, ns=st; + if(t>=TH_CRIT_C) ns=TH_CRITICAL; + else if(t>=TH_THROT_C)ns=TH_THROTTLE; + else if(t>=TH_WARN_C) ns=TH_WARN; + else if(t<=TH_CLEAR_C)ns=TH_NORMAL; /* hysteresis */ + switch(ns){ + case TH_CRITICAL: g_thermal_pause_ms=400; break; + case TH_THROTTLE: g_thermal_pause_ms=120; break; + default: g_thermal_pause_ms=0; break; + } + if(ns!=st){ + static const char *nm[4]={"NORMAL","WARN","THROTTLE","CRITICAL"}; + fprintf(stderr,"[THERMAL] GPU %u\xC2\xB0""C -> %s%s\n",t,nm[ns], + ns>=TH_THROTTLE?" (step-down active)":""); + g_thermal_state=ns; + } + } + struct timespec ts={0,500L*1000L*1000L}; nanosleep(&ts,NULL); /* poll ~500 ms */ + } + return NULL; +} + +static void thermal_monitor_stop(void){ + if(!g_thermal_run) return; + g_thermal_run=0; + pthread_join(g_thermal_thr,NULL); + if(p_nvmlShutdown) p_nvmlShutdown(); +} + +static void thermal_monitor_start(void){ + if(getenv("SAFE_MODE") && atoi(getenv("SAFE_MODE"))==0){ + fprintf(stderr,"[THERMAL] SAFE_MODE=0: thermal monitor disabled.\n"); + return; + } + if(!g_cuda_enabled) return; + if(!nvml_load()){ fprintf(stderr,"[THERMAL] NVML unavailable: thermal safety not active.\n"); return; } + if(p_nvmlInit()!=0){ fprintf(stderr,"[THERMAL] nvmlInit failed: thermal safety not active.\n"); return; } + if(p_nvmlHandle((unsigned)g_cuda_devices[0],&g_nvml_dev)!=0 || !g_nvml_dev){ + fprintf(stderr,"[THERMAL] NVML handle for device %d not obtained.\n",g_cuda_devices[0]); + if(p_nvmlShutdown) p_nvmlShutdown(); + return; + } + g_thermal_run=1; + if(pthread_create(&g_thermal_thr,NULL,thermal_worker,NULL)!=0){ + g_thermal_run=0; + fprintf(stderr,"[THERMAL] pthread_create failed.\n"); + if(p_nvmlShutdown) p_nvmlShutdown(); + return; + } + fprintf(stderr,"[THERMAL] heartbeat active on device %d (warn %d\xC2\xB0""C / throttle %d\xC2\xB0""C / crit %d\xC2\xB0""C).\n", + g_cuda_devices[0],TH_WARN_C,TH_THROT_C,TH_CRIT_C); +} +#endif /* COLI_CUDA */ + int main(int argc, char **argv){ /* ---- Permanent OpenMP hot-thread tuning. The per-expert matmul regions are * tiny and back-to-back; with the default passive wait policy libgomp parks @@ -5757,6 +6015,9 @@ int main(int argc, char **argv){ } #endif const char *snap=getenv("SNAP"); if(!snap){fprintf(stderr,"SNAP=\n");return 1;} + /* PORTING: gate on checkpoint disk (free space + bus type) before allocating. + * FORCE_SLOW_DRIVE=1 skips the interactive prompt for headless starts (Web UI). */ + if(!probe_storage_drive(snap)) return 3; g_nopack = getenv("NOPACK")?1:0; g_drop = getenv("DROP")?1:0; g_prefetch = getenv("PREFETCH")?atoi(getenv("PREFETCH")):0; @@ -5897,6 +6158,9 @@ int main(int argc, char **argv){ if(g_cuda_ndev<1){ fprintf(stderr,"invalid COLI_GPUS: use a list such as 0,1,2\n"); return 2; } g_cuda_enabled=coli_cuda_init(g_cuda_devices,g_cuda_ndev); if(!g_cuda_enabled){ fprintf(stderr,"[CUDA] requested backend is unavailable\n"); return 2; } + /* PORTING: NVML thermal heartbeat (SAFE_MODE=0 to disable). No-op if NVML absent. */ + thermal_monitor_start(); + atexit(thermal_monitor_stop); } g_cuda_dense=getenv("CUDA_DENSE")?atoi(getenv("CUDA_DENSE")):0; g_cuda_pipe=getenv("COLI_CUDA_PIPE")?atoi(getenv("COLI_CUDA_PIPE")):0; @@ -6117,3 +6381,4 @@ int main(int argc, char **argv){ if(stats) stats_dump(&m,stats); return 0; } + diff --git a/c/openai_server.py b/c/openai_server.py index ba19e437..96451d56 100644 --- a/c/openai_server.py +++ b/c/openai_server.py @@ -457,8 +457,17 @@ class Engine: def __init__(self, executable, model, cap=8, max_tokens=1024, env=None, kv_slots=1): child_env = dict(env or os.environ, SNAP=str(model), SERVE="1", SERVE_BATCH="1", NGEN=str(max_tokens), KV_SLOTS=str(kv_slots)) + # PORTING: allow the quantization bit-width to be selected via env so the GUI can + # run the 4-bit config required on 16 GB GPUs. Absent -> engine defaults (8/8), i.e. + # upstream behavior is preserved. + engine_args = [str(executable), str(cap)] + _ebits = os.environ.get("COLI_EBITS") + _dbits = os.environ.get("COLI_DBITS") + if _ebits: + engine_args.append(_ebits) + engine_args.append(_dbits if _dbits else _ebits) self.process = subprocess.Popen( - [str(executable), str(cap)], env=child_env, stdin=subprocess.PIPE, + engine_args, env=child_env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0, ) self.write_lock = threading.Lock() diff --git a/web/src/App.tsx b/web/src/App.tsx index d4e0e415..3d3efb47 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -258,9 +258,9 @@ export default function App() {
Runtime
{health?.hwinfo ?
{health.hwinfo.cpu ?
{health.hwinfo.cpu}
: null} - {health.hwinfo.gpus > 0 ?
{health.hwinfo.gpus}× GPU{health.hwinfo.vram_total_gb.toFixed(0)} GB VRAM
: null} + {health.hwinfo.gpus > 0 ?
{health.hwinfo.gpu || `${health.hwinfo.gpus}× GPU`}{health.hwinfo.vram_total_gb.toFixed(0)} GB VRAM
: null}
{health.hwinfo.ram_total_gb.toFixed(0)} GB RAM{health.hwinfo.ram_avail_gb.toFixed(0)} GB free
-
{health.hwinfo.cores} cores
+
{health.hwinfo.cores} threads
: null} {health?.scheduler ? <>