From 85dfdccc232b42ef463d1833dde769bb96f7d487 Mon Sep 17 00:00:00 2001 From: Dave Page Date: Tue, 11 Aug 2026 14:17:00 +0100 Subject: [PATCH 1/2] Size the provider auth headers dynamically and stop truncating floats Audit of every fixed-length buffer in the extension. Two of the ten had a path where an input of unbounded length could reach them, and both failed by silently truncating rather than by complaining. The auth headers in the OpenAI, Voyage and Gemini providers were built into a 512 byte stack buffer, which leaves about 489 for the key itself. Nothing kept the key under that: provider_load_api_key() accepts a file of up to MAX_API_KEY_FILE_SIZE, which is 4096, so the two bounds disagreed by a factor of eight. That gap is reachable rather than theoretical, because JWT-style bearer tokens for OpenAI-compatible gateways routinely run past a thousand characters. The result would have been a credential quietly cut in half and an authentication failure indistinguishable from a wrong key. These now use psprintf(), matching how the url in the same functions is already built and freed, which removes the second bound altogether. provider_parse_float_array() read numeric literals into a 32 byte buffer and kept only the leading characters of anything longer, then handed that to atof(). A run that long is not a number any provider could legitimately send, since %.17g never exceeds 24 characters, but the failure mode was to score on a value of an entirely different magnitude without a word. It now stops instead, and because all three callers treat a short count as a dimension mismatch, the malformed response is reported as one. The remaining seven buffers are provably bounded and are left alone. The BM25 term keys cannot be reached by anything too long since #50 added the guard that drops such terms at tokenization; the three dbname buffers are NAMEDATALEN, which is the bound PostgreSQL itself puts on a database name; and the worker's captured error message is diagnostic text whose full version has already reached the server log, which is now said explicitly in the comment so the next audit need not rederive it. Closes #31 --- src/provider_common.c | 18 ++++++++++++++++-- src/provider_gemini.c | 7 ++++--- src/provider_openai.c | 13 +++++++------ src/provider_voyage.c | 7 ++++--- src/worker.c | 5 +++++ 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/provider_common.c b/src/provider_common.c index 6a0cd01..c9dd928 100644 --- a/src/provider_common.c +++ b/src/provider_common.c @@ -250,8 +250,22 @@ provider_parse_float_array(const char **pos, float *output, int dim) while (*p && (isdigit((unsigned char) *p) || *p == '.' || *p == '-' || *p == '+' || *p == 'e' || *p == 'E')) { - if (value_pos < (int) sizeof(value_buf) - 1) - value_buf[value_pos++] = *p; + /* + * Stop rather than truncate. No double needs this many + * characters -- %.17g never exceeds 24 -- so a longer run means a + * malformed response, and keeping only the leading digits would + * feed atof() a number of an entirely different magnitude and + * score on it silently. Every caller treats a short count as a + * dimension mismatch, so returning here reports the response as + * bad instead. + */ + if (value_pos >= (int) sizeof(value_buf) - 1) + { + *pos = p; + return idx; + } + + value_buf[value_pos++] = *p; p++; } value_buf[value_pos] = '\0'; diff --git a/src/provider_gemini.c b/src/provider_gemini.c index 8df08a6..b11293d 100644 --- a/src/provider_gemini.c +++ b/src/provider_gemini.c @@ -127,7 +127,7 @@ gemini_generate_batch(const char **texts, int count, int *dim, char **error_msg) char *json_request; char *url; const char *base_url; - char auth_header[512]; + char *auth_header; StringInfoData request_buf; ResponseBuffer response; float **embeddings; @@ -164,8 +164,7 @@ gemini_generate_batch(const char **texts, int count, int *dim, char **error_msg) pgedge_vectorizer_model); /* Build auth header */ - snprintf(auth_header, sizeof(auth_header), - "x-goog-api-key: %s", api_key); + auth_header = psprintf("x-goog-api-key: %s", api_key); /* Perform request */ if (!provider_do_curl_request(url, auth_header, json_request, @@ -173,6 +172,7 @@ gemini_generate_batch(const char **texts, int count, int *dim, char **error_msg) { pfree(json_request); pfree(url); + pfree(auth_header); if (response.data) pfree(response.data); return NULL; @@ -184,6 +184,7 @@ gemini_generate_batch(const char **texts, int count, int *dim, char **error_msg) pfree(json_request); pfree(url); + pfree(auth_header); pfree(response.data); return embeddings; } diff --git a/src/provider_openai.c b/src/provider_openai.c index ed2c19e..9626add 100644 --- a/src/provider_openai.c +++ b/src/provider_openai.c @@ -135,8 +135,7 @@ openai_generate_batch(const char **texts, int count, int *dim, char **error_msg) char *json_request; char *url; const char *base_url; - char auth_header[512]; - const char *auth_header_ptr = NULL; + char *auth_header = NULL; ResponseBuffer response; float **embeddings; @@ -160,17 +159,17 @@ openai_generate_batch(const char **texts, int count, int *dim, char **error_msg) /* Build auth header if we have a key */ if (api_key != NULL) { - snprintf(auth_header, sizeof(auth_header), - "Authorization: Bearer %s", api_key); - auth_header_ptr = auth_header; + auth_header = psprintf("Authorization: Bearer %s", api_key); } /* Perform request */ - if (!provider_do_curl_request(url, auth_header_ptr, json_request, + if (!provider_do_curl_request(url, auth_header, json_request, "OpenAI", &response, error_msg)) { pfree(json_request); pfree(url); + if (auth_header) + pfree(auth_header); if (response.data) pfree(response.data); return NULL; @@ -182,6 +181,8 @@ openai_generate_batch(const char **texts, int count, int *dim, char **error_msg) pfree(json_request); pfree(url); + if (auth_header) + pfree(auth_header); pfree(response.data); return embeddings; } diff --git a/src/provider_voyage.c b/src/provider_voyage.c index 2f37a6b..929da93 100644 --- a/src/provider_voyage.c +++ b/src/provider_voyage.c @@ -116,7 +116,7 @@ voyage_generate_batch(const char **texts, int count, int *dim, char **error_msg) char *json_request; char *url; const char *base_url; - char auth_header[512]; + char *auth_header; ResponseBuffer response; float **embeddings; @@ -138,8 +138,7 @@ voyage_generate_batch(const char **texts, int count, int *dim, char **error_msg) url = psprintf("%s/embeddings", base_url); /* Build auth header */ - snprintf(auth_header, sizeof(auth_header), - "Authorization: Bearer %s", api_key); + auth_header = psprintf("Authorization: Bearer %s", api_key); /* Perform request */ if (!provider_do_curl_request(url, auth_header, json_request, @@ -147,6 +146,7 @@ voyage_generate_batch(const char **texts, int count, int *dim, char **error_msg) { pfree(json_request); pfree(url); + pfree(auth_header); if (response.data) pfree(response.data); return NULL; @@ -158,6 +158,7 @@ voyage_generate_batch(const char **texts, int count, int *dim, char **error_msg) pfree(json_request); pfree(url); + pfree(auth_header); pfree(response.data); return embeddings; } diff --git a/src/worker.c b/src/worker.c index a021dc9..29c64cd 100644 --- a/src/worker.c +++ b/src/worker.c @@ -56,6 +56,11 @@ static time_t last_cleanup_time = 0; * a fixed string tells them the item failed, which they can already see from * its status, but not why. A fixed buffer rather than a palloc'd copy, since * the transaction context this is captured in does not survive the abort. + * + * Truncation here is deliberate and harmless: this is diagnostic text, the + * untruncated message has already gone to the server log via + * EmitErrorReport(), and a bounded buffer is what lets the copy outlive the + * abort at all. */ #define FAILED_ITEM_ERROR_LEN 1024 From 9aa568bc09750c04c28fa3dd7ea502f05d268dfd Mon Sep 17 00:00:00 2001 From: Mason Sharp Date: Tue, 11 Aug 2026 14:38:06 -0700 Subject: [PATCH 2/2] Name the fault when a numeric literal is too long A rejected literal returned a short count, which every caller reports as "Dimension mismatch" -- pointing the operator at their model or dimension configuration rather than at a malformed response. Return a distinguished value instead and give it its own message. A sentinel rather than an error_msg out-param, since error_msg is only initialised at the top-level entry points. It also fails safe: -1 can never equal a dimension, so a missed call site still rejects. --- src/provider_common.c | 17 ++++++++++------- src/provider_common.h | 9 ++++++++- src/provider_gemini.c | 7 +++++-- src/provider_ollama.c | 7 +++++-- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/provider_common.c b/src/provider_common.c index c9dd928..6a773b7 100644 --- a/src/provider_common.c +++ b/src/provider_common.c @@ -226,7 +226,9 @@ provider_count_array_dimensions(const char *p) * * Reads up to `dim` float values from the current position (after '['). * Advances *pos past the parsed values (to ']' or end of parsed data). - * Returns the number of values successfully parsed. + * Returns the number of values successfully parsed, or + * PROVIDER_PARSE_MALFORMED on a numeric literal too long to be one, in which + * case *pos is left on the offending run rather than past it. */ int provider_parse_float_array(const char **pos, float *output, int dim) @@ -255,14 +257,12 @@ provider_parse_float_array(const char **pos, float *output, int dim) * characters -- %.17g never exceeds 24 -- so a longer run means a * malformed response, and keeping only the leading digits would * feed atof() a number of an entirely different magnitude and - * score on it silently. Every caller treats a short count as a - * dimension mismatch, so returning here reports the response as - * bad instead. + * score on it silently. */ if (value_pos >= (int) sizeof(value_buf) - 1) { *pos = p; - return idx; + return PROVIDER_PARSE_MALFORMED; } value_buf[value_pos++] = *p; @@ -471,8 +471,11 @@ provider_parse_openai_embedding_response(const char *json_response, int count, if (parsed != *dim) { - *error_msg = psprintf("Dimension mismatch: expected %d, got %d", - *dim, parsed); + *error_msg = (parsed == PROVIDER_PARSE_MALFORMED) + ? pstrdup("Malformed embedding response: numeric literal too " + "long to be a number") + : psprintf("Dimension mismatch: expected %d, got %d", + *dim, parsed); provider_free_embeddings(embeddings, embedding_idx + 1); return NULL; } diff --git a/src/provider_common.h b/src/provider_common.h index 133d003..eb9f61d 100644 --- a/src/provider_common.h +++ b/src/provider_common.h @@ -75,7 +75,14 @@ float **provider_parse_openai_embedding_response(const char *json_response, /* Count dimensions by counting commas in a JSON float array (after '[') */ int provider_count_array_dimensions(const char *p); -/* Parse a JSON float array into pre-allocated output; returns count parsed */ +/* + * Parse a JSON float array into pre-allocated output; returns count parsed, + * or PROVIDER_PARSE_MALFORMED if a numeric literal was too long to be one. + * Callers that only compare against the expected count still reject that, + * since it can never equal a dimension. + */ +#define PROVIDER_PARSE_MALFORMED (-1) + int provider_parse_float_array(const char **pos, float *output, int dim); /* Append extra headers from GUC to a curl header list */ diff --git a/src/provider_gemini.c b/src/provider_gemini.c index b11293d..0ff5c00 100644 --- a/src/provider_gemini.c +++ b/src/provider_gemini.c @@ -237,8 +237,11 @@ parse_gemini_batch_embedding_response(const char *json_response, int count, if (parsed != *dim) { - *error_msg = psprintf("Dimension mismatch: expected %d, got %d", - *dim, parsed); + *error_msg = (parsed == PROVIDER_PARSE_MALFORMED) + ? pstrdup("Malformed embedding response: numeric literal too " + "long to be a number") + : psprintf("Dimension mismatch: expected %d, got %d", + *dim, parsed); provider_free_embeddings(embeddings, embedding_idx + 1); return NULL; } diff --git a/src/provider_ollama.c b/src/provider_ollama.c index 8da9dc8..4dc87d0 100644 --- a/src/provider_ollama.c +++ b/src/provider_ollama.c @@ -205,8 +205,11 @@ parse_ollama_embedding_response(const char *json_response, int *dim, if (parsed != *dim) { - *error_msg = psprintf("Dimension mismatch: expected %d, got %d", - *dim, parsed); + *error_msg = (parsed == PROVIDER_PARSE_MALFORMED) + ? pstrdup("Malformed embedding response: numeric literal too " + "long to be a number") + : psprintf("Dimension mismatch: expected %d, got %d", + *dim, parsed); pfree(embedding); return NULL; }