From a4d38aea37e77ed710df638cbeee95c118f1830a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Fri, 10 Jul 2026 21:12:29 +0200 Subject: [PATCH 1/2] fix(pipeline): stop silently truncating aggregated docs on transient GitHub errors The docs aggregator fetched each file/directory with one unchecked HTTP request and wrapped the per-source loop in a catch(JsonException) that merely logged and continued. A single transient GitHub response mid-burst (a 5xx page, a truncated body, or a secondary-rate-limit throttle) threw while parsing, got swallowed, and abandoned every remaining file in that section - all with a passing build. This is why aweXpect stopped after "events", Mockolate after "benchmarks", and aweXpect.Json disappeared entirely, without CI ever failing. Route every GitHub request through GetGithubJsonAsync, which: - checks the status code before parsing, - retries transient failures (5xx, 429, 403 secondary-rate-limit, and malformed 2xx bodies) with backoff honouring Retry-After, - returns null only on a genuine 404 so a legitimately absent source or README is still skipped cleanly, - throws on any other non-success status or exhausted retries, so a transient hiccup fails the build loudly instead of deploying a half-empty documentation set. --- Pipeline/Build.Pages.cs | 113 +++++++++++++++++++++++++++++----------- 1 file changed, 83 insertions(+), 30 deletions(-) diff --git a/Pipeline/Build.Pages.cs b/Pipeline/Build.Pages.cs index 9bac8c0a..91ab96cb 100644 --- a/Pipeline/Build.Pages.cs +++ b/Pipeline/Build.Pages.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; @@ -167,43 +168,31 @@ async Task DownloadDocsContent(DocsSource source, AbsolutePath baseDirectory) } : null; - HttpResponseMessage response = await client.GetAsync( + using JsonDocument? listing = await GetGithubJsonAsync(client, $"https://api.github.com/repos/{source.Organization}/{source.Repository}/contents/{source.SourcePath}"); - - string responseContent = await response.Content.ReadAsStringAsync(); - if (!response.IsSuccessStatusCode) + if (listing is null) { Log.Warning( - $"Skipping {source.Organization}/{source.Repository}: could not list '{source.SourcePath}' ({(int)response.StatusCode} {response.StatusCode}): {responseContent}"); + $"Skipping {source.Organization}/{source.Repository}: '{source.SourcePath}' does not exist (404)."); return; } - try - { - JsonDocument jsonDocument = JsonDocument.Parse(responseContent); - foreach (JsonElement file in jsonDocument.RootElement.EnumerateArray()) - { - await DownloadFileOrDirectory(client, source, "/", file, baseDirectory, transformer); - } - } - catch (JsonException e) + foreach (JsonElement file in listing.RootElement.EnumerateArray()) { - Log.Error($"Could not parse JSON: {e.Message}\n{responseContent}"); + await DownloadFileOrDirectory(client, source, "/", file, baseDirectory, transformer); } } async Task FetchReadmeIntro(HttpClient client, DocsSource source) { - HttpResponseMessage response = await client.GetAsync( + using JsonDocument? document = await GetGithubJsonAsync(client, $"https://api.github.com/repos/{source.Organization}/{source.Repository}/contents/README.md"); - string responseContent = await response.Content.ReadAsStringAsync(); - if (!response.IsSuccessStatusCode) + if (document is null) { - Log.Warning($"Could not fetch README.md from {source.Organization}/{source.Repository}: {response.StatusCode}"); + Log.Warning($"No README.md in {source.Organization}/{source.Repository}."); return string.Empty; } - using JsonDocument document = JsonDocument.Parse(responseContent); string readme = Base64Decode(document.RootElement.GetProperty("content").GetString()!); int indexOfFirstH2 = readme.IndexOf("\n##", StringComparison.Ordinal); return indexOfFirstH2 > 0 ? readme.Substring(indexOfFirstH2) : string.Empty; @@ -216,16 +205,14 @@ async Task FetchReadmeIntro(HttpClient client, DocsSource source) /// async Task FetchReadmeBody(HttpClient client, string organization, string repository) { - HttpResponseMessage response = await client.GetAsync( + using JsonDocument? document = await GetGithubJsonAsync(client, $"https://api.github.com/repos/{organization}/{repository}/contents/README.md"); - string responseContent = await response.Content.ReadAsStringAsync(); - if (!response.IsSuccessStatusCode) + if (document is null) { - Log.Warning($"Could not fetch README.md from {organization}/{repository}: {response.StatusCode}"); + Log.Warning($"No README.md in {organization}/{repository}."); return string.Empty; } - using JsonDocument document = JsonDocument.Parse(responseContent); string readme = Base64Decode(document.RootElement.GetProperty("content").GetString()!); return StripReadmeFront(readme); } @@ -276,11 +263,16 @@ async Task DownloadFileOrDirectory(HttpClient client, DocsSource source, string string targetName = indexMatch.Success ? "index" + indexMatch.Groups[2].Value : name; int? sidebarPosition = indexMatch.Success ? int.Parse(indexMatch.Groups[1].Value) : null; string filePath = targetDirectory / targetName; - HttpResponseMessage fileResponse = - await client.GetAsync( - $"https://api.github.com/repos/{source.Organization}/{source.Repository}/contents/{source.SourcePath}{subPath}{name}"); - string fileResponseContent = await fileResponse.Content.ReadAsStringAsync(); - using JsonDocument document = JsonDocument.Parse(fileResponseContent); + using JsonDocument? document = await GetGithubJsonAsync(client, + $"https://api.github.com/repos/{source.Organization}/{source.Repository}/contents/{source.SourcePath}{subPath}{name}"); + if (document is null) + { + // The parent directory listing just named this entry, so a 404 here is not a + // legitimate "missing" case but a real error (renamed/removed between calls, or a + // bad response) — fail loudly rather than silently dropping the page. + throw new Exception( + $"GitHub returned 404 for '{source.SourcePath}{subPath}{name}', which its parent directory listing referenced."); + } if (document.RootElement.ValueKind == JsonValueKind.Array) { AbsolutePath subDirectory = targetDirectory / name; @@ -336,6 +328,67 @@ static string EnsureSidebarPosition(string content, int position) return $"---\nsidebar_position: {position}\n---\n\n{content}"; } + /// + /// GETs a GitHub API URL and returns the parsed JSON body. Retries transient failures + /// (HTTP 5xx, 429, 403 secondary-rate-limit, and truncated/malformed 2xx bodies) with + /// backoff, honouring any Retry-After header. Returns null only on a + /// definitive 404 so callers can decide whether a missing resource is fatal. Any other + /// non-success status, or exhausted retries, throws — so a transient GitHub hiccup fails + /// the build loudly instead of silently producing a partial documentation set. + /// + async Task GetGithubJsonAsync(HttpClient client, string url) + { + const int maxAttempts = 5; + for (int attempt = 1;; attempt++) + { + HttpResponseMessage response = await client.GetAsync(url); + string body = await response.Content.ReadAsStringAsync(); + + if (response.IsSuccessStatusCode) + { + try + { + return JsonDocument.Parse(body); + } + catch (JsonException) when (attempt < maxAttempts) + { + Log.Warning($"Malformed JSON from '{url}' (attempt {attempt}/{maxAttempts}); retrying."); + } + } + else if (response.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + else if (!IsTransientStatus(response.StatusCode) || attempt >= maxAttempts) + { + throw new Exception( + $"GitHub request '{url}' failed with {(int)response.StatusCode} {response.StatusCode} after {attempt} attempt(s): {body}"); + } + else + { + Log.Warning( + $"Transient {(int)response.StatusCode} from '{url}' (attempt {attempt}/{maxAttempts}); retrying."); + } + + await Task.Delay(RetryDelay(response, attempt)); + } + } + + static bool IsTransientStatus(HttpStatusCode status) + => (int)status >= 500 || + status == HttpStatusCode.TooManyRequests || + status == HttpStatusCode.Forbidden; + + static TimeSpan RetryDelay(HttpResponseMessage response, int attempt) + { + if (response.Headers.RetryAfter?.Delta is { } delta && delta > TimeSpan.Zero) + { + return delta < TimeSpan.FromMinutes(2) ? delta : TimeSpan.FromMinutes(2); + } + double seconds = Math.Min(30, Math.Pow(2, attempt)); + return TimeSpan.FromSeconds(seconds); + } + static string Base64Decode(string base64EncodedData) { byte[] base64EncodedBytes = Convert.FromBase64String(base64EncodedData); From 36038892f3cdffc35a547c6005ec0b900f84bd38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Fri, 10 Jul 2026 21:38:41 +0200 Subject: [PATCH 2/2] fix(pipeline): fail the build when an aggregated source path 404s A listing-level 404 previously logged a warning and skipped the whole slice, silently dropping an entire documentation section. Since every aggregated source is a hardcoded, non-optional slice, a 404 means the configured repo/path is wrong -- fail loudly instead, matching the child-level 404 handling. --- Pipeline/Build.Pages.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Pipeline/Build.Pages.cs b/Pipeline/Build.Pages.cs index 91ab96cb..308b4cee 100644 --- a/Pipeline/Build.Pages.cs +++ b/Pipeline/Build.Pages.cs @@ -172,9 +172,8 @@ async Task DownloadDocsContent(DocsSource source, AbsolutePath baseDirectory) $"https://api.github.com/repos/{source.Organization}/{source.Repository}/contents/{source.SourcePath}"); if (listing is null) { - Log.Warning( - $"Skipping {source.Organization}/{source.Repository}: '{source.SourcePath}' does not exist (404)."); - return; + throw new Exception( + $"GitHub returned 404 for '{source.SourcePath}' in {source.Organization}/{source.Repository}; the aggregated source no longer exists at the configured path."); } foreach (JsonElement file in listing.RootElement.EnumerateArray())