diff --git a/Pipeline/Build.Pages.cs b/Pipeline/Build.Pages.cs index 9bac8c0a..308b4cee 100644 --- a/Pipeline/Build.Pages.cs +++ b/Pipeline/Build.Pages.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; @@ -167,43 +168,30 @@ async Task DownloadDocsContent(DocsSource source, AbsolutePath baseDirectory) } : null; - HttpResponseMessage response = await client.GetAsync( + using JsonDocument? listing = await GetGithubJsonAsync(client, $"https://api.github.com/repos/{source.Organization}/{source.Repository}/contents/{source.SourcePath}"); - - string responseContent = await response.Content.ReadAsStringAsync(); - if (!response.IsSuccessStatusCode) + if (listing is null) { - Log.Warning( - $"Skipping {source.Organization}/{source.Repository}: could not list '{source.SourcePath}' ({(int)response.StatusCode} {response.StatusCode}): {responseContent}"); - return; + throw new Exception( + $"GitHub returned 404 for '{source.SourcePath}' in {source.Organization}/{source.Repository}; the aggregated source no longer exists at the configured path."); } - try - { - JsonDocument jsonDocument = JsonDocument.Parse(responseContent); - foreach (JsonElement file in jsonDocument.RootElement.EnumerateArray()) - { - await DownloadFileOrDirectory(client, source, "/", file, baseDirectory, transformer); - } - } - catch (JsonException e) + foreach (JsonElement file in listing.RootElement.EnumerateArray()) { - Log.Error($"Could not parse JSON: {e.Message}\n{responseContent}"); + await DownloadFileOrDirectory(client, source, "/", file, baseDirectory, transformer); } } async Task FetchReadmeIntro(HttpClient client, DocsSource source) { - HttpResponseMessage response = await client.GetAsync( + using JsonDocument? document = await GetGithubJsonAsync(client, $"https://api.github.com/repos/{source.Organization}/{source.Repository}/contents/README.md"); - string responseContent = await response.Content.ReadAsStringAsync(); - if (!response.IsSuccessStatusCode) + if (document is null) { - Log.Warning($"Could not fetch README.md from {source.Organization}/{source.Repository}: {response.StatusCode}"); + Log.Warning($"No README.md in {source.Organization}/{source.Repository}."); return string.Empty; } - using JsonDocument document = JsonDocument.Parse(responseContent); string readme = Base64Decode(document.RootElement.GetProperty("content").GetString()!); int indexOfFirstH2 = readme.IndexOf("\n##", StringComparison.Ordinal); return indexOfFirstH2 > 0 ? readme.Substring(indexOfFirstH2) : string.Empty; @@ -216,16 +204,14 @@ async Task FetchReadmeIntro(HttpClient client, DocsSource source) /// async Task FetchReadmeBody(HttpClient client, string organization, string repository) { - HttpResponseMessage response = await client.GetAsync( + using JsonDocument? document = await GetGithubJsonAsync(client, $"https://api.github.com/repos/{organization}/{repository}/contents/README.md"); - string responseContent = await response.Content.ReadAsStringAsync(); - if (!response.IsSuccessStatusCode) + if (document is null) { - Log.Warning($"Could not fetch README.md from {organization}/{repository}: {response.StatusCode}"); + Log.Warning($"No README.md in {organization}/{repository}."); return string.Empty; } - using JsonDocument document = JsonDocument.Parse(responseContent); string readme = Base64Decode(document.RootElement.GetProperty("content").GetString()!); return StripReadmeFront(readme); } @@ -276,11 +262,16 @@ async Task DownloadFileOrDirectory(HttpClient client, DocsSource source, string string targetName = indexMatch.Success ? "index" + indexMatch.Groups[2].Value : name; int? sidebarPosition = indexMatch.Success ? int.Parse(indexMatch.Groups[1].Value) : null; string filePath = targetDirectory / targetName; - HttpResponseMessage fileResponse = - await client.GetAsync( - $"https://api.github.com/repos/{source.Organization}/{source.Repository}/contents/{source.SourcePath}{subPath}{name}"); - string fileResponseContent = await fileResponse.Content.ReadAsStringAsync(); - using JsonDocument document = JsonDocument.Parse(fileResponseContent); + using JsonDocument? document = await GetGithubJsonAsync(client, + $"https://api.github.com/repos/{source.Organization}/{source.Repository}/contents/{source.SourcePath}{subPath}{name}"); + if (document is null) + { + // The parent directory listing just named this entry, so a 404 here is not a + // legitimate "missing" case but a real error (renamed/removed between calls, or a + // bad response) — fail loudly rather than silently dropping the page. + throw new Exception( + $"GitHub returned 404 for '{source.SourcePath}{subPath}{name}', which its parent directory listing referenced."); + } if (document.RootElement.ValueKind == JsonValueKind.Array) { AbsolutePath subDirectory = targetDirectory / name; @@ -336,6 +327,67 @@ static string EnsureSidebarPosition(string content, int position) return $"---\nsidebar_position: {position}\n---\n\n{content}"; } + /// + /// GETs a GitHub API URL and returns the parsed JSON body. Retries transient failures + /// (HTTP 5xx, 429, 403 secondary-rate-limit, and truncated/malformed 2xx bodies) with + /// backoff, honouring any Retry-After header. Returns null only on a + /// definitive 404 so callers can decide whether a missing resource is fatal. Any other + /// non-success status, or exhausted retries, throws — so a transient GitHub hiccup fails + /// the build loudly instead of silently producing a partial documentation set. + /// + async Task GetGithubJsonAsync(HttpClient client, string url) + { + const int maxAttempts = 5; + for (int attempt = 1;; attempt++) + { + HttpResponseMessage response = await client.GetAsync(url); + string body = await response.Content.ReadAsStringAsync(); + + if (response.IsSuccessStatusCode) + { + try + { + return JsonDocument.Parse(body); + } + catch (JsonException) when (attempt < maxAttempts) + { + Log.Warning($"Malformed JSON from '{url}' (attempt {attempt}/{maxAttempts}); retrying."); + } + } + else if (response.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + else if (!IsTransientStatus(response.StatusCode) || attempt >= maxAttempts) + { + throw new Exception( + $"GitHub request '{url}' failed with {(int)response.StatusCode} {response.StatusCode} after {attempt} attempt(s): {body}"); + } + else + { + Log.Warning( + $"Transient {(int)response.StatusCode} from '{url}' (attempt {attempt}/{maxAttempts}); retrying."); + } + + await Task.Delay(RetryDelay(response, attempt)); + } + } + + static bool IsTransientStatus(HttpStatusCode status) + => (int)status >= 500 || + status == HttpStatusCode.TooManyRequests || + status == HttpStatusCode.Forbidden; + + static TimeSpan RetryDelay(HttpResponseMessage response, int attempt) + { + if (response.Headers.RetryAfter?.Delta is { } delta && delta > TimeSpan.Zero) + { + return delta < TimeSpan.FromMinutes(2) ? delta : TimeSpan.FromMinutes(2); + } + double seconds = Math.Min(30, Math.Pow(2, attempt)); + return TimeSpan.FromSeconds(seconds); + } + static string Base64Decode(string base64EncodedData) { byte[] base64EncodedBytes = Convert.FromBase64String(base64EncodedData);