Do more fetching with retries (#45)

Author: dsherret

.github/workflows/ci.yml

@@ -9,5 +9,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: npm install
+      - name: Type check
+        run: npx tsc --noEmit
       - name: Run tests
         run: npx vitest run

env.d.ts

@@ -8,7 +8,9 @@ declare module "*.txt" {
   export default content;
 }
 
-interface Env {
-  DPRINT_PLUGINS_GH_TOKEN?: string;
-  PLUGIN_CACHE: R2Bucket;
+declare namespace Cloudflare {
+  interface Env {
+    DPRINT_PLUGINS_GH_TOKEN?: string;
+    PLUGIN_CACHE: R2Bucket;
+  }
 }

handleRequest.ts

@@ -1,14 +1,10 @@
 import { renderHome } from "./home.jsx";
 import oldMappings from "./old_redirects.json" with { type: "json" };
-import {
-  tryResolveAssetUrl,
-  tryResolveLatestJson,
-  tryResolvePluginUrl,
-  tryResolveSchemaUrl,
-} from "./plugins.js";
+import { tryResolveAssetUrl, tryResolveLatestJson, tryResolvePluginUrl, tryResolveSchemaUrl } from "./plugins.js";
 import { readInfoFile } from "./readInfoFile.js";
 import robotsTxt from "./robots.txt";
 import styleCSS from "./style.css";
+import { fetchWithRetries } from "./utils/fetchWithRetries.js";
 import { LruCache } from "./utils/LruCache.js";
 import { getCliInfo } from "./utils/mod.js";
 import { r2Get, r2Put } from "./utils/r2Cache.js";
@@ -94,7 +90,7 @@ export function createRequestHandler() {
       }
 
       if (url.pathname === "/") {
-        return renderHome().then((home) =>
+        return renderHome(url.origin).then((home) =>
           new Response(home, {
             headers: {
               "content-type": contentTypes.html,
@@ -130,14 +126,16 @@ export function createRequestHandler() {
     githubUrl: string,
     ctx?: ExecutionContext,
   ): Promise<{ body: ArrayBuffer | ReadableStream | null; status: number; contentType: string }> {
-    // L1: in-memory cache (already rewritten)
+    // L1: in-memory cache
     const cached = memoryCache.get(githubUrl);
     if (cached != null) {
       return { body: cached.body, status: 200, contentType: cached.contentType };
     }
 
     const result = await fetchBody(githubUrl, ctx);
-    if (result.status === 200 && result.body instanceof ArrayBuffer && result.body.byteLength <= MAX_MEM_CACHE_BODY_SIZE) {
+    if (
+      result.status === 200 && result.body instanceof ArrayBuffer && result.body.byteLength <= MAX_MEM_CACHE_BODY_SIZE
+    ) {
       memoryCache.set(githubUrl, { body: result.body, contentType: result.contentType });
     }
 
@@ -160,7 +158,11 @@ export function createRequestHandler() {
     }
 
     // L3: fetch from GitHub
-    const response = await fetchWithRetries(githubUrl);
+    const response = await fetchWithRetries(githubUrl, {
+      // don't need the github token here because these assets
+      // are not part of the github api
+      headers: { "user-agent": "dprint-plugins" },
+    });
     if (!response.ok) {
       if (response.status !== 404) {
         console.error(`GitHub fetch error: ${response.status} ${response.statusText} for ${githubUrl}`);
@@ -227,27 +229,12 @@ function githubUrlToAssetPath(githubUrl: string) {
   return `/${username}/${repo}/${tag}/asset/${fileName}`;
 }
 
-
 function contentTypeForUrl(url: string) {
   if (url.endsWith(".wasm")) return contentTypes.wasm;
   if (url.endsWith(".json") || url.endsWith(".exe-plugin")) return contentTypes.json;
   return contentTypes.octetStream;
 }
 
-async function fetchWithRetries(url: string, retries = 3): Promise<Response> {
-  for (let i = 0; i <= retries; i++) {
-    const response = await fetch(url, {
-      headers: { "user-agent": "dprint-plugins" },
-    });
-    if (response.status < 500 || i === retries) {
-      return response;
-    }
-    console.error(`GitHub fetch attempt ${i + 1} failed: ${response.status} for ${url}`);
-    await new Promise((resolve) => setTimeout(resolve, Math.min(1000 * 2 ** i, 2500)));
-  }
-  throw new Error("unreachable");
-}
-
 function create404Response() {
   return new Response(null, {
     status: 404,

home.tsx

@@ -1,8 +1,8 @@
 import { renderToString } from "preact-render-to-string";
 import { PluginData, PluginsData, readInfoFile } from "./readInfoFile.js";
 
-export async function renderHome() {
-  const content = await renderContent();
+export async function renderHome(origin: string) {
+  const content = await renderContent(origin);
   return `<!DOCTYPE html>
 <html lang="en">
   <head>
@@ -34,8 +34,8 @@ export async function renderHome() {
 `;
 }
 
-async function renderContent() {
-  const pluginsData = await readInfoFile();
+async function renderContent(origin: string) {
+  const pluginsData = await readInfoFile(origin);
   const section = (
     <section id="content">
       <h1>Plugins</h1>

package-lock.json

@@ -14,6 +14,7 @@
   "devDependencies": {
     "@cloudflare/vitest-pool-workers": "^0.13.4",
     "@cloudflare/workers-types": "^4.20260317.1",
+    "typescript": "^6.0.2",
     "vitest": "~4.1.1",
     "wrangler": "^4.77.0"
   }

package.json

@@ -0,0 +1,15 @@
+export async function fetchWithRetries(
+  url: string,
+  init?: RequestInit,
+  retries = 3,
+): Promise<Response> {
+  for (let i = 0; i <= retries; i++) {
+    const response = await fetch(url, init);
+    if (response.status < 500 || i === retries) {
+      return response;
+    }
+    console.error(`Fetch attempt ${i + 1} failed: ${response.status} for ${url}`);
+    await new Promise((resolve) => setTimeout(resolve, Math.min(1000 * 2 ** i, 2500)));
+  }
+  throw new Error("unreachable");
+}

utils/fetchWithRetries.ts

@@ -1,4 +1,5 @@
 import { env } from "cloudflare:workers";
+import { fetchWithRetries } from "./fetchWithRetries.js";
 import { LazyExpirableValue } from "./LazyExpirableValue.js";
 import { LruCache, LruCacheWithExpiry } from "./LruCache.js";
 import { createSynchronizedActioner } from "./synchronizedActioner.js";
@@ -170,17 +171,18 @@ const synchronizedActioner = createSynchronizedActioner();
 function makeGitHubGetRequest(url: string, method: "GET" | "HEAD") {
   console.log(`Making request to ${url}`);
   return synchronizedActioner.doActionWithTimeout((signal) => {
-    return fetch(url, {
+    return fetchWithRetries(url, {
       method,
       headers: getGitHubHeaders(),
       signal,
-    });
+    }, /* retries */ 1);
   }, 10_000);
 }
 
-function getGitHubHeaders() {
+// headers for GitHub API requests (not raw asset downloads,
+// which don't need auth for public repos)
+function getGitHubDownloadHeaders() {
   const headers: Record<string, string> = {
-    "accept": "application/vnd.github.v3+json",
     "user-agent": "dprint-plugins",
   };
   const token = env.DPRINT_PLUGINS_GH_TOKEN;
@@ -189,3 +191,10 @@ function getGitHubHeaders() {
   }
   return headers;
 }
+
+function getGitHubHeaders() {
+  return {
+    ...getGitHubDownloadHeaders(),
+    "accept": "application/vnd.github.v3+json",
+  };
+}