refactor: share corpus archive handling

tools/corpus/acquire.ts

@@ -1,7 +1,16 @@
-import { execFileSync } from "node:child_process";
 import { createHash } from "node:crypto";
 import { mkdirSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
+import {
+  ACQUIRE_FONT_EXTENSIONS,
+  type ArchiveFormat,
+  archiveFormatOf,
+  archivePathFor,
+  hasFontExtension,
+  listArchiveMembers,
+  readArchiveMember,
+  requireArchiveTool,
+} from "./src/archive";
 
 const GUST_LICENSE_URL =
   "https://www.gust.org.pl/projects/e-foundry/licenses/GUST-FONT-LICENSE.txt/at_download/file";
@@ -37,9 +46,6 @@ export type LicenseFamily =
   | "GPL-2.0-FE"
   | "Bitstream-Vera-DejaVu";
 
-/** Container format an archive source ships in. Absent is treated as "zip". */
-export type ArchiveFormat = "zip" | "tar.gz";
-
 /** Fields shared by every acquisition source, regardless of how it is fetched. */
 interface BaseSource {
   sourceId: string;
@@ -553,15 +559,14 @@ type SourceSnapshot = ArchiveSnapshot | GitHubTreeSnapshot;
 
 const REPO_DIR = join(import.meta.dir, "..", "..");
 const DEFAULT_CACHE_DIR = join(REPO_DIR, ".cache", "corpus");
-const FONT_EXTENSIONS = [".otf", ".ttf", ".otc", ".ttc", ".woff2", ".woff"];
 
 const sha256 = (bytes: Uint8Array): string =>
   createHash("sha256").update(bytes).digest("hex");
 
 const basename = (path: string): string => path.split("/").pop() ?? path;
 
 const isFontFile = (path: string): boolean =>
-  FONT_EXTENSIONS.some((ext) => path.toLowerCase().endsWith(ext));
+  hasFontExtension(path, ACQUIRE_FONT_EXTENSIONS);
 
 async function fetchBytes(url: string): Promise<Uint8Array> {
   const res = await fetch(url);
@@ -670,73 +675,16 @@ async function mapLimit<T, U>(
   return results;
 }
 
-const archiveFormatOf = (source: ArchiveSource): ArchiveFormat =>
-  source.archiveFormat ?? "zip";
-
-const archiveExtensions: Record<ArchiveFormat, string> = {
-  zip: "zip",
-  "tar.gz": "tar.gz",
-};
-
-function requireArchiveTool(format: ArchiveFormat): void {
-  const tool = format === "tar.gz" ? "tar" : "unzip";
-  const probe = format === "tar.gz" ? "--version" : "-v";
-  try {
-    execFileSync(tool, [probe], { stdio: "ignore" });
-  } catch {
-    throw new Error(`\`${tool}\` is required on PATH.`);
-  }
-}
-
-function listArchive(archivePath: string, format: ArchiveFormat): string[] {
-  const out =
-    format === "tar.gz"
-      ? execFileSync("tar", ["-tzf", archivePath], { encoding: "utf8" })
-      : execFileSync("unzip", ["-Z1", archivePath], { encoding: "utf8" });
-  return out
-    .split("\n")
-    .map((line) => line.trim())
-    .filter(Boolean);
-}
-
-// `unzip -p` matches its member argument as a glob, so members with literal glob
-// metacharacters (e.g. variable-font names like `NotoSans-Italic[wdth,wght].ttf`)
-// must be escaped to extract by exact name.
-const escapeArchiveMember = (name: string): string =>
-  name.replace(/[\\*?[\]]/g, "\\$&");
-
-function readArchiveMember(
-  archivePath: string,
-  name: string,
-  format: ArchiveFormat,
-): Uint8Array {
-  const opts = { maxBuffer: 256 * 1024 * 1024 };
-  // tar takes the member as a path after `--`; the tar.gz sources use plain ASCII
-  // member names, so the glob escaping that `unzip -p` needs does not apply here.
-  return new Uint8Array(
-    format === "tar.gz"
-      ? execFileSync("tar", ["-xzOf", archivePath, "--", name], opts)
-      : execFileSync(
-          "unzip",
-          ["-p", archivePath, escapeArchiveMember(name)],
-          opts,
-        ),
-  );
-}
-
 async function acquireArchive(
   source: ArchiveSource,
   cacheDir: string,
 ): Promise<ArchiveSnapshot> {
   const format = archiveFormatOf(source);
   const archive = await fetchBytes(source.downloadUrl);
-  const archivePath = join(
-    cacheDir,
-    `${source.sourceId}.${archiveExtensions[format]}`,
-  );
+  const archivePath = archivePathFor(cacheDir, source.sourceId, format);
   writeFileSync(archivePath, archive);
 
-  const members = listArchive(archivePath, format).filter(isFontFile);
+  const members = listArchiveMembers(archivePath, format).filter(isFontFile);
   if (members.length === 0)
     throw new Error(`${source.sourceId}: archive has no font files`);
 

tools/corpus/archive.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, test } from "bun:test";
+import {
+  ACQUIRE_FONT_EXTENSIONS,
+  archiveFormatOf,
+  archivePathFor,
+  hasFontExtension,
+  RAW_SFNT_EXTENSIONS,
+} from "./src/archive";
+
+describe("archive helpers", () => {
+  test("defaults archive sources to zip and names tar.gz cache files", () => {
+    expect(archiveFormatOf({})).toBe("zip");
+    expect(archiveFormatOf({ archiveFormat: "tar.gz" })).toBe("tar.gz");
+    expect(archivePathFor("/cache", "liberation", "tar.gz")).toBe(
+      "/cache/liberation.tar.gz",
+    );
+    expect(archivePathFor("/cache", "selawik", "zip")).toBe(
+      "/cache/selawik.zip",
+    );
+  });
+
+  test("keeps acquire and compare font extension policies explicit", () => {
+    expect(hasFontExtension("Example-Regular.ttf", RAW_SFNT_EXTENSIONS)).toBe(
+      true,
+    );
+    expect(hasFontExtension("Example-Regular.woff2", RAW_SFNT_EXTENSIONS)).toBe(
+      false,
+    );
+    expect(
+      hasFontExtension("Example-Regular.woff2", ACQUIRE_FONT_EXTENSIONS),
+    ).toBe(true);
+    expect(
+      hasFontExtension("ExampleCollection.ttc", ACQUIRE_FONT_EXTENSIONS),
+    ).toBe(true);
+  });
+});

tools/corpus/src/archive.ts

@@ -0,0 +1,84 @@
+import { execFileSync } from "node:child_process";
+import { join } from "node:path";
+
+export type ArchiveFormat = "zip" | "tar.gz";
+
+export const RAW_SFNT_EXTENSIONS = [".otf", ".ttf"] as const;
+export const ACQUIRE_FONT_EXTENSIONS = [
+  ...RAW_SFNT_EXTENSIONS,
+  ".otc",
+  ".ttc",
+  ".woff2",
+  ".woff",
+] as const;
+
+const ARCHIVE_EXTENSIONS: Record<ArchiveFormat, string> = {
+  zip: "zip",
+  "tar.gz": "tar.gz",
+};
+
+export const archiveFormatOf = <T extends { archiveFormat?: ArchiveFormat }>(
+  source: T,
+): ArchiveFormat => source.archiveFormat ?? "zip";
+
+export function archivePathFor(
+  cacheDir: string,
+  sourceId: string,
+  format: ArchiveFormat,
+): string {
+  return join(cacheDir, `${sourceId}.${ARCHIVE_EXTENSIONS[format]}`);
+}
+
+export function requireArchiveTool(format: ArchiveFormat): void {
+  const tool = format === "tar.gz" ? "tar" : "unzip";
+  const probe = format === "tar.gz" ? "--version" : "-v";
+  try {
+    execFileSync(tool, [probe], { stdio: "ignore" });
+  } catch {
+    throw new Error(`\`${tool}\` is required on PATH.`);
+  }
+}
+
+export function hasFontExtension(
+  path: string,
+  extensions: readonly string[],
+): boolean {
+  const lower = path.toLowerCase();
+  return extensions.some((ext) => lower.endsWith(ext));
+}
+
+export function listArchiveMembers(
+  archivePath: string,
+  format: ArchiveFormat,
+): string[] {
+  const out =
+    format === "tar.gz"
+      ? execFileSync("tar", ["-tzf", archivePath], { encoding: "utf8" })
+      : execFileSync("unzip", ["-Z1", archivePath], { encoding: "utf8" });
+  return out
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(Boolean);
+}
+
+// `unzip -p` matches its member argument as a glob, so members with literal glob
+// metacharacters must be escaped to extract by exact name.
+const escapeArchiveMember = (name: string): string =>
+  name.replace(/[\\*?[\]]/g, "\\$&");
+
+export function readArchiveMember(
+  archivePath: string,
+  member: string,
+  format: ArchiveFormat,
+): Uint8Array {
+  const opts = { maxBuffer: 256 * 1024 * 1024 };
+  return new Uint8Array(
+    format === "tar.gz"
+      ? execFileSync("tar", ["-xzOf", archivePath, "--", member], opts)
+      : execFileSync(
+          "unzip",
+          ["-p", archivePath, escapeArchiveMember(member)],
+          opts,
+        ),
+  );
+}

tools/corpus/src/cache.ts

@@ -1,8 +1,16 @@
-import { execFileSync } from "node:child_process";
 import { existsSync, readFileSync } from "node:fs";
 import { basename, join } from "node:path";
+import {
+  type ArchiveFormat,
+  archiveFormatOf,
+  archivePathFor,
+  hasFontExtension,
+  listArchiveMembers,
+  RAW_SFNT_EXTENSIONS,
+  readArchiveMember,
+  requireArchiveTool,
+} from "./archive";
 
-const RAW_SFNT_EXTENSIONS = [".otf", ".ttf"];
 const SNAPSHOT_FILE = "source-snapshot.json";
 
 /** One snapshot file entry: a font member by path, with its display name. */
@@ -11,8 +19,6 @@ interface SnapshotFile {
   path: string;
 }
 
-export type ArchiveFormat = "zip" | "tar.gz";
-
 /**
  * A source as recorded in `source-snapshot.json`. Archive sources extract their candidate fonts from a
  * cached release archive; GitHub tree sources read each `files[].path` directly from the cache. `kind` is
@@ -33,61 +39,12 @@ export interface CandidateFile {
   bytes: Uint8Array;
 }
 
-export const archiveFormatOf = (source: SnapshotSource): ArchiveFormat =>
-  source.archiveFormat ?? "zip";
-
-const archiveExtensions: Record<ArchiveFormat, string> = {
-  zip: "zip",
-  "tar.gz": "tar.gz",
-};
-
-export function requireArchiveTool(format: ArchiveFormat): void {
-  const tool = format === "tar.gz" ? "tar" : "unzip";
-  const probe = format === "tar.gz" ? "--version" : "-v";
-  try {
-    execFileSync(tool, [probe], { stdio: "ignore" });
-  } catch {
-    throw new Error(`\`${tool}\` is required on PATH.`);
-  }
-}
-
-function isFontFile(path: string): boolean {
-  return RAW_SFNT_EXTENSIONS.some((ext) => path.toLowerCase().endsWith(ext));
-}
+export { type ArchiveFormat, archiveFormatOf, requireArchiveTool };
 
 /** Font members inside a source archive, by their in-archive path. */
 function listFontMembers(archivePath: string, format: ArchiveFormat): string[] {
-  const out =
-    format === "tar.gz"
-      ? execFileSync("tar", ["-tzf", archivePath], { encoding: "utf8" })
-      : execFileSync("unzip", ["-Z1", archivePath], { encoding: "utf8" });
-  return out
-    .split("\n")
-    .map((line) => line.trim())
-    .filter(Boolean)
-    .filter(isFontFile);
-}
-
-// `unzip -p` matches its member argument as a glob, so members with literal glob
-// metacharacters (e.g. variable-font names like `NotoSans-Italic[wdth,wght].ttf`)
-// must be escaped to extract by exact name.
-const escapeArchiveMember = (name: string): string =>
-  name.replace(/[\\*?[\]]/g, "\\$&");
-
-function readArchiveMember(
-  archivePath: string,
-  member: string,
-  format: ArchiveFormat,
-): Uint8Array {
-  const opts = { maxBuffer: 256 * 1024 * 1024 };
-  return new Uint8Array(
-    format === "tar.gz"
-      ? execFileSync("tar", ["-xzOf", archivePath, "--", member], opts)
-      : execFileSync(
-          "unzip",
-          ["-p", archivePath, escapeArchiveMember(member)],
-          opts,
-        ),
+  return listArchiveMembers(archivePath, format).filter((member) =>
+    hasFontExtension(member, RAW_SFNT_EXTENSIONS),
   );
 }
 
@@ -135,10 +92,7 @@ export function collectCandidates(
   }
 
   const format = archiveFormatOf(source);
-  const archivePath = join(
-    cacheDir,
-    `${source.sourceId}.${archiveExtensions[format]}`,
-  );
+  const archivePath = archivePathFor(cacheDir, source.sourceId, format);
   if (!existsSync(archivePath))
     throw new Error(
       `candidate archive missing for ${source.sourceId}: ${archivePath}. Run \`bun run corpus:acquire\` first.`,