diff --git a/tools/corpus/acquire.ts b/tools/corpus/acquire.ts index a6ff5c5..7c69d3f 100644 --- a/tools/corpus/acquire.ts +++ b/tools/corpus/acquire.ts @@ -1,7 +1,16 @@ -import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdirSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; +import { + ACQUIRE_FONT_EXTENSIONS, + type ArchiveFormat, + archiveFormatOf, + archivePathFor, + hasFontExtension, + listArchiveMembers, + readArchiveMember, + requireArchiveTool, +} from "./src/archive"; const GUST_LICENSE_URL = "https://www.gust.org.pl/projects/e-foundry/licenses/GUST-FONT-LICENSE.txt/at_download/file"; @@ -37,9 +46,6 @@ export type LicenseFamily = | "GPL-2.0-FE" | "Bitstream-Vera-DejaVu"; -/** Container format an archive source ships in. Absent is treated as "zip". */ -export type ArchiveFormat = "zip" | "tar.gz"; - /** Fields shared by every acquisition source, regardless of how it is fetched. */ interface BaseSource { sourceId: string; @@ -553,7 +559,6 @@ type SourceSnapshot = ArchiveSnapshot | GitHubTreeSnapshot; const REPO_DIR = join(import.meta.dir, "..", ".."); const DEFAULT_CACHE_DIR = join(REPO_DIR, ".cache", "corpus"); -const FONT_EXTENSIONS = [".otf", ".ttf", ".otc", ".ttc", ".woff2", ".woff"]; const sha256 = (bytes: Uint8Array): string => createHash("sha256").update(bytes).digest("hex"); @@ -561,7 +566,7 @@ const sha256 = (bytes: Uint8Array): string => const basename = (path: string): string => path.split("/").pop() ?? path; const isFontFile = (path: string): boolean => - FONT_EXTENSIONS.some((ext) => path.toLowerCase().endsWith(ext)); + hasFontExtension(path, ACQUIRE_FONT_EXTENSIONS); async function fetchBytes(url: string): Promise { const res = await fetch(url); @@ -670,73 +675,16 @@ async function mapLimit( return results; } -const archiveFormatOf = (source: ArchiveSource): ArchiveFormat => - source.archiveFormat ?? "zip"; - -const archiveExtensions: Record = { - zip: "zip", - "tar.gz": "tar.gz", -}; - -function requireArchiveTool(format: ArchiveFormat): void { - const tool = format === "tar.gz" ? "tar" : "unzip"; - const probe = format === "tar.gz" ? "--version" : "-v"; - try { - execFileSync(tool, [probe], { stdio: "ignore" }); - } catch { - throw new Error(`\`${tool}\` is required on PATH.`); - } -} - -function listArchive(archivePath: string, format: ArchiveFormat): string[] { - const out = - format === "tar.gz" - ? execFileSync("tar", ["-tzf", archivePath], { encoding: "utf8" }) - : execFileSync("unzip", ["-Z1", archivePath], { encoding: "utf8" }); - return out - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); -} - -// `unzip -p` matches its member argument as a glob, so members with literal glob -// metacharacters (e.g. variable-font names like `NotoSans-Italic[wdth,wght].ttf`) -// must be escaped to extract by exact name. -const escapeArchiveMember = (name: string): string => - name.replace(/[\\*?[\]]/g, "\\$&"); - -function readArchiveMember( - archivePath: string, - name: string, - format: ArchiveFormat, -): Uint8Array { - const opts = { maxBuffer: 256 * 1024 * 1024 }; - // tar takes the member as a path after `--`; the tar.gz sources use plain ASCII - // member names, so the glob escaping that `unzip -p` needs does not apply here. - return new Uint8Array( - format === "tar.gz" - ? execFileSync("tar", ["-xzOf", archivePath, "--", name], opts) - : execFileSync( - "unzip", - ["-p", archivePath, escapeArchiveMember(name)], - opts, - ), - ); -} - async function acquireArchive( source: ArchiveSource, cacheDir: string, ): Promise { const format = archiveFormatOf(source); const archive = await fetchBytes(source.downloadUrl); - const archivePath = join( - cacheDir, - `${source.sourceId}.${archiveExtensions[format]}`, - ); + const archivePath = archivePathFor(cacheDir, source.sourceId, format); writeFileSync(archivePath, archive); - const members = listArchive(archivePath, format).filter(isFontFile); + const members = listArchiveMembers(archivePath, format).filter(isFontFile); if (members.length === 0) throw new Error(`${source.sourceId}: archive has no font files`); diff --git a/tools/corpus/archive.test.ts b/tools/corpus/archive.test.ts new file mode 100644 index 0000000..efd7ed7 --- /dev/null +++ b/tools/corpus/archive.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, test } from "bun:test"; +import { + ACQUIRE_FONT_EXTENSIONS, + archiveFormatOf, + archivePathFor, + hasFontExtension, + RAW_SFNT_EXTENSIONS, +} from "./src/archive"; + +describe("archive helpers", () => { + test("defaults archive sources to zip and names tar.gz cache files", () => { + expect(archiveFormatOf({})).toBe("zip"); + expect(archiveFormatOf({ archiveFormat: "tar.gz" })).toBe("tar.gz"); + expect(archivePathFor("/cache", "liberation", "tar.gz")).toBe( + "/cache/liberation.tar.gz", + ); + expect(archivePathFor("/cache", "selawik", "zip")).toBe( + "/cache/selawik.zip", + ); + }); + + test("keeps acquire and compare font extension policies explicit", () => { + expect(hasFontExtension("Example-Regular.ttf", RAW_SFNT_EXTENSIONS)).toBe( + true, + ); + expect(hasFontExtension("Example-Regular.woff2", RAW_SFNT_EXTENSIONS)).toBe( + false, + ); + expect( + hasFontExtension("Example-Regular.woff2", ACQUIRE_FONT_EXTENSIONS), + ).toBe(true); + expect( + hasFontExtension("ExampleCollection.ttc", ACQUIRE_FONT_EXTENSIONS), + ).toBe(true); + }); +}); diff --git a/tools/corpus/src/archive.ts b/tools/corpus/src/archive.ts new file mode 100644 index 0000000..8d4ca45 --- /dev/null +++ b/tools/corpus/src/archive.ts @@ -0,0 +1,84 @@ +import { execFileSync } from "node:child_process"; +import { join } from "node:path"; + +export type ArchiveFormat = "zip" | "tar.gz"; + +export const RAW_SFNT_EXTENSIONS = [".otf", ".ttf"] as const; +export const ACQUIRE_FONT_EXTENSIONS = [ + ...RAW_SFNT_EXTENSIONS, + ".otc", + ".ttc", + ".woff2", + ".woff", +] as const; + +const ARCHIVE_EXTENSIONS: Record = { + zip: "zip", + "tar.gz": "tar.gz", +}; + +export const archiveFormatOf = ( + source: T, +): ArchiveFormat => source.archiveFormat ?? "zip"; + +export function archivePathFor( + cacheDir: string, + sourceId: string, + format: ArchiveFormat, +): string { + return join(cacheDir, `${sourceId}.${ARCHIVE_EXTENSIONS[format]}`); +} + +export function requireArchiveTool(format: ArchiveFormat): void { + const tool = format === "tar.gz" ? "tar" : "unzip"; + const probe = format === "tar.gz" ? "--version" : "-v"; + try { + execFileSync(tool, [probe], { stdio: "ignore" }); + } catch { + throw new Error(`\`${tool}\` is required on PATH.`); + } +} + +export function hasFontExtension( + path: string, + extensions: readonly string[], +): boolean { + const lower = path.toLowerCase(); + return extensions.some((ext) => lower.endsWith(ext)); +} + +export function listArchiveMembers( + archivePath: string, + format: ArchiveFormat, +): string[] { + const out = + format === "tar.gz" + ? execFileSync("tar", ["-tzf", archivePath], { encoding: "utf8" }) + : execFileSync("unzip", ["-Z1", archivePath], { encoding: "utf8" }); + return out + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); +} + +// `unzip -p` matches its member argument as a glob, so members with literal glob +// metacharacters must be escaped to extract by exact name. +const escapeArchiveMember = (name: string): string => + name.replace(/[\\*?[\]]/g, "\\$&"); + +export function readArchiveMember( + archivePath: string, + member: string, + format: ArchiveFormat, +): Uint8Array { + const opts = { maxBuffer: 256 * 1024 * 1024 }; + return new Uint8Array( + format === "tar.gz" + ? execFileSync("tar", ["-xzOf", archivePath, "--", member], opts) + : execFileSync( + "unzip", + ["-p", archivePath, escapeArchiveMember(member)], + opts, + ), + ); +} diff --git a/tools/corpus/src/cache.ts b/tools/corpus/src/cache.ts index 053d2bc..089b07e 100644 --- a/tools/corpus/src/cache.ts +++ b/tools/corpus/src/cache.ts @@ -1,8 +1,16 @@ -import { execFileSync } from "node:child_process"; import { existsSync, readFileSync } from "node:fs"; import { basename, join } from "node:path"; +import { + type ArchiveFormat, + archiveFormatOf, + archivePathFor, + hasFontExtension, + listArchiveMembers, + RAW_SFNT_EXTENSIONS, + readArchiveMember, + requireArchiveTool, +} from "./archive"; -const RAW_SFNT_EXTENSIONS = [".otf", ".ttf"]; const SNAPSHOT_FILE = "source-snapshot.json"; /** One snapshot file entry: a font member by path, with its display name. */ @@ -11,8 +19,6 @@ interface SnapshotFile { path: string; } -export type ArchiveFormat = "zip" | "tar.gz"; - /** * A source as recorded in `source-snapshot.json`. Archive sources extract their candidate fonts from a * cached release archive; GitHub tree sources read each `files[].path` directly from the cache. `kind` is @@ -33,61 +39,12 @@ export interface CandidateFile { bytes: Uint8Array; } -export const archiveFormatOf = (source: SnapshotSource): ArchiveFormat => - source.archiveFormat ?? "zip"; - -const archiveExtensions: Record = { - zip: "zip", - "tar.gz": "tar.gz", -}; - -export function requireArchiveTool(format: ArchiveFormat): void { - const tool = format === "tar.gz" ? "tar" : "unzip"; - const probe = format === "tar.gz" ? "--version" : "-v"; - try { - execFileSync(tool, [probe], { stdio: "ignore" }); - } catch { - throw new Error(`\`${tool}\` is required on PATH.`); - } -} - -function isFontFile(path: string): boolean { - return RAW_SFNT_EXTENSIONS.some((ext) => path.toLowerCase().endsWith(ext)); -} +export { type ArchiveFormat, archiveFormatOf, requireArchiveTool }; /** Font members inside a source archive, by their in-archive path. */ function listFontMembers(archivePath: string, format: ArchiveFormat): string[] { - const out = - format === "tar.gz" - ? execFileSync("tar", ["-tzf", archivePath], { encoding: "utf8" }) - : execFileSync("unzip", ["-Z1", archivePath], { encoding: "utf8" }); - return out - .split("\n") - .map((line) => line.trim()) - .filter(Boolean) - .filter(isFontFile); -} - -// `unzip -p` matches its member argument as a glob, so members with literal glob -// metacharacters (e.g. variable-font names like `NotoSans-Italic[wdth,wght].ttf`) -// must be escaped to extract by exact name. -const escapeArchiveMember = (name: string): string => - name.replace(/[\\*?[\]]/g, "\\$&"); - -function readArchiveMember( - archivePath: string, - member: string, - format: ArchiveFormat, -): Uint8Array { - const opts = { maxBuffer: 256 * 1024 * 1024 }; - return new Uint8Array( - format === "tar.gz" - ? execFileSync("tar", ["-xzOf", archivePath, "--", member], opts) - : execFileSync( - "unzip", - ["-p", archivePath, escapeArchiveMember(member)], - opts, - ), + return listArchiveMembers(archivePath, format).filter((member) => + hasFontExtension(member, RAW_SFNT_EXTENSIONS), ); } @@ -135,10 +92,7 @@ export function collectCandidates( } const format = archiveFormatOf(source); - const archivePath = join( - cacheDir, - `${source.sourceId}.${archiveExtensions[format]}`, - ); + const archivePath = archivePathFor(cacheDir, source.sourceId, format); if (!existsSync(archivePath)) throw new Error( `candidate archive missing for ${source.sourceId}: ${archivePath}. Run \`bun run corpus:acquire\` first.`,