Add website crawling and single-file URL support for refdocs add

Adds two new ingestion modes to `refdocs add`:
- `--crawl` flag to spider a docs website, extract content with
  Readability, and convert to markdown with Turndown
- Direct file URL support for llms.txt-style single files

Includes 150ms request throttle, path-prefix scoping, 200 page default
limit, and discriminated Source union (github/file/crawl) for tracking.
Also indexes .txt files alongside .md. Bumps to v0.5.0.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: Chris Arter

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@dynamik-dev/refdocs",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "type": "module",
   "description": "Local CLI tool that indexes markdown documentation and exposes fast fuzzy search with intelligent chunking",
   "main": "dist/src/index.js",
@@ -24,16 +24,22 @@
     "url": "https://github.com/dynamik-dev/refdoc-cli.git"
   },
   "dependencies": {
+    "@mozilla/readability": "^0.6.0",
+    "cheerio": "^1.2.0",
     "commander": "^13.1.0",
+    "jsdom": "^28.0.0",
     "mdast-util-from-markdown": "^2.0.2",
     "minisearch": "^7.1.1",
     "picomatch": "^4.0.2",
-    "tar-stream": "^3.1.7"
+    "tar-stream": "^3.1.7",
+    "turndown": "^7.2.2"
   },
   "devDependencies": {
+    "@types/jsdom": "^27.0.0",
     "@types/node": "^22.0.0",
     "@types/picomatch": "^3.0.1",
     "@types/tar-stream": "^3.1.4",
+    "@types/turndown": "^5.0.6",
     "tsx": "^4.19.0",
     "typescript": "^5.7.3",
     "vitest": "^3.0.5"

package.json

@@ -6,7 +6,15 @@ import { createGunzip } from "node:zlib";
 import tar from "tar-stream";
 import { parseGitHubUrl, downloadTarball } from "./github.js";
 import { saveConfig } from "./config.js";
-import type { RefdocsConfig, Source } from "./types.js";
+import {
+  isGitHubUrl,
+  isTextFileUrl,
+  deriveLocalPath,
+  deriveCrawlDir,
+  fetchSingleFile,
+  crawlSite,
+} from "./crawl.js";
+import type { RefdocsConfig, Source, GitHubSource, FileSource, CrawlSource } from "./types.js";
 
 export interface AddOptions {
   path?: string;
@@ -25,7 +33,13 @@ export interface UpdateResult {
   filesWritten: number;
 }
 
-export async function addFromUrl(
+export interface CrawlAddOptions {
+  path?: string;
+  maxPages?: number;
+  depth?: number;
+}
+
+export async function addFromGitHub(
   url: string,
   options: AddOptions,
   configDir: string,
@@ -47,7 +61,8 @@ export async function addFromUrl(
     join(configDir, localPath),
   );
 
-  const source: Source = {
+  const source: GitHubSource = {
+    type: "github",
     url,
     owner: parsed.owner,
     repo: parsed.repo,
@@ -68,6 +83,74 @@ export async function addFromUrl(
   return { filesWritten, localPath, source };
 }
 
+export async function addFromFileUrl(
+  url: string,
+  options: { path?: string },
+  configDir: string,
+  config: RefdocsConfig,
+): Promise<AddResult> {
+  const localPath = options.path ?? deriveLocalPath(url);
+  const fullPath = join(configDir, localPath);
+
+  const { content } = await fetchSingleFile(url);
+
+  mkdirSync(dirname(fullPath), { recursive: true });
+  writeFileSync(fullPath, content, "utf-8");
+
+  const source: FileSource = {
+    type: "file",
+    url,
+    localPath,
+    addedAt: new Date().toISOString(),
+  };
+
+  // Add the parent directory to paths (not the file itself)
+  const pathDir = dirname(localPath);
+  const paths = isPathCovered(config.paths, pathDir)
+    ? config.paths
+    : [...config.paths, pathDir];
+
+  const sources = upsertSource(config.sources ?? [], source);
+
+  saveConfig({ paths, sources }, configDir);
+
+  return { filesWritten: 1, localPath, source };
+}
+
+export async function addFromCrawl(
+  url: string,
+  options: CrawlAddOptions,
+  configDir: string,
+  config: RefdocsConfig,
+): Promise<AddResult> {
+  const localPath = options.path ?? deriveCrawlDir(url);
+  const outputDir = join(configDir, localPath);
+
+  const result = await crawlSite(url, outputDir, {
+    maxPages: options.maxPages,
+    depth: options.depth,
+  });
+
+  const source: CrawlSource = {
+    type: "crawl",
+    url,
+    scope: url,
+    localPath,
+    pagesCrawled: result.filesWritten,
+    addedAt: new Date().toISOString(),
+  };
+
+  const paths = isPathCovered(config.paths, localPath)
+    ? config.paths
+    : [...config.paths, localPath];
+
+  const sources = upsertSource(config.sources ?? [], source);
+
+  saveConfig({ paths, sources }, configDir);
+
+  return { filesWritten: result.filesWritten, localPath, source };
+}
+
 export async function updateSources(
   config: RefdocsConfig,
   configDir: string,
@@ -80,14 +163,48 @@ export async function updateSources(
 
   const results: UpdateResult[] = [];
   for (const source of sources) {
-    const ref = source.branch === "HEAD" ? undefined : source.branch;
-    const tarball = await downloadTarball(source.owner, source.repo, ref, token);
-    const filesWritten = await extractMarkdownFiles(
-      Buffer.from(tarball),
-      source.subpath,
-      join(configDir, source.localPath),
-    );
-    results.push({ source, filesWritten });
+    switch (source.type) {
+      case "github": {
+        const ref = source.branch === "HEAD" ? undefined : source.branch;
+        const tarball = await downloadTarball(source.owner, source.repo, ref, token);
+        const filesWritten = await extractMarkdownFiles(
+          Buffer.from(tarball),
+          source.subpath,
+          join(configDir, source.localPath),
+        );
+        results.push({ source, filesWritten });
+        break;
+      }
+      case "file": {
+        const { content } = await fetchSingleFile(source.url);
+        const fullPath = join(configDir, source.localPath);
+        mkdirSync(dirname(fullPath), { recursive: true });
+        writeFileSync(fullPath, content, "utf-8");
+        results.push({ source, filesWritten: 1 });
+        break;
+      }
+      case "crawl": {
+        const outputDir = join(configDir, source.localPath);
+        const crawlResult = await crawlSite(source.url, outputDir);
+        results.push({ source, filesWritten: crawlResult.filesWritten });
+        break;
+      }
+      default: {
+        // Handle legacy sources without a type field
+        const legacy = source as Record<string, unknown>;
+        if (typeof legacy.owner === "string" && typeof legacy.repo === "string") {
+          const ref = (legacy.branch as string) === "HEAD" ? undefined : legacy.branch as string;
+          const tarball = await downloadTarball(legacy.owner as string, legacy.repo as string, ref, token);
+          const filesWritten = await extractMarkdownFiles(
+            Buffer.from(tarball),
+            (legacy.subpath as string) ?? "",
+            join(configDir, (legacy.localPath as string) ?? ""),
+          );
+          results.push({ source, filesWritten });
+        }
+        break;
+      }
+    }
   }
 
   return results;
@@ -227,9 +344,18 @@ export function isPathCovered(existingPaths: string[], newPath: string): boolean
 }
 
 function upsertSource(sources: Source[], newSource: Source): Source[] {
-  const key = `${newSource.owner}/${newSource.repo}/${newSource.subpath}`;
-  const filtered = sources.filter(
-    (s) => `${s.owner}/${s.repo}/${s.subpath}` !== key,
-  );
+  const key = sourceKey(newSource);
+  const filtered = sources.filter((s) => sourceKey(s) !== key);
   return [...filtered, newSource];
 }
+
+function sourceKey(source: Source): string {
+  switch (source.type) {
+    case "github":
+      return `github:${source.owner}/${source.repo}/${source.subpath}`;
+    case "file":
+      return `file:${source.url}`;
+    case "crawl":
+      return `crawl:${source.url}`;
+  }
+}

src/add.ts

@@ -250,7 +250,7 @@ function extractText(node: Content): string {
 function fileTitle(filePath?: string): string {
   if (!filePath) return "Untitled";
   const name = filePath.split("/").pop() || filePath;
-  return name.replace(/\.md$/i, "");
+  return name.replace(/\.(md|txt)$/i, "");
 }
 
 function makeChunk(