crawl/package.json at main · just-every/crawl · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
{
  "name": "@just-every/crawl",
  "version": "1.0.8",
  "description": "Fast, token-efficient web content extraction - fetch web pages and convert to clean Markdown",
  "main": "dist/index.js",
  "types": "dist/index.d.ts",
  "type": "module",
  "bin": {
    "web-crawl": "./dist/cli.js"
  },
  "repository": {
    "type": "git",
    "url": "git+https://github.com/just-every/crawl.git"
  },
  "homepage": "https://github.com/just-every/crawl#readme",
  "scripts": {
    "build": "tsc",
    "dev": "tsx src/cli.ts",
    "prepublishOnly": "npm run build",
    "typecheck": "tsc --noEmit"
  },
  "keywords": [
    "web",
    "crawl",
    "crawler",
    "markdown",
    "scraper",
    "readability",
    "content-extraction",
    "llm",
    "rag"
  ],
  "author": "Just Every",
  "license": "MIT",
  "dependencies": {
    "@just-every/ensemble": "^0.2.160",
    "@mozilla/readability": "^0.6.0",
    "commander": "^14.0.0",
    "jsdom": "^26.1.0",
    "p-limit": "^7.1.1",
    "robots-parser": "^3.0.1",
    "turndown": "^7.2.1",
    "turndown-plugin-gfm": "^1.0.2",
    "undici": "^7.15.0",
    "yargs": "^18.0.0"
  },
  "devDependencies": {
    "@types/jsdom": "^21.1.7",
    "@types/node": "^24.3.0",
    "@types/turndown": "^5.0.5",
    "@types/yargs": "^17.0.33",
    "tsx": "^4.20.5",
    "typescript": "^5.9.2"
  },
  "engines": {
    "node": ">=20.0.0"
  },
  "files": [
    "dist"
  ]
}