From 67b709952554566f25abbf6b4c1ec60cb7f928ec Mon Sep 17 00:00:00 2001 From: kevin Date: Fri, 31 Jul 2026 10:53:20 +0800 Subject: [PATCH] Align crawl controls and contributor docs --- CHANGELOG.md | 13 +++++ CONTRIBUTING.md | 56 ++++++++++++++++++++ README.md | 6 +-- cli/clone.go | 7 +-- clone/config.go | 6 +-- docs/content/getting-started/installation.md | 8 +++ docs/content/getting-started/introduction.md | 2 +- docs/content/guides/scoping-a-crawl.md | 5 +- docs/content/reference/cli.md | 5 +- docs/content/reference/release-notes.md | 4 ++ urlx/urlx.go | 26 ++++++++- urlx/urlx_test.go | 7 ++- 12 files changed, 125 insertions(+), 20 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 07f7992..16df479 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ All notable changes to kage are recorded here. The format follows ## [Unreleased] +### Fixed + +- `--exclude` matches path prefixes (and descendants), not arbitrary path substrings, matching the docs. + +### Changed + +- Removed the unused `--traversal` flag (it was accepted but never read by the crawl engine). +- `--max-pages` is documented as "attempt at most N page renders"; failed renders count toward the cap. + +### Added + +- `CONTRIBUTING.md` with build, test, and pull-request expectations. + ## [0.3.9] - 2026-07-08 ### Fixed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..010671d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,56 @@ +# Contributing to kage + +Thanks for helping. kage is a small, opinionated tool: patches that stay focused +and come with tests land fastest. + +## Development setup + +You need a recent Go toolchain (the `go` version in `go.mod`; Go 1.21+ will +download it automatically) and, for browser tests, Chrome or Chromium. + +```bash +git clone https://github.com/tamnd/kage +cd kage +make build # -> bin/kage +make test-short # unit tests, no Chrome +make test # full suite, including Chrome e2e when a browser is present +make vet +``` + +Point kage at a browser with `KAGE_CHROME` / `CHROME_BIN` or `--chrome` if it is +not on the default path. + +## What makes a good PR + +- **One concern per PR.** Fix install, or sanitize, or packing — not all three + unless they are inseparable. +- **Tests for the regression.** Prefer a small unit test over a full site clone. + Chrome-driven tests should skip under `-short` and when no browser is found. +- **Update `CHANGELOG.md`** under `[Unreleased]` when the change is user-visible. +- **Match the local style.** Packages are deep modules with package comments; + exported behaviour is described in prose, not only in names. Run `gofmt -s`. +- **No new dependencies** unless they remove more code than they add. Explain + why a new module belongs in kage and what local code it replaces. + +## Suggested first contributions + +Open issues and the review notes are a good map. High-impact areas that have +already bitten users: + +- Authenticated clones (cookies / session) with **origin-aware** header handling + (do not leak credentials to third-party assets or arbitrary sitemaps). +- SPA / lazy-content wait strategies for sites that never finish loading. +- Windows CI for `-short` tests. + +Please open an issue before large design changes so the approach can be agreed. + +## Reporting bugs + +Include the kage version (`kage version`), OS, the exact command, and a minimal +reproducing URL when possible. Logs from a failed page render are especially +useful. + +## License + +By contributing, you agree that your contributions are licensed under the MIT +License (see [LICENSE](LICENSE)). diff --git a/README.md b/README.md index 76195d2..fdd554c 100644 --- a/README.md +++ b/README.md @@ -107,18 +107,18 @@ kage clone example.com --subdomains --scroll kage clone paulgraham.com --refresh ``` -A clone is a polite, breadth-first crawl. It reads `robots.txt`, seeds itself from `sitemap.xml`, and stays on the seed host unless you tell it otherwise. It is also stubbornly idempotent: each page is keyed by the file it writes, so the same essay reached over http and https, with or without a trailing slash, gets fetched exactly once. Hit Ctrl-C and it saves its place on the way out; run it again and it picks up where it stopped. `--refresh` re-renders in place, `--force` wipes the host and starts clean. +A clone is a polite, concurrent crawl. It reads `robots.txt`, seeds itself from `sitemap.xml`, and stays on the seed host unless you tell it otherwise. It is also stubbornly idempotent: each page is keyed by the file it writes, so the same essay reached over http and https, with or without a trailing slash, gets fetched exactly once. Hit Ctrl-C and it saves its place on the way out; run it again and it picks up where it stopped. `--refresh` re-renders in place, `--force` wipes the host and starts clean. The flags you'll actually reach for: | Flag | Default | Meaning | |------|---------|---------| | `-o, --out` | `$HOME/data/kage` | Output root; the mirror lands in `//` | -| `-p, --max-pages` | `0` | Stop after N pages (0 = no limit) | +| `-p, --max-pages` | `0` | Attempt at most N page renders (0 = no limit); failures count | | `-d, --max-depth` | `0` | How many links deep to follow (0 = no limit) | | `--scope-prefix` | | Only crawl paths starting with this prefix | | `--subdomains` | `false` | Treat subdomains of the seed host as in scope | -| `--exclude` | | Path prefixes to skip (repeatable) | +| `--exclude` | | Path prefixes to skip (path and descendants; repeatable) | | `--scroll` | `false` | Auto-scroll each page to trigger lazy loading | | `--workers` | `4` | How many pages to render at once | | `--no-robots` | `false` | Ignore `robots.txt` (be nice) | diff --git a/cli/clone.go b/cli/clone.go index a3b402a..276b8af 100644 --- a/cli/clone.go +++ b/cli/clone.go @@ -23,7 +23,6 @@ type cloneFlags struct { browserPages int maxPages int maxDepth int - traversal string maxAssetMB int64 keepMedia bool skipExt []string @@ -69,9 +68,8 @@ func newCloneCmd() *cobra.Command { fs.IntVar(&f.workers, "workers", 4, "concurrent page render workers") fs.IntVar(&f.assetWorkers, "asset-workers", 8, "concurrent asset download workers") fs.IntVar(&f.browserPages, "browser-pages", 4, "Chrome page-pool size") - fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "stop after N pages (0 = unlimited)") + fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "attempt at most N page renders (0 = unlimited); failures count toward the cap") fs.IntVarP(&f.maxDepth, "max-depth", "d", 0, "link-follow depth cap (0 = unlimited)") - fs.StringVar(&f.traversal, "traversal", "bfs", "frontier order: bfs or dfs") fs.Int64Var(&f.maxAssetMB, "max-asset-mb", 25, "skip assets larger than N MB (left on the live web)") fs.BoolVar(&f.keepMedia, "keep-media", false, "download bulk media, installers, and PDFs instead of leaving them remote") fs.StringSliceVar(&f.skipExt, "skip-ext", nil, "extra asset extensions to leave remote, e.g. .svg (repeatable)") @@ -83,7 +81,7 @@ func newCloneCmd() *cobra.Command { fs.StringVar(&f.userAgent, "user-agent", clone.DefaultUserAgent, "User-Agent for asset and robots fetches") fs.BoolVar(&f.subdomains, "subdomains", false, "treat subdomains of the seed host as in scope") fs.StringVar(&f.scopePrefix, "scope-prefix", "", "only crawl pages whose path starts with this prefix") - fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip (repeatable)") + fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip, e.g. /archive (repeatable; matches the path and its descendants)") fs.BoolVar(&f.noRobots, "no-robots", false, "ignore robots.txt (be careful and polite)") fs.DurationVar(&f.crawlDelay, "crawl-delay", 0, "override robots.txt Crawl-delay between page starts (0 = use robots.txt)") fs.BoolVar(&f.noSitemap, "no-sitemap", false, "do not seed URLs from sitemap.xml") @@ -116,7 +114,6 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error { cfg.BrowserPages = f.browserPages cfg.MaxPages = f.maxPages cfg.MaxDepth = f.maxDepth - cfg.Traversal = f.traversal cfg.MaxAssetBytes = f.maxAssetMB << 20 cfg.AssetSameDomain = !f.allAssetHosts if f.keepMedia { diff --git a/clone/config.go b/clone/config.go index 97d7eed..d63622f 100644 --- a/clone/config.go +++ b/clone/config.go @@ -31,9 +31,8 @@ type Config struct { Workers int // page render workers AssetWorkers int // HTTP asset download workers BrowserPages int // Chrome page-pool size - MaxPages int // stop after N pages (0 = unlimited) - MaxDepth int // BFS/DFS depth cap (0 = unlimited) - Traversal string + MaxPages int // attempt at most N page renders (0 = unlimited) + MaxDepth int // link-follow depth cap (0 = unlimited) MaxAssetBytes int64 // AssetSameDomain, when set, localizes only assets whose host shares the @@ -118,7 +117,6 @@ func DefaultConfig() Config { MaxAssetBytes: 25 << 20, AssetSameDomain: true, SkipAssetExts: DefaultSkipAssetExts(), - Traversal: "bfs", Timeout: 30 * time.Second, Settle: 1500 * time.Millisecond, RenderTimeout: 30 * time.Second, diff --git a/docs/content/getting-started/installation.md b/docs/content/getting-started/installation.md index 6152047..04796e0 100644 --- a/docs/content/getting-started/installation.md +++ b/docs/content/getting-started/installation.md @@ -88,4 +88,12 @@ export KAGE_CHROME=/path/to/chromium If no browser is found, kage's launcher can download a private copy of Chromium on first use. +To block kage from crawling your own site, add a robots.txt group for its agent +token (the robots fetch matches `User-agent: kage` / `User-agent: Kage`): + +``` +User-agent: kage +Disallow: / +``` + Next: [the quick start](/getting-started/quick-start/). diff --git a/docs/content/getting-started/introduction.md b/docs/content/getting-started/introduction.md index fc6bb83..b7a579b 100644 --- a/docs/content/getting-started/introduction.md +++ b/docs/content/getting-started/introduction.md @@ -36,7 +36,7 @@ A page with no working CSS or images is not much of a clone, so kage keeps the p ## The shape of a clone -kage crawls breadth-first from a seed URL, staying within the seed's host (and optionally its subdomains). It is polite by default: it honours `robots.txt` and seeds itself from `sitemap.xml`. Output lands in `$HOME/data/kage/paulgraham.com/`, with pages as `/index.html` and assets under a reserved `_kage/` directory alongside the crawl state that powers resuming. +kage crawls concurrently from a seed URL, staying within the seed's host (and optionally its subdomains). It is polite by default: it honours `robots.txt` and seeds itself from `sitemap.xml`. Output lands in `$HOME/data/kage/paulgraham.com/`, with pages as `/index.html` and assets under a reserved `_kage/` directory alongside the crawl state that powers resuming. ## Then what? diff --git a/docs/content/guides/scoping-a-crawl.md b/docs/content/guides/scoping-a-crawl.md index c8e2667..457a105 100644 --- a/docs/content/guides/scoping-a-crawl.md +++ b/docs/content/guides/scoping-a-crawl.md @@ -32,7 +32,10 @@ kage clone example.com --scope-prefix /docs Only pages whose path starts with `/docs` are followed. Assets are still fetched from wherever the page references them, so the section renders correctly. -To skip parts of a site, exclude path prefixes (repeatable): +To skip parts of a site, exclude path prefixes (repeatable). An exclude matches +that path and everything under it (`/archive` skips `/archive` and +`/archive/2020`), but not an unrelated path that merely contains the string +(`/map/archive-index` is still crawled): ```bash kage clone example.com --exclude /archive --exclude /tags diff --git a/docs/content/reference/cli.md b/docs/content/reference/cli.md index c3b6f4c..cb2f77e 100644 --- a/docs/content/reference/cli.md +++ b/docs/content/reference/cli.md @@ -35,12 +35,11 @@ images, and fonts, and writes a browsable mirror to `//`. | Flag | Default | Meaning | |------|---------|---------| -| `-p, --max-pages` | `0` | Stop after N pages (0 = unlimited) | +| `-p, --max-pages` | `0` | Attempt at most N page renders (0 = unlimited); failures count toward the cap | | `-d, --max-depth` | `0` | Link-follow depth cap (0 = unlimited) | | `--scope-prefix` | | Only crawl pages whose path starts with this prefix | | `--subdomains` | `false` | Treat subdomains of the seed host as in scope | -| `--exclude` | | Path prefixes to skip (repeatable) | -| `--traversal` | `bfs` | Frontier order: `bfs` or `dfs` | +| `--exclude` | | Path prefixes to skip (repeatable); matches the path and its descendants, not substrings elsewhere | ### Politeness diff --git a/docs/content/reference/release-notes.md b/docs/content/reference/release-notes.md index a26b83a..eecda80 100644 --- a/docs/content/reference/release-notes.md +++ b/docs/content/reference/release-notes.md @@ -6,6 +6,10 @@ weight: 40 The authoritative, commit-level history lives in [`CHANGELOG.md`](https://github.com/tamnd/kage/blob/main/CHANGELOG.md) and on the [releases page](https://github.com/tamnd/kage/releases). This page summarises each version. +## Unreleased + +- **Crawl controls match their documentation.** `--exclude` is a real path prefix, the unused `--traversal` flag is gone, and `--max-pages` clearly counts failed render attempts. + ## v0.3.9 A fix for the antivirus warning some Windows users hit when installing kage. diff --git a/urlx/urlx.go b/urlx/urlx.go index de618c4..19def1a 100644 --- a/urlx/urlx.go +++ b/urlx/urlx.go @@ -204,7 +204,7 @@ func Key(u *url.URL) string { return u.String() } type ScopeConfig struct { IncludeSubdomains bool ScopePrefix string // only crawl paths under this prefix, e.g. "/docs/" - ExcludePaths []string // skip any path containing one of these substrings + ExcludePaths []string // skip any path that equals or is under one of these prefixes } // SameSite reports whether u belongs to the seed's site: the same host, or a @@ -257,13 +257,35 @@ func InScope(seed, u *url.URL, cfg ScopeConfig) bool { return false } for _, ex := range cfg.ExcludePaths { - if ex != "" && strings.Contains(u.Path, ex) { + if ex != "" && pathHasPrefix(u.Path, ex) { return false } } return true } +// pathHasPrefix reports whether path equals prefix or is a descendant of it. +// Both sides are treated as URL paths: a prefix of "/api" matches "/api", +// "/api/", and "/api/v1", but not "/apiv1" or "/map/api". A prefix without a +// leading slash is normalised to one so "--exclude api" behaves like "/api". +func pathHasPrefix(path, prefix string) bool { + if prefix == "" { + return false + } + if !strings.HasPrefix(prefix, "/") { + prefix = "/" + prefix + } + // Exact match, or prefix followed by '/' so "/api" does not match "/apiv1". + if path == prefix || path == strings.TrimSuffix(prefix, "/") { + return true + } + p := prefix + if !strings.HasSuffix(p, "/") { + p += "/" + } + return strings.HasPrefix(path, p) +} + // LikelyPage reports whether an target should be rendered as a page // rather than downloaded as a file. Links ending in a known binary/document // extension are treated as assets. diff --git a/urlx/urlx_test.go b/urlx/urlx_test.go index 265f0b0..eb476f2 100644 --- a/urlx/urlx_test.go +++ b/urlx/urlx_test.go @@ -105,7 +105,12 @@ func TestInScope(t *testing.T) { {"https://sub.ex.com/a", ScopeConfig{IncludeSubdomains: true}, true}, {"https://ex.com/docs/x", ScopeConfig{ScopePrefix: "/docs/"}, true}, {"https://ex.com/blog/x", ScopeConfig{ScopePrefix: "/docs/"}, false}, - {"https://ex.com/a/private/x", ScopeConfig{ExcludePaths: []string{"/private/"}}, false}, + // Exclude is a path prefix, not a substring: /private matches /private + // and /private/x, but not /a/private/x or /privatething. + {"https://ex.com/private/x", ScopeConfig{ExcludePaths: []string{"/private"}}, false}, + {"https://ex.com/private", ScopeConfig{ExcludePaths: []string{"/private"}}, false}, + {"https://ex.com/a/private/x", ScopeConfig{ExcludePaths: []string{"/private"}}, true}, + {"https://ex.com/privatething", ScopeConfig{ExcludePaths: []string{"/private"}}, true}, {"https://ex.com/a/public/x", ScopeConfig{ExcludePaths: []string{"/private/"}}, true}, } for _, c := range cases {