Align crawl controls and contributor docs

CHANGELOG.md

@@ -6,6 +6,19 @@ All notable changes to kage are recorded here. The format follows
 
 ## [Unreleased]
 
+### Fixed
+
+- `--exclude` matches path prefixes (and descendants), not arbitrary path substrings, matching the docs.
+
+### Changed
+
+- Removed the unused `--traversal` flag (it was accepted but never read by the crawl engine).
+- `--max-pages` is documented as "attempt at most N page renders"; failed renders count toward the cap.
+
+### Added
+
+- `CONTRIBUTING.md` with build, test, and pull-request expectations.
+
 ## [0.3.9] - 2026-07-08
 
 ### Fixed

CONTRIBUTING.md

@@ -0,0 +1,56 @@
+# Contributing to kage
+
+Thanks for helping. kage is a small, opinionated tool: patches that stay focused
+and come with tests land fastest.
+
+## Development setup
+
+You need a recent Go toolchain (the `go` version in `go.mod`; Go 1.21+ will
+download it automatically) and, for browser tests, Chrome or Chromium.
+
+```bash
+git clone https://github.com/tamnd/kage
+cd kage
+make build          # -> bin/kage
+make test-short     # unit tests, no Chrome
+make test           # full suite, including Chrome e2e when a browser is present
+make vet
+```
+
+Point kage at a browser with `KAGE_CHROME` / `CHROME_BIN` or `--chrome` if it is
+not on the default path.
+
+## What makes a good PR
+
+- **One concern per PR.** Fix install, or sanitize, or packing — not all three
+  unless they are inseparable.
+- **Tests for the regression.** Prefer a small unit test over a full site clone.
+  Chrome-driven tests should skip under `-short` and when no browser is found.
+- **Update `CHANGELOG.md`** under `[Unreleased]` when the change is user-visible.
+- **Match the local style.** Packages are deep modules with package comments;
+  exported behaviour is described in prose, not only in names. Run `gofmt -s`.
+- **No new dependencies** unless they remove more code than they add. Explain
+  why a new module belongs in kage and what local code it replaces.
+
+## Suggested first contributions
+
+Open issues and the review notes are a good map. High-impact areas that have
+already bitten users:
+
+- Authenticated clones (cookies / session) with **origin-aware** header handling
+  (do not leak credentials to third-party assets or arbitrary sitemaps).
+- SPA / lazy-content wait strategies for sites that never finish loading.
+- Windows CI for `-short` tests.
+
+Please open an issue before large design changes so the approach can be agreed.
+
+## Reporting bugs
+
+Include the kage version (`kage version`), OS, the exact command, and a minimal
+reproducing URL when possible. Logs from a failed page render are especially
+useful.
+
+## License
+
+By contributing, you agree that your contributions are licensed under the MIT
+License (see [LICENSE](LICENSE)).

README.md

@@ -107,18 +107,18 @@ kage clone example.com --subdomains --scroll
 kage clone paulgraham.com --refresh
 ```
 
-A clone is a polite, breadth-first crawl. It reads `robots.txt`, seeds itself from `sitemap.xml`, and stays on the seed host unless you tell it otherwise. It is also stubbornly idempotent: each page is keyed by the file it writes, so the same essay reached over http and https, with or without a trailing slash, gets fetched exactly once. Hit Ctrl-C and it saves its place on the way out; run it again and it picks up where it stopped. `--refresh` re-renders in place, `--force` wipes the host and starts clean.
+A clone is a polite, concurrent crawl. It reads `robots.txt`, seeds itself from `sitemap.xml`, and stays on the seed host unless you tell it otherwise. It is also stubbornly idempotent: each page is keyed by the file it writes, so the same essay reached over http and https, with or without a trailing slash, gets fetched exactly once. Hit Ctrl-C and it saves its place on the way out; run it again and it picks up where it stopped. `--refresh` re-renders in place, `--force` wipes the host and starts clean.
 
 The flags you'll actually reach for:
 
 | Flag | Default | Meaning |
 |------|---------|---------|
 | `-o, --out` | `$HOME/data/kage` | Output root; the mirror lands in `<out>/<host>/` |
-| `-p, --max-pages` | `0` | Stop after N pages (0 = no limit) |
+| `-p, --max-pages` | `0` | Attempt at most N page renders (0 = no limit); failures count |
 | `-d, --max-depth` | `0` | How many links deep to follow (0 = no limit) |
 | `--scope-prefix` | | Only crawl paths starting with this prefix |
 | `--subdomains` | `false` | Treat subdomains of the seed host as in scope |
-| `--exclude` | | Path prefixes to skip (repeatable) |
+| `--exclude` | | Path prefixes to skip (path and descendants; repeatable) |
 | `--scroll` | `false` | Auto-scroll each page to trigger lazy loading |
 | `--workers` | `4` | How many pages to render at once |
 | `--no-robots` | `false` | Ignore `robots.txt` (be nice) |

cli/clone.go

@@ -23,7 +23,6 @@ type cloneFlags struct {
 	browserPages   int
 	maxPages       int
 	maxDepth       int
-	traversal      string
 	maxAssetMB     int64
 	keepMedia      bool
 	skipExt        []string
@@ -69,9 +68,8 @@ func newCloneCmd() *cobra.Command {
 	fs.IntVar(&f.workers, "workers", 4, "concurrent page render workers")
 	fs.IntVar(&f.assetWorkers, "asset-workers", 8, "concurrent asset download workers")
 	fs.IntVar(&f.browserPages, "browser-pages", 4, "Chrome page-pool size")
-	fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "stop after N pages (0 = unlimited)")
+	fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "attempt at most N page renders (0 = unlimited); failures count toward the cap")
 	fs.IntVarP(&f.maxDepth, "max-depth", "d", 0, "link-follow depth cap (0 = unlimited)")
-	fs.StringVar(&f.traversal, "traversal", "bfs", "frontier order: bfs or dfs")
 	fs.Int64Var(&f.maxAssetMB, "max-asset-mb", 25, "skip assets larger than N MB (left on the live web)")
 	fs.BoolVar(&f.keepMedia, "keep-media", false, "download bulk media, installers, and PDFs instead of leaving them remote")
 	fs.StringSliceVar(&f.skipExt, "skip-ext", nil, "extra asset extensions to leave remote, e.g. .svg (repeatable)")
@@ -83,7 +81,7 @@ func newCloneCmd() *cobra.Command {
 	fs.StringVar(&f.userAgent, "user-agent", clone.DefaultUserAgent, "User-Agent for asset and robots fetches")
 	fs.BoolVar(&f.subdomains, "subdomains", false, "treat subdomains of the seed host as in scope")
 	fs.StringVar(&f.scopePrefix, "scope-prefix", "", "only crawl pages whose path starts with this prefix")
-	fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip (repeatable)")
+	fs.StringSliceVar(&f.exclude, "exclude", nil, "path prefixes to skip, e.g. /archive (repeatable; matches the path and its descendants)")
 	fs.BoolVar(&f.noRobots, "no-robots", false, "ignore robots.txt (be careful and polite)")
 	fs.DurationVar(&f.crawlDelay, "crawl-delay", 0, "override robots.txt Crawl-delay between page starts (0 = use robots.txt)")
 	fs.BoolVar(&f.noSitemap, "no-sitemap", false, "do not seed URLs from sitemap.xml")
@@ -116,7 +114,6 @@ func runClone(ctx context.Context, arg string, f *cloneFlags) error {
 	cfg.BrowserPages = f.browserPages
 	cfg.MaxPages = f.maxPages
 	cfg.MaxDepth = f.maxDepth
-	cfg.Traversal = f.traversal
 	cfg.MaxAssetBytes = f.maxAssetMB << 20
 	cfg.AssetSameDomain = !f.allAssetHosts
 	if f.keepMedia {

clone/config.go

@@ -31,9 +31,8 @@ type Config struct {
 	Workers       int // page render workers
 	AssetWorkers  int // HTTP asset download workers
 	BrowserPages  int // Chrome page-pool size
-	MaxPages      int // stop after N pages (0 = unlimited)
-	MaxDepth      int // BFS/DFS depth cap (0 = unlimited)
-	Traversal     string
+	MaxPages      int // attempt at most N page renders (0 = unlimited)
+	MaxDepth      int // link-follow depth cap (0 = unlimited)
 	MaxAssetBytes int64
 
 	// AssetSameDomain, when set, localizes only assets whose host shares the
@@ -118,7 +117,6 @@ func DefaultConfig() Config {
 		MaxAssetBytes:   25 << 20,
 		AssetSameDomain: true,
 		SkipAssetExts:   DefaultSkipAssetExts(),
-		Traversal:       "bfs",
 		Timeout:         30 * time.Second,
 		Settle:          1500 * time.Millisecond,
 		RenderTimeout:   30 * time.Second,

docs/content/getting-started/installation.md

@@ -88,4 +88,12 @@ export KAGE_CHROME=/path/to/chromium
 If no browser is found, kage's launcher can download a private copy of Chromium
 on first use.
 
+To block kage from crawling your own site, add a robots.txt group for its agent
+token (the robots fetch matches `User-agent: kage` / `User-agent: Kage`):
+
+```
+User-agent: kage
+Disallow: /
+```
+
 Next: [the quick start](/getting-started/quick-start/).

docs/content/getting-started/introduction.md

@@ -36,7 +36,7 @@ A page with no working CSS or images is not much of a clone, so kage keeps the p
 
 ## The shape of a clone
 
-kage crawls breadth-first from a seed URL, staying within the seed's host (and optionally its subdomains). It is polite by default: it honours `robots.txt` and seeds itself from `sitemap.xml`. Output lands in `$HOME/data/kage/paulgraham.com/`, with pages as `<path>/index.html` and assets under a reserved `_kage/` directory alongside the crawl state that powers resuming.
+kage crawls concurrently from a seed URL, staying within the seed's host (and optionally its subdomains). It is polite by default: it honours `robots.txt` and seeds itself from `sitemap.xml`. Output lands in `$HOME/data/kage/paulgraham.com/`, with pages as `<path>/index.html` and assets under a reserved `_kage/` directory alongside the crawl state that powers resuming.
 
 ## Then what?
 

docs/content/guides/scoping-a-crawl.md

@@ -32,7 +32,10 @@ kage clone example.com --scope-prefix /docs
 Only pages whose path starts with `/docs` are followed. Assets are still fetched
 from wherever the page references them, so the section renders correctly.
 
-To skip parts of a site, exclude path prefixes (repeatable):
+To skip parts of a site, exclude path prefixes (repeatable). An exclude matches
+that path and everything under it (`/archive` skips `/archive` and
+`/archive/2020`), but not an unrelated path that merely contains the string
+(`/map/archive-index` is still crawled):
 
 ```bash
 kage clone example.com --exclude /archive --exclude /tags

docs/content/reference/cli.md

@@ -35,12 +35,11 @@ images, and fonts, and writes a browsable mirror to `<out>/<host>/`.
 
 | Flag | Default | Meaning |
 |------|---------|---------|
-| `-p, --max-pages` | `0` | Stop after N pages (0 = unlimited) |
+| `-p, --max-pages` | `0` | Attempt at most N page renders (0 = unlimited); failures count toward the cap |
 | `-d, --max-depth` | `0` | Link-follow depth cap (0 = unlimited) |
 | `--scope-prefix` | | Only crawl pages whose path starts with this prefix |
 | `--subdomains` | `false` | Treat subdomains of the seed host as in scope |
-| `--exclude` | | Path prefixes to skip (repeatable) |
-| `--traversal` | `bfs` | Frontier order: `bfs` or `dfs` |
+| `--exclude` | | Path prefixes to skip (repeatable); matches the path and its descendants, not substrings elsewhere |
 
 ### Politeness
 

docs/content/reference/release-notes.md

@@ -6,6 +6,10 @@ weight: 40
 
 The authoritative, commit-level history lives in [`CHANGELOG.md`](https://github.com/tamnd/kage/blob/main/CHANGELOG.md) and on the [releases page](https://github.com/tamnd/kage/releases). This page summarises each version.
 
+## Unreleased
+
+- **Crawl controls match their documentation.** `--exclude` is a real path prefix, the unused `--traversal` flag is gone, and `--max-pages` clearly counts failed render attempts.
+
 ## v0.3.9
 
 A fix for the antivirus warning some Windows users hit when installing kage.

urlx/urlx.go

@@ -204,7 +204,7 @@ func Key(u *url.URL) string { return u.String() }
 type ScopeConfig struct {
 	IncludeSubdomains bool
 	ScopePrefix       string   // only crawl paths under this prefix, e.g. "/docs/"
-	ExcludePaths      []string // skip any path containing one of these substrings
+	ExcludePaths      []string // skip any path that equals or is under one of these prefixes
 }
 
 // SameSite reports whether u belongs to the seed's site: the same host, or a
@@ -257,13 +257,35 @@ func InScope(seed, u *url.URL, cfg ScopeConfig) bool {
 		return false
 	}
 	for _, ex := range cfg.ExcludePaths {
-		if ex != "" && strings.Contains(u.Path, ex) {
+		if ex != "" && pathHasPrefix(u.Path, ex) {
 			return false
 		}
 	}
 	return true
 }
 
+// pathHasPrefix reports whether path equals prefix or is a descendant of it.
+// Both sides are treated as URL paths: a prefix of "/api" matches "/api",
+// "/api/", and "/api/v1", but not "/apiv1" or "/map/api". A prefix without a
+// leading slash is normalised to one so "--exclude api" behaves like "/api".
+func pathHasPrefix(path, prefix string) bool {
+	if prefix == "" {
+		return false
+	}
+	if !strings.HasPrefix(prefix, "/") {
+		prefix = "/" + prefix
+	}
+	// Exact match, or prefix followed by '/' so "/api" does not match "/apiv1".
+	if path == prefix || path == strings.TrimSuffix(prefix, "/") {
+		return true
+	}
+	p := prefix
+	if !strings.HasSuffix(p, "/") {
+		p += "/"
+	}
+	return strings.HasPrefix(path, p)
+}
+
 // LikelyPage reports whether an <a href> target should be rendered as a page
 // rather than downloaded as a file. Links ending in a known binary/document
 // extension are treated as assets.

urlx/urlx_test.go

@@ -105,7 +105,12 @@ func TestInScope(t *testing.T) {
 		{"https://sub.ex.com/a", ScopeConfig{IncludeSubdomains: true}, true},
 		{"https://ex.com/docs/x", ScopeConfig{ScopePrefix: "/docs/"}, true},
 		{"https://ex.com/blog/x", ScopeConfig{ScopePrefix: "/docs/"}, false},
-		{"https://ex.com/a/private/x", ScopeConfig{ExcludePaths: []string{"/private/"}}, false},
+		// Exclude is a path prefix, not a substring: /private matches /private
+		// and /private/x, but not /a/private/x or /privatething.
+		{"https://ex.com/private/x", ScopeConfig{ExcludePaths: []string{"/private"}}, false},
+		{"https://ex.com/private", ScopeConfig{ExcludePaths: []string{"/private"}}, false},
+		{"https://ex.com/a/private/x", ScopeConfig{ExcludePaths: []string{"/private"}}, true},
+		{"https://ex.com/privatething", ScopeConfig{ExcludePaths: []string{"/private"}}, true},
 		{"https://ex.com/a/public/x", ScopeConfig{ExcludePaths: []string{"/private/"}}, true},
 	}
 	for _, c := range cases {