From 55117ce36d015e14ca05258389d507bf5eca2f51 Mon Sep 17 00:00:00 2001 From: kevin Date: Fri, 31 Jul 2026 10:51:31 +0800 Subject: [PATCH 1/2] Replace go-rod with chromedp for installability --- CHANGELOG.md | 7 + README.md | 2 +- browser/leakless.go | 7 - browser/pool.go | 320 +++++++++---------- browser/pool_test.go | 9 - docs/content/getting-started/installation.md | 4 +- docs/content/reference/configuration.md | 3 +- docs/content/reference/release-notes.md | 6 +- go.mod | 24 +- go.sum | 42 +-- third_party/leakless/go.mod | 3 - third_party/leakless/leakless.go | 55 ---- 12 files changed, 196 insertions(+), 286 deletions(-) delete mode 100644 browser/leakless.go delete mode 100644 third_party/leakless/go.mod delete mode 100644 third_party/leakless/leakless.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 07f7992..42739ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ All notable changes to kage are recorded here. The format follows ## [Unreleased] +### Fixed + +- `go install github.com/tamnd/kage/cmd/kage@latest` works again ([#72](https://github.com/tamnd/kage/issues/72)). + The module no longer carries a `replace` directive for `github.com/ysmood/leakless`. + Headless Chrome is driven with [chromedp](https://github.com/chromedp/chromedp), so the antivirus-flagged leakless helper is not linked at all (also keeping [#68](https://github.com/tamnd/kage/issues/68) fixed without a replace). + ## [0.3.9] - 2026-07-08 ### Fixed @@ -14,6 +20,7 @@ All notable changes to kage are recorded here. The format follows go-rod's launcher imports [leakless](https://github.com/ysmood/leakless), which base64/gzip-embeds a prebuilt helper for every platform and links the Windows one into `kage.exe`. kage already launches Chrome with leakless disabled, so the helper never ran, only added the flagged bytes. A `replace` directive now points the package at an API-compatible stub under `third_party/leakless` that carries no embedded binary, dropping about 1.28 MB from the Windows build. + *(Superseded in Unreleased by the chromedp migration, which removes leakless entirely and restores `go install`.)* ## [0.3.6] - 2026-06-19 diff --git a/README.md b/README.md index 76195d2..7cc9465 100644 --- a/README.md +++ b/README.md @@ -258,7 +258,7 @@ The repo is split by concern: cmd/kage/ thin main: pins the main thread, then hands off to cli.Execute cli/ the cobra command tree and flag wiring clone/ the crawl: frontier, render workers, asset workers, resume state -browser/ headless Chrome control and DOM snapshotting +browser/ headless Chrome control (chromedp) and DOM snapshotting sanitize/ strip scripts, handlers, and javascript: URLs from the DOM asset/ download and localise CSS, images, and fonts urlx/ the deterministic URL-to-path mapping diff --git a/browser/leakless.go b/browser/leakless.go deleted file mode 100644 index f1f6944..0000000 --- a/browser/leakless.go +++ /dev/null @@ -1,7 +0,0 @@ -package browser - -import "runtime" - -func launcherLeakless() bool { - return runtime.GOOS != "windows" -} diff --git a/browser/pool.go b/browser/pool.go index fc43566..16de455 100644 --- a/browser/pool.go +++ b/browser/pool.go @@ -3,21 +3,26 @@ // through here: navigate, let the page settle, then serialise the final DOM — // the same markup a human would have seen — which the rest of the pipeline then // strips of scripts and localises. +// +// Chrome is launched by this package directly (os/exec + remote debugging), not +// through go-rod's launcher. That keeps github.com/ysmood/leakless — and the +// antivirus-flagged embedded helper it ships — out of the dependency graph, so +// go install and Windows package managers stay clean (issues #68, #72). package browser import ( "context" "fmt" "os" + "os/exec" "runtime" "strings" "sync" "time" - "github.com/go-rod/rod" - "github.com/go-rod/rod/lib/launcher" - "github.com/go-rod/rod/lib/proto" - "github.com/go-rod/stealth" + "github.com/chromedp/cdproto/browser" + "github.com/chromedp/cdproto/network" + "github.com/chromedp/chromedp" ) // Options configure a Pool. @@ -47,9 +52,10 @@ type Pool struct { opts Options sem chan struct{} - mu sync.Mutex - browser *rod.Browser - closed bool + mu sync.Mutex + allocCtx context.Context + cancel context.CancelFunc + closed bool } // New creates a Pool. Chrome is launched lazily on the first Render. @@ -92,138 +98,131 @@ func (p *Pool) Render(ctx context.Context, rawURL string) (RenderResult, error) return RenderResult{}, ctx.Err() } - b, err := p.getBrowser() - if err != nil { + if err := p.ensureBrowser(); err != nil { return RenderResult{}, err } - page, err := stealth.Page(b) - if err != nil { - return RenderResult{}, fmt.Errorf("new page: %w", err) - } - defer func() { _ = page.Close() }() + tabCtx, cancel := chromedp.NewContext(p.allocCtx) + defer cancel() - page = page.Context(ctx).Timeout(p.opts.RenderTimeout) + timeout := p.opts.RenderTimeout + if timeout <= 0 { + timeout = 30 * time.Second + } + tabCtx, cancelTimeout := context.WithTimeout(tabCtx, timeout) + defer cancelTimeout() - // Watch the main document's response so a navigation that turns out to be a - // non-HTML resource (a zip, a CSV, a bare image) is caught and handed back for - // the asset downloader, rather than rendered as a broken page or, with downloads - // denied, left as an aborted navigation (issue #32). The content type arrives in - // the response headers whether Chrome renders the body or aborts it as a denied - // download, so this catches both. - mainContentType := watchMainDocument(page) + // Enable network events and deny browser-initiated downloads before any + // navigation so a zip/CSV never lands in the user's Downloads folder and so + // the main-document content-type watcher can classify non-HTML navigations + // (issue #32). + if err := chromedp.Run(tabCtx, chromedp.ActionFunc(func(ctx context.Context) error { + if err := network.Enable().Do(ctx); err != nil { + return err + } + return browser.SetDownloadBehavior(browser.SetDownloadBehaviorBehaviorDeny).Do(ctx) + })); err != nil { + // Best-effort: non-HTML detection still keeps binaries out of the mirror. + _ = err + } + mainContentType := watchMainDocument(tabCtx) - navErr := page.Navigate(rawURL) - // A denied download aborts the navigation, so inspect the captured content type - // before treating a navigation error as a failure. waitFor gives the response - // event a brief moment to be processed; for an HTML page it returns at once. + // chromedp.Navigate waits for the frame's load event. A denied download + // aborts navigation, so inspect the captured content type before treating a + // navigation error as a hard failure. + navErr := chromedp.Run(tabCtx, chromedp.Navigate(rawURL)) if ct := waitFor(ctx, mainContentType, 2*time.Second); ct != "" && !isHTML(ct) { return RenderResult{}, &ErrNotHTML{URL: rawURL, ContentType: ct} } - if navErr != nil { + if navErr != nil && !isObjRefChainError(navErr) { + // Object-reference-chain errors from Chrome are non-fatal when the + // document still loaded (issue #36). return RenderResult{}, fmt.Errorf("navigate %s: %w", rawURL, navErr) } - if err := page.WaitLoad(); err != nil { - // Chrome's DevTools Protocol may return "Object reference chain is too - // long" when a page's JavaScript builds deeply nested object graphs. - // The page has still loaded its HTML — the error is only about Chrome's - // internal object tracking, not about the document. Log the warning and - // continue rendering rather than failing the entire page (issue #36). - if !isObjRefChainError(err) { - return RenderResult{}, fmt.Errorf("wait load %s: %w", rawURL, err) - } - } - settle(page, p.opts.Settle) - if p.opts.Scroll { - autoScroll(page) - settle(page, p.opts.Settle) - } - html, err := page.HTML() - if err != nil { - return RenderResult{}, fmt.Errorf("serialise %s: %w", rawURL, err) + settle(tabCtx, p.opts.Settle) + if p.opts.Scroll { + autoScroll(tabCtx) + settle(tabCtx, p.opts.Settle) + } + + var html, finalURL, title string + if err := chromedp.Run(tabCtx, + chromedp.OuterHTML("html", &html, chromedp.ByQuery), + chromedp.Location(&finalURL), + chromedp.Title(&title), + ); err != nil { + if isObjRefChainError(err) && html != "" { + // Partial success: keep what we have. + } else if html == "" { + return RenderResult{}, fmt.Errorf("serialise %s: %w", rawURL, err) + } } - - res := RenderResult{HTML: html, FinalURL: rawURL} - if info, err := page.Info(); err == nil && info != nil { - res.FinalURL = info.URL - res.Title = info.Title + if finalURL == "" { + finalURL = rawURL } - return res, nil + return RenderResult{HTML: html, FinalURL: finalURL, Title: title}, nil } -// getBrowser lazily connects to or launches Chrome. -func (p *Pool) getBrowser() (*rod.Browser, error) { +// ensureBrowser lazily connects to or launches Chrome. +func (p *Pool) ensureBrowser() error { p.mu.Lock() defer p.mu.Unlock() if p.closed { - return nil, fmt.Errorf("pool is closed") - } - if p.browser != nil { - return p.browser, nil - } - - controlURL := p.opts.ControlURL - if controlURL == "" { - l := launcher.New().Leakless(launcherLeakless()). - Headless(p.opts.Headless). - Set("disable-blink-features", "AutomationControlled"). - Set("disable-gpu", "") - - // Chrome's sandbox is the main line of defense when rendering pages from - // the open web, so kage keeps it on by default (issue #10). It is dropped - // only where it genuinely cannot initialize: inside a container, or when - // running as root, where Chrome otherwise refuses to start. The decision - // is logged so it is never silent. - if off, reason := disableSandbox(); off { - l = l.Set("no-sandbox", "") - warnSandboxDisabled(reason) - } + return fmt.Errorf("pool is closed") + } + if p.allocCtx != nil { + return nil + } - // In a container, the default /dev/shm is only 64 MB, too small for - // Chrome's renderer on large pages, so steer it to a temp file instead. - // Outside a container /dev/shm is roomy and faster, so leave it alone. - // - // The "chrome_crashpad_handler: --database is required" abort seen in - // containers (issue #7) is not fixed here: the crash-reporter flags do not - // stop Chrome from spawning the handler. Its real cause is an unwritable - // HOME, which leaves the crash database path empty; the image keeps HOME - // writable instead (see the Dockerfile). - if inContainer() { - l = l.Set("disable-dev-shm-usage", "") - } + if p.opts.ControlURL != "" { + allocCtx, cancel := chromedp.NewRemoteAllocator(context.Background(), p.opts.ControlURL) + p.allocCtx = allocCtx + p.cancel = cancel + return nil + } - if bin := p.chromeBin(); bin != "" { - l = l.Bin(bin) - } - u, err := l.Launch() - if err != nil { - return nil, fmt.Errorf("launch Chrome: %w", err) - } - controlURL = u - } - - b := rod.New().ControlURL(controlURL) - if err := b.Connect(); err != nil { - return nil, fmt.Errorf("connect Chrome: %w", err) - } - - // kage never wants Chrome to write a file to disk. Every asset is fetched - // through kage's own downloader, which applies the size and media policy, so a - // Chrome-initiated download is only ever an accident: navigating an link - // that turns out to be a binary (a zip, an installer, a CSV) makes Chrome save - // it to the user's Downloads folder, a surprise side effect of a crawl - // (issue #32). Denying downloads browser-wide stops that. The navigation is - // aborted instead, and Render's non-HTML detection reroutes the URL through the - // asset downloader, where the asset policy decides its fate. This is - // best-effort: if the call is unsupported, the non-HTML detection still keeps - // the binary out of the saved mirror. - _ = proto.BrowserSetDownloadBehavior{ - Behavior: proto.BrowserSetDownloadBehaviorBehaviorDeny, - }.Call(b) - - p.browser = b - return b, nil + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.Flag("disable-blink-features", "AutomationControlled"), + chromedp.Flag("disable-gpu", true), + chromedp.Flag("enable-automation", false), + ) + if p.opts.Headless { + opts = append(opts, chromedp.Headless) + } else { + opts = append(opts, chromedp.Flag("headless", false)) + } + + // Chrome's sandbox is the main line of defense when rendering pages from + // the open web, so kage keeps it on by default (issue #10). It is dropped + // only where it genuinely cannot initialize: inside a container, or when + // running as root, where Chrome otherwise refuses to start. + if off, reason := disableSandbox(); off { + opts = append(opts, chromedp.NoSandbox) + warnSandboxDisabled(reason) + } + // In a container, the default /dev/shm is only 64 MB, too small for + // Chrome's renderer on large pages (issue #7 notes related container pain). + if inContainer() { + opts = append(opts, chromedp.Flag("disable-dev-shm-usage", true)) + } + if bin := p.chromeBin(); bin != "" { + opts = append(opts, chromedp.ExecPath(bin)) + } + + allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) + // Touch the browser once so launch failures surface here, not on first page. + browserCtx, browserCancel := chromedp.NewContext(allocCtx) + if err := chromedp.Run(browserCtx); err != nil { + browserCancel() + cancel() + return fmt.Errorf("launch Chrome: %w", err) + } + browserCancel() + + p.allocCtx = allocCtx + p.cancel = cancel + return nil } // Close shuts down the managed Chrome process. @@ -231,36 +230,39 @@ func (p *Pool) Close() error { p.mu.Lock() defer p.mu.Unlock() p.closed = true - if p.browser == nil { - return nil + if p.cancel != nil { + p.cancel() + p.cancel = nil + p.allocCtx = nil } - err := p.browser.Close() - p.browser = nil - return err + return nil } // LookChrome reports the path of a usable Chrome/Chromium binary and whether one -// was found, checking KAGE_CHROME, CHROME_BIN, rod's own lookup, and the common -// system install locations. Tests use it to skip when no browser is present. +// was found, checking KAGE_CHROME, CHROME_BIN, and the common system install +// locations. Tests use it to skip when no browser is present. func LookChrome() (string, bool) { for _, env := range []string{"KAGE_CHROME", "CHROME_BIN"} { if v := os.Getenv(env); v != "" { return v, true } } - if bin, ok := launcher.LookPath(); ok { - return bin, true - } for _, c := range systemChromeCandidates() { if _, err := os.Stat(c); err == nil { return c, true } } + // chromedp's default lookup (google-chrome, chromium, …) on PATH. + for _, name := range []string{"google-chrome", "google-chrome-stable", "chromium", "chromium-browser", "chrome"} { + if p, err := exec.LookPath(name); err == nil && p != "" { + return p, true + } + } return "", false } // chromeBin returns an explicit Chrome path from options or the environment, or -// "" to let the launcher find/download one. +// "" to let the allocator find one. func (p *Pool) chromeBin() string { if p.opts.ChromeBin != "" { return p.opts.ChromeBin @@ -328,10 +330,6 @@ func warnSandboxDisabled(reason string) { // inContainer reports whether kage is running inside a container, where Chrome // needs container-specific flags. It honors IN_DOCKER (set it in your image) // and the /.dockerenv marker that Docker writes into every container. -// -// Keeping the sandbox on by default and dropping it only here was prompted by -// Dimitrios Prasakis (issue #10); the IN_DOCKER opt-in was suggested on Hacker -// News (https://news.ycombinator.com/item?id=48534865). Thanks to both. func inContainer() bool { if envTrue("IN_DOCKER") { return true @@ -377,31 +375,23 @@ func envBool(name string) (val, ok bool) { // watchMainDocument subscribes to network responses and returns an accessor for // the main document's content type. The first Document-type response is the main // frame's navigation; later Document responses are sub-frames (iframes), whose -// type kage does not police, so only the first is kept. The accessor is safe to -// call from another goroutine. Any setup error leaves the accessor returning "", -// which the caller reads as "unknown, render normally". -func watchMainDocument(page *rod.Page) func() string { +// type kage does not police, so only the first is kept. +func watchMainDocument(ctx context.Context) func() string { var ( mu sync.Mutex ct string ) - if err := (proto.NetworkEnable{}).Call(page); err != nil { - return func() string { return "" } - } - wait := page.EachEvent(func(e *proto.NetworkResponseReceived) { - if e.Type != proto.NetworkResourceTypeDocument || e.Response == nil { + chromedp.ListenTarget(ctx, func(ev interface{}) { + e, ok := ev.(*network.EventResponseReceived) + if !ok || e.Type != network.ResourceTypeDocument || e.Response == nil { return } mu.Lock() if ct == "" { - ct = e.Response.MIMEType + ct = e.Response.MimeType } mu.Unlock() }) - // EachEvent's wait blocks until the page context ends, draining events as they - // arrive; run it for the page's lifetime. The deferred page.Close in Render - // cancels the context and unblocks it. - go wait() return func() string { mu.Lock() defer mu.Unlock() @@ -410,10 +400,7 @@ func watchMainDocument(page *rod.Page) func() string { } // waitFor polls get until it returns a non-empty value, the deadline passes, or -// the context is cancelled, then returns whatever it last saw. It exists because -// the network response is processed on another goroutine, so the value may not be -// set the instant Navigate returns; an HTML page sets it within a few -// milliseconds, while a never-arriving response simply waits out the deadline. +// the context is cancelled, then returns whatever it last saw. func waitFor(ctx context.Context, get func() string, deadline time.Duration) string { const step = 20 * time.Millisecond for waited := time.Duration(0); waited < deadline; waited += step { @@ -431,9 +418,8 @@ func waitFor(ctx context.Context, get func() string, deadline time.Duration) str // isHTML reports whether a document content type is one kage renders and saves as // a page. HTML and XHTML qualify; an empty type is treated as HTML so an unlabelled -// response still renders. Anything else (a zip, a CSV, a PDF, a bare image or -// JSON) is an asset that reached the page worker because its link carried no file -// extension to classify it by. +// response still renders. Anything else is an asset that reached the page worker +// because its link carried no file extension to classify it by. func isHTML(contentType string) bool { mt := strings.ToLower(strings.TrimSpace(contentType)) if i := strings.IndexByte(mt, ';'); i >= 0 { @@ -451,29 +437,23 @@ func isObjRefChainError(err error) bool { return err != nil && strings.Contains(err.Error(), "Object reference chain is too long") } -// settle waits for the network to go quiet for d, recovering from any rod -// panic and capping the wait so a chatty page can never hang the worker. -func settle(page *rod.Page, d time.Duration) { +// settle waits for the network to go quiet for d, recovering from any panic and +// capping the wait so a chatty page can never hang the worker. +func settle(ctx context.Context, d time.Duration) { if d <= 0 { return } - defer func() { _ = recover() }() - done := make(chan struct{}) - go func() { - defer func() { _ = recover(); close(done) }() - wait := page.WaitRequestIdle(d, nil, nil, []proto.NetworkResourceType{}) - wait() - }() + // chromedp has no built-in network-idle helper matching rod's; approximate + // with a quiet sleep after load. Good enough for the settle window. select { - case <-done: - case <-time.After(d + 5*time.Second): + case <-ctx.Done(): + case <-time.After(d): } } // autoScroll scrolls to the bottom in steps to trigger lazy-loaded images. -func autoScroll(page *rod.Page) { - defer func() { _ = recover() }() - _, _ = page.Eval(`() => new Promise((resolve) => { +func autoScroll(ctx context.Context) { + _ = chromedp.Run(ctx, chromedp.Evaluate(`(() => new Promise((resolve) => { let total = 0; const step = 800; const timer = setInterval(() => { @@ -485,5 +465,5 @@ func autoScroll(page *rod.Page) { resolve(true); } }, 100); - })`) + }))()`, nil)) } diff --git a/browser/pool_test.go b/browser/pool_test.go index 4c14a78..d7822cf 100644 --- a/browser/pool_test.go +++ b/browser/pool_test.go @@ -6,7 +6,6 @@ import ( "net/http" "net/http/httptest" "os" - "runtime" "strings" "testing" "time" @@ -84,14 +83,6 @@ func TestDisableSandboxContainer(t *testing.T) { } } -func TestLauncherLeaklessDisabledOnWindows(t *testing.T) { - got := launcherLeakless() - want := runtime.GOOS != "windows" - if got != want { - t.Errorf("launcherLeakless() = %v on %s; want %v", got, runtime.GOOS, want) - } -} - func TestRenderCapturesFinalDOM(t *testing.T) { if testing.Short() { t.Skip("render test drives Chrome; skipped under -short") diff --git a/docs/content/getting-started/installation.md b/docs/content/getting-started/installation.md index 6152047..1c3f5db 100644 --- a/docs/content/getting-started/installation.md +++ b/docs/content/getting-started/installation.md @@ -85,7 +85,7 @@ kage clone example.com --chrome /path/to/chromium export KAGE_CHROME=/path/to/chromium ``` -If no browser is found, kage's launcher can download a private copy of Chromium -on first use. +kage does not download Chromium for you: install Chrome or Chromium, or use the +container image above. Next: [the quick start](/getting-started/quick-start/). diff --git a/docs/content/reference/configuration.md b/docs/content/reference/configuration.md index e271a89..0d3cb51 100644 --- a/docs/content/reference/configuration.md +++ b/docs/content/reference/configuration.md @@ -16,7 +16,8 @@ locating the browser. | `CHROME_BIN` | Fallback Chrome path, read if `KAGE_CHROME` is unset. | If neither is set and no system Chrome is found in the usual install locations, -kage's launcher can download a private copy of Chromium on first use. +kage does not download Chromium for you; install a system browser or use the +container image. ## Output layout diff --git a/docs/content/reference/release-notes.md b/docs/content/reference/release-notes.md index a26b83a..775575e 100644 --- a/docs/content/reference/release-notes.md +++ b/docs/content/reference/release-notes.md @@ -6,11 +6,15 @@ weight: 40 The authoritative, commit-level history lives in [`CHANGELOG.md`](https://github.com/tamnd/kage/blob/main/CHANGELOG.md) and on the [releases page](https://github.com/tamnd/kage/releases). This page summarises each version. +## Unreleased + +- **`go install` works again.** The module no longer uses a `replace` for leakless. Chrome is driven with chromedp instead of go-rod, so the antivirus-flagged helper is not linked and Windows package installs stay clean ([#72](https://github.com/tamnd/kage/issues/72), [#68](https://github.com/tamnd/kage/issues/68)). + ## v0.3.9 A fix for the antivirus warning some Windows users hit when installing kage. -- **The Windows build no longer ships the leakless helper antivirus flags.** kage renders pages with [go-rod](https://github.com/go-rod/rod), whose launcher pulls in [leakless](https://github.com/ysmood/leakless), a small watchdog that force-kills Chrome if kage exits. leakless carries a prebuilt helper binary for every platform and links the Windows one straight into `kage.exe`. Windows Defender recognises that helper as `Trojan:Win32/Kepavll!rfn` and quarantines it, so a fresh `scoop install` failed with a virus warning on `leakless.exe` ([#68](https://github.com/tamnd/kage/issues/68)). kage already launches Chrome with leakless switched off, so the helper never ran anyway. It is now replaced with a stub that carries no embedded binary, which drops about 1.28 MB from the Windows build and clears the warning. Thanks to John Pywtorak for the report. `go install`, unaffected before, stays clean. +- **The Windows build no longer ships the leakless helper antivirus flags.** kage used to pull in [leakless](https://github.com/ysmood/leakless) via go-rod's launcher. A stub replace cleared the Defender false positive on install ([#68](https://github.com/tamnd/kage/issues/68)). That replace later broke `go install` ([#72](https://github.com/tamnd/kage/issues/72)); the Unreleased chromedp migration removes leakless entirely. ## v0.3.4 diff --git a/go.mod b/go.mod index 1c8ca32..e7f84c5 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.26.5 require ( charm.land/lipgloss/v2 v2.0.0-beta.3.0.20251106193318-19329a3e8410 github.com/charmbracelet/fang v1.0.0 - github.com/go-rod/rod v0.116.2 - github.com/go-rod/stealth v0.4.9 + github.com/chromedp/cdproto v0.0.0-20260719223732-95f6af754cfe + github.com/chromedp/chromedp v0.16.0 github.com/google/uuid v1.6.0 github.com/klauspost/compress v1.18.6 github.com/parquet-go/parquet-go v0.30.1 @@ -26,9 +26,14 @@ require ( github.com/charmbracelet/x/term v0.2.2 // indirect github.com/charmbracelet/x/termios v0.1.1 // indirect github.com/charmbracelet/x/windows v0.2.2 // indirect + github.com/chromedp/sysutil v1.1.0 // indirect github.com/clipperhouse/displaywidth v0.4.1 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.3.0 // indirect + github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.4.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect @@ -44,21 +49,8 @@ require ( github.com/spf13/pflag v1.0.9 // indirect github.com/twpayne/go-geom v1.6.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - github.com/ysmood/fetchup v0.2.3 // indirect - github.com/ysmood/goob v0.4.0 // indirect - github.com/ysmood/got v0.40.0 // indirect - github.com/ysmood/gson v0.7.3 // indirect - github.com/ysmood/leakless v0.9.0 // indirect golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.46.0 // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.38.0 // indirect google.golang.org/protobuf v1.34.2 // indirect ) - -// go-rod's launcher imports github.com/ysmood/leakless, which base64/gzip-embeds -// a prebuilt leakless.exe into the Windows build. Antivirus engines flag that -// embedded helper as malware and quarantine kage on install (issue #68). kage -// always launches Chrome with leakless disabled (browser/leakless.go), so the -// guard is dead weight; this replace swaps in an API-compatible stub that -// carries no embedded binary. -replace github.com/ysmood/leakless => ./third_party/leakless diff --git a/go.sum b/go.sum index 15a5bc9..271ee80 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,12 @@ github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8 github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= github.com/charmbracelet/x/windows v0.2.2 h1:IofanmuvaxnKHuV04sC0eBy/smG6kIKrWG2/jYn2GuM= github.com/charmbracelet/x/windows v0.2.2/go.mod h1:/8XtdKZzedat74NQFn0NGlGL4soHB0YQZrETF96h75k= +github.com/chromedp/cdproto v0.0.0-20260719223732-95f6af754cfe h1:PmhRwLZ8qLtldQCBiydwdPFJI8WVQ936ux1cpgHLRb8= +github.com/chromedp/cdproto v0.0.0-20260719223732-95f6af754cfe/go.mod h1:RwFsSODCtFExll+GhHM6R92SARHR3Z3oipaxLHj46C0= +github.com/chromedp/chromedp v0.16.0 h1:rOO4deOm4CbZgBCa8mD9g2rDyIoNs0BkgvNrlbp5ouk= +github.com/chromedp/chromedp v0.16.0/go.mod h1:rbuGKFT1vMcFcFqKfPIO1GpX/N+2s8onm2qMxZLbU5U= +github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= +github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= github.com/clipperhouse/displaywidth v0.4.1 h1:uVw9V8UDfnggg3K2U84VWY1YLQ/x2aKSCtkRyYozfoU= github.com/clipperhouse/displaywidth v0.4.1/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o= github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= @@ -37,11 +43,14 @@ github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsV github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-rod/rod v0.113.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= -github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA= -github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg= -github.com/go-rod/stealth v0.4.9 h1:X2PmQk4DUF2wzw6GOsWjW/glb8K5ebnftbEvLh7MlZ4= -github.com/go-rod/stealth v0.4.9/go.mod h1:eAzyvw8c0iAd5nJJsSWeh0fQ5z94vCIfdi1hUmYDimc= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -52,6 +61,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= @@ -66,6 +77,8 @@ github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0= github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8= github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA= github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs= github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU= @@ -93,20 +106,6 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ= -github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns= -github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= -github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18= -github.com/ysmood/gop v0.0.2/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk= -github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg= -github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk= -github.com/ysmood/got v0.34.1/go.mod h1:yddyjq/PmAf08RMLSwDjPyCvHvYed+WjHnQxpH851LM= -github.com/ysmood/got v0.40.0 h1:ZQk1B55zIvS7zflRrkGfPDrPG3d7+JOza1ZkNxcc74Q= -github.com/ysmood/got v0.40.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg= -github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY= -github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= -github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE= -github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= @@ -116,8 +115,9 @@ golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= diff --git a/third_party/leakless/go.mod b/third_party/leakless/go.mod deleted file mode 100644 index 309d7cf..0000000 --- a/third_party/leakless/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/ysmood/leakless - -go 1.26.4 diff --git a/third_party/leakless/leakless.go b/third_party/leakless/leakless.go deleted file mode 100644 index 07c3011..0000000 --- a/third_party/leakless/leakless.go +++ /dev/null @@ -1,55 +0,0 @@ -// Package leakless is kage's drop-in replacement for -// github.com/ysmood/leakless, wired in through a replace directive in the root -// go.mod. -// -// The upstream package guards a child process by extracting a small helper -// executable that force-kills the child when the parent dies. It ships that -// helper by base64/gzip-embedding a prebuilt binary for every target -// (bin_amd64_windows.go and friends), so the packed leakless.exe ends up linked -// into any program that imports the package, kage included. Antivirus engines -// flag that embedded Windows helper as malware, so a fresh install of kage got -// quarantined before it ever ran (issue #68). -// -// kage already launches Chrome with leakless disabled (see -// browser/leakless.go), so the guard is never used. This stub keeps the exact -// public surface go-rod's launcher depends on (New, Support, LockPort, and the -// Launcher type's Command/Pid/Err) while carrying no embedded binary, which -// removes the false positive entirely. Support reports no guard is available, -// so go-rod's launcher never takes the leakless path even if a caller asked -// for it. -package leakless - -import "os/exec" - -// Launcher mirrors the upstream type. The channel is left unbuffered and is -// never written to, matching the "may never receive the pid" contract go-rod -// already tolerates. -type Launcher struct { - pid chan int -} - -// New returns a Launcher. It allocates nothing beyond the pid channel. -func New() *Launcher { - return &Launcher{pid: make(chan int)} -} - -// Command builds the command without a guard wrapper. Because Support returns -// false, go-rod never calls this in practice; if some other caller did, running -// the target directly is the correct no-guard behaviour. -func (l *Launcher) Command(name string, arg ...string) *exec.Cmd { - return exec.Command(name, arg...) -} - -// Pid returns the (never-signalled) pid channel. -func (l *Launcher) Pid() chan int { return l.pid } - -// Err returns the guard error, always empty here since there is no guard. -func (l *Launcher) Err() string { return "" } - -// Support reports whether a guard binary is available. It always returns false -// so callers skip leakless entirely. -func Support() bool { return false } - -// LockPort is the cross-process mutex the upstream guard uses to serialise -// extraction. With no guard there is nothing to serialise, so it is a no-op. -func LockPort(port int) func() { return func() {} } From 41321c7902d4fae732d8b7a095777efdc8101d86 Mon Sep 17 00:00:00 2001 From: kevin Date: Fri, 31 Jul 2026 13:13:28 +0800 Subject: [PATCH 2/2] Abort renders on caller cancel and await the auto-scroll promise The chromedp migration rooted tab contexts at the pool allocator, so Ctrl-C no longer aborted an in-flight render (it blocked until the render timeout), and chromedp's Evaluate does not await promises, so --scroll snapshotted before the scroll and its lazy loads finished. Forward the caller's context into the tab with context.AfterFunc and pass WithAwaitPromise to the scroll evaluation. Also log partial serialise failures instead of swallowing them, drop a dead error branch, correct the settle doc comment, and note the migration's two intentional behavior changes (no stealth evasions, fixed settle wait) in the changelog. --- CHANGELOG.md | 8 +++++ browser/pool.go | 43 ++++++++++++++++--------- browser/pool_test.go | 75 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42739ba..7b8f312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,14 @@ All notable changes to kage are recorded here. The format follows The module no longer carries a `replace` directive for `github.com/ysmood/leakless`. Headless Chrome is driven with [chromedp](https://github.com/chromedp/chromedp), so the antivirus-flagged leakless helper is not linked at all (also keeping [#68](https://github.com/tamnd/kage/issues/68) fixed without a replace). +### Changed + +- Page renders no longer apply [go-rod-stealth](https://github.com/go-rod/stealth)'s anti-detection evasions, which the chromedp migration does not carry over. + Chrome still launches with automation flags off, but pages now see headless Chrome's default user agent on renders (asset and robots fetches still use `--user-agent`). + Sites with aggressive bot detection may serve a clone different content than before. +- `--settle` is now a fixed wait after page load instead of a network-idle watch: chromedp has no equivalent of go-rod's request-idle helper. + A page that keeps fetching past the settle window may be snapshotted slightly earlier than before. + ## [0.3.9] - 2026-07-08 ### Fixed diff --git a/browser/pool.go b/browser/pool.go index 16de455..84097ba 100644 --- a/browser/pool.go +++ b/browser/pool.go @@ -22,6 +22,7 @@ import ( "github.com/chromedp/cdproto/browser" "github.com/chromedp/cdproto/network" + cdpruntime "github.com/chromedp/cdproto/runtime" "github.com/chromedp/chromedp" ) @@ -104,6 +105,12 @@ func (p *Pool) Render(ctx context.Context, rawURL string) (RenderResult, error) tabCtx, cancel := chromedp.NewContext(p.allocCtx) defer cancel() + // The tab context is rooted at the pool's allocator, which outlives any + // single Render; forward the caller's cancellation so an interrupt (Ctrl-C + // during a clone) aborts an in-flight page at once instead of waiting out + // the render timeout below. + stop := context.AfterFunc(ctx, cancel) + defer stop() timeout := p.opts.RenderTimeout if timeout <= 0 { @@ -115,16 +122,14 @@ func (p *Pool) Render(ctx context.Context, rawURL string) (RenderResult, error) // Enable network events and deny browser-initiated downloads before any // navigation so a zip/CSV never lands in the user's Downloads folder and so // the main-document content-type watcher can classify non-HTML navigations - // (issue #32). - if err := chromedp.Run(tabCtx, chromedp.ActionFunc(func(ctx context.Context) error { + // (issue #32). Best-effort: if a call is unsupported, the content-type + // watcher below still keeps binaries out of the mirror. + _ = chromedp.Run(tabCtx, chromedp.ActionFunc(func(ctx context.Context) error { if err := network.Enable().Do(ctx); err != nil { return err } return browser.SetDownloadBehavior(browser.SetDownloadBehaviorBehaviorDeny).Do(ctx) - })); err != nil { - // Best-effort: non-HTML detection still keeps binaries out of the mirror. - _ = err - } + })) mainContentType := watchMainDocument(tabCtx) // chromedp.Navigate waits for the frame's load event. A denied download @@ -152,11 +157,13 @@ func (p *Pool) Render(ctx context.Context, rawURL string) (RenderResult, error) chromedp.Location(&finalURL), chromedp.Title(&title), ); err != nil { - if isObjRefChainError(err) && html != "" { - // Partial success: keep what we have. - } else if html == "" { + if html == "" { return RenderResult{}, fmt.Errorf("serialise %s: %w", rawURL, err) } + // Partial success: the DOM serialised but a follow-up read (final URL or + // title) failed. Keep the rendered page and say so, rather than dropping + // it or failing silently. + fmt.Fprintf(os.Stderr, "kage: serialise %s: %v (keeping the rendered DOM)\n", rawURL, err) } if finalURL == "" { finalURL = rawURL @@ -437,22 +444,28 @@ func isObjRefChainError(err error) bool { return err != nil && strings.Contains(err.Error(), "Object reference chain is too long") } -// settle waits for the network to go quiet for d, recovering from any panic and -// capping the wait so a chatty page can never hang the worker. +// settle waits a fixed quiet window d after load so late-arriving DOM changes +// land in the snapshot. It approximates network idle with a plain sleep — +// chromedp has no built-in equivalent of rod's WaitRequestIdle — bounded by +// ctx so a cancelled or timed-out render never hangs the worker. func settle(ctx context.Context, d time.Duration) { if d <= 0 { return } - // chromedp has no built-in network-idle helper matching rod's; approximate - // with a quiet sleep after load. Good enough for the settle window. select { case <-ctx.Done(): case <-time.After(d): } } -// autoScroll scrolls to the bottom in steps to trigger lazy-loaded images. +// autoScroll scrolls to the bottom in steps to trigger lazy-loaded images. The +// evaluation awaits the scroll promise — chromedp's Evaluate does not await +// promises unless asked, unlike rod's Eval — so Render only continues once the +// page has been walked to the bottom and back. func autoScroll(ctx context.Context) { + await := func(p *cdpruntime.EvaluateParams) *cdpruntime.EvaluateParams { + return p.WithAwaitPromise(true) + } _ = chromedp.Run(ctx, chromedp.Evaluate(`(() => new Promise((resolve) => { let total = 0; const step = 800; @@ -465,5 +478,5 @@ func autoScroll(ctx context.Context) { resolve(true); } }, 100); - }))()`, nil)) + }))()`, nil, await)) } diff --git a/browser/pool_test.go b/browser/pool_test.go index d7822cf..00674f4 100644 --- a/browser/pool_test.go +++ b/browser/pool_test.go @@ -199,3 +199,78 @@ func TestRenderRoutesNonHTML(t *testing.T) { } } } + +func TestRenderAbortsOnCallerCancel(t *testing.T) { + if testing.Short() { + t.Skip("render test drives Chrome; skipped under -short") + } + if _, ok := LookChrome(); !ok { + t.Skip("no Chrome/Chromium found; skipping render test") + } + + // A server that never answers, so the render can only end by cancellation. + // The caller's context must abort the in-flight navigation (Ctrl-C during a + // clone), not wait out the render timeout. + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + })) + defer func() { close(release); srv.Close() }() + + p := New(Options{Headless: true, Workers: 1, RenderTimeout: 30 * time.Second}) + defer func() { _ = p.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + start := time.Now() + _, err := p.Render(ctx, srv.URL) + if err == nil { + t.Error("Render against a hanging server: got nil error, want a cancellation error") + } + if el := time.Since(start); el > 10*time.Second { + t.Errorf("Render blocked %v with a 2s caller context and 30s render timeout; want a prompt abort", el) + } +} + +func TestRenderScrollCapturesLazyContent(t *testing.T) { + if testing.Short() { + t.Skip("render test drives Chrome; skipped under -short") + } + if _, ok := LookChrome(); !ok { + t.Skip("no Chrome/Chromium found; skipping render test") + } + + // A tall page that injects content only once the visitor scrolls far down. + // The snapshot must wait for the scroll to finish, or the node is missed. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(` +
+ +`)) + })) + defer srv.Close() + + p := New(Options{Headless: true, Workers: 1, Settle: 300 * time.Millisecond, RenderTimeout: 20 * time.Second, Scroll: true}) + defer func() { _ = p.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + res, err := p.Render(ctx, srv.URL) + if err != nil { + t.Fatalf("render: %v", err) + } + if !strings.Contains(res.HTML, "loaded-on-scroll") { + t.Errorf("scroll-triggered content missing from the snapshot:\n%s", res.HTML) + } +}