diff --git a/CHANGELOG.md b/CHANGELOG.md index 07f7992..7b8f312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ All notable changes to kage are recorded here. The format follows ## [Unreleased] +### Fixed + +- `go install github.com/tamnd/kage/cmd/kage@latest` works again ([#72](https://github.com/tamnd/kage/issues/72)). + The module no longer carries a `replace` directive for `github.com/ysmood/leakless`. + Headless Chrome is driven with [chromedp](https://github.com/chromedp/chromedp), so the antivirus-flagged leakless helper is not linked at all (also keeping [#68](https://github.com/tamnd/kage/issues/68) fixed without a replace). + +### Changed + +- Page renders no longer apply [go-rod-stealth](https://github.com/go-rod/stealth)'s anti-detection evasions, which the chromedp migration does not carry over. + Chrome still launches with automation flags off, but pages now see headless Chrome's default user agent on renders (asset and robots fetches still use `--user-agent`). + Sites with aggressive bot detection may serve a clone different content than before. +- `--settle` is now a fixed wait after page load instead of a network-idle watch: chromedp has no equivalent of go-rod's request-idle helper. + A page that keeps fetching past the settle window may be snapshotted slightly earlier than before. + ## [0.3.9] - 2026-07-08 ### Fixed @@ -14,6 +28,7 @@ All notable changes to kage are recorded here. The format follows go-rod's launcher imports [leakless](https://github.com/ysmood/leakless), which base64/gzip-embeds a prebuilt helper for every platform and links the Windows one into `kage.exe`. kage already launches Chrome with leakless disabled, so the helper never ran, only added the flagged bytes. A `replace` directive now points the package at an API-compatible stub under `third_party/leakless` that carries no embedded binary, dropping about 1.28 MB from the Windows build. + *(Superseded in Unreleased by the chromedp migration, which removes leakless entirely and restores `go install`.)* ## [0.3.6] - 2026-06-19 diff --git a/README.md b/README.md index 76195d2..7cc9465 100644 --- a/README.md +++ b/README.md @@ -258,7 +258,7 @@ The repo is split by concern: cmd/kage/ thin main: pins the main thread, then hands off to cli.Execute cli/ the cobra command tree and flag wiring clone/ the crawl: frontier, render workers, asset workers, resume state -browser/ headless Chrome control and DOM snapshotting +browser/ headless Chrome control (chromedp) and DOM snapshotting sanitize/ strip scripts, handlers, and javascript: URLs from the DOM asset/ download and localise CSS, images, and fonts urlx/ the deterministic URL-to-path mapping diff --git a/browser/leakless.go b/browser/leakless.go deleted file mode 100644 index f1f6944..0000000 --- a/browser/leakless.go +++ /dev/null @@ -1,7 +0,0 @@ -package browser - -import "runtime" - -func launcherLeakless() bool { - return runtime.GOOS != "windows" -} diff --git a/browser/pool.go b/browser/pool.go index fc43566..84097ba 100644 --- a/browser/pool.go +++ b/browser/pool.go @@ -3,21 +3,27 @@ // through here: navigate, let the page settle, then serialise the final DOM — // the same markup a human would have seen — which the rest of the pipeline then // strips of scripts and localises. +// +// Chrome is launched by this package directly (os/exec + remote debugging), not +// through go-rod's launcher. That keeps github.com/ysmood/leakless — and the +// antivirus-flagged embedded helper it ships — out of the dependency graph, so +// go install and Windows package managers stay clean (issues #68, #72). package browser import ( "context" "fmt" "os" + "os/exec" "runtime" "strings" "sync" "time" - "github.com/go-rod/rod" - "github.com/go-rod/rod/lib/launcher" - "github.com/go-rod/rod/lib/proto" - "github.com/go-rod/stealth" + "github.com/chromedp/cdproto/browser" + "github.com/chromedp/cdproto/network" + cdpruntime "github.com/chromedp/cdproto/runtime" + "github.com/chromedp/chromedp" ) // Options configure a Pool. @@ -47,9 +53,10 @@ type Pool struct { opts Options sem chan struct{} - mu sync.Mutex - browser *rod.Browser - closed bool + mu sync.Mutex + allocCtx context.Context + cancel context.CancelFunc + closed bool } // New creates a Pool. Chrome is launched lazily on the first Render. @@ -92,138 +99,137 @@ func (p *Pool) Render(ctx context.Context, rawURL string) (RenderResult, error) return RenderResult{}, ctx.Err() } - b, err := p.getBrowser() - if err != nil { + if err := p.ensureBrowser(); err != nil { return RenderResult{}, err } - page, err := stealth.Page(b) - if err != nil { - return RenderResult{}, fmt.Errorf("new page: %w", err) - } - defer func() { _ = page.Close() }() - - page = page.Context(ctx).Timeout(p.opts.RenderTimeout) - - // Watch the main document's response so a navigation that turns out to be a - // non-HTML resource (a zip, a CSV, a bare image) is caught and handed back for - // the asset downloader, rather than rendered as a broken page or, with downloads - // denied, left as an aborted navigation (issue #32). The content type arrives in - // the response headers whether Chrome renders the body or aborts it as a denied - // download, so this catches both. - mainContentType := watchMainDocument(page) - - navErr := page.Navigate(rawURL) - // A denied download aborts the navigation, so inspect the captured content type - // before treating a navigation error as a failure. waitFor gives the response - // event a brief moment to be processed; for an HTML page it returns at once. + tabCtx, cancel := chromedp.NewContext(p.allocCtx) + defer cancel() + // The tab context is rooted at the pool's allocator, which outlives any + // single Render; forward the caller's cancellation so an interrupt (Ctrl-C + // during a clone) aborts an in-flight page at once instead of waiting out + // the render timeout below. + stop := context.AfterFunc(ctx, cancel) + defer stop() + + timeout := p.opts.RenderTimeout + if timeout <= 0 { + timeout = 30 * time.Second + } + tabCtx, cancelTimeout := context.WithTimeout(tabCtx, timeout) + defer cancelTimeout() + + // Enable network events and deny browser-initiated downloads before any + // navigation so a zip/CSV never lands in the user's Downloads folder and so + // the main-document content-type watcher can classify non-HTML navigations + // (issue #32). Best-effort: if a call is unsupported, the content-type + // watcher below still keeps binaries out of the mirror. + _ = chromedp.Run(tabCtx, chromedp.ActionFunc(func(ctx context.Context) error { + if err := network.Enable().Do(ctx); err != nil { + return err + } + return browser.SetDownloadBehavior(browser.SetDownloadBehaviorBehaviorDeny).Do(ctx) + })) + mainContentType := watchMainDocument(tabCtx) + + // chromedp.Navigate waits for the frame's load event. A denied download + // aborts navigation, so inspect the captured content type before treating a + // navigation error as a hard failure. + navErr := chromedp.Run(tabCtx, chromedp.Navigate(rawURL)) if ct := waitFor(ctx, mainContentType, 2*time.Second); ct != "" && !isHTML(ct) { return RenderResult{}, &ErrNotHTML{URL: rawURL, ContentType: ct} } - if navErr != nil { + if navErr != nil && !isObjRefChainError(navErr) { + // Object-reference-chain errors from Chrome are non-fatal when the + // document still loaded (issue #36). return RenderResult{}, fmt.Errorf("navigate %s: %w", rawURL, navErr) } - if err := page.WaitLoad(); err != nil { - // Chrome's DevTools Protocol may return "Object reference chain is too - // long" when a page's JavaScript builds deeply nested object graphs. - // The page has still loaded its HTML — the error is only about Chrome's - // internal object tracking, not about the document. Log the warning and - // continue rendering rather than failing the entire page (issue #36). - if !isObjRefChainError(err) { - return RenderResult{}, fmt.Errorf("wait load %s: %w", rawURL, err) - } - } - settle(page, p.opts.Settle) - if p.opts.Scroll { - autoScroll(page) - settle(page, p.opts.Settle) - } - html, err := page.HTML() - if err != nil { - return RenderResult{}, fmt.Errorf("serialise %s: %w", rawURL, err) + settle(tabCtx, p.opts.Settle) + if p.opts.Scroll { + autoScroll(tabCtx) + settle(tabCtx, p.opts.Settle) + } + + var html, finalURL, title string + if err := chromedp.Run(tabCtx, + chromedp.OuterHTML("html", &html, chromedp.ByQuery), + chromedp.Location(&finalURL), + chromedp.Title(&title), + ); err != nil { + if html == "" { + return RenderResult{}, fmt.Errorf("serialise %s: %w", rawURL, err) + } + // Partial success: the DOM serialised but a follow-up read (final URL or + // title) failed. Keep the rendered page and say so, rather than dropping + // it or failing silently. + fmt.Fprintf(os.Stderr, "kage: serialise %s: %v (keeping the rendered DOM)\n", rawURL, err) } - - res := RenderResult{HTML: html, FinalURL: rawURL} - if info, err := page.Info(); err == nil && info != nil { - res.FinalURL = info.URL - res.Title = info.Title + if finalURL == "" { + finalURL = rawURL } - return res, nil + return RenderResult{HTML: html, FinalURL: finalURL, Title: title}, nil } -// getBrowser lazily connects to or launches Chrome. -func (p *Pool) getBrowser() (*rod.Browser, error) { +// ensureBrowser lazily connects to or launches Chrome. +func (p *Pool) ensureBrowser() error { p.mu.Lock() defer p.mu.Unlock() if p.closed { - return nil, fmt.Errorf("pool is closed") - } - if p.browser != nil { - return p.browser, nil - } - - controlURL := p.opts.ControlURL - if controlURL == "" { - l := launcher.New().Leakless(launcherLeakless()). - Headless(p.opts.Headless). - Set("disable-blink-features", "AutomationControlled"). - Set("disable-gpu", "") - - // Chrome's sandbox is the main line of defense when rendering pages from - // the open web, so kage keeps it on by default (issue #10). It is dropped - // only where it genuinely cannot initialize: inside a container, or when - // running as root, where Chrome otherwise refuses to start. The decision - // is logged so it is never silent. - if off, reason := disableSandbox(); off { - l = l.Set("no-sandbox", "") - warnSandboxDisabled(reason) - } + return fmt.Errorf("pool is closed") + } + if p.allocCtx != nil { + return nil + } - // In a container, the default /dev/shm is only 64 MB, too small for - // Chrome's renderer on large pages, so steer it to a temp file instead. - // Outside a container /dev/shm is roomy and faster, so leave it alone. - // - // The "chrome_crashpad_handler: --database is required" abort seen in - // containers (issue #7) is not fixed here: the crash-reporter flags do not - // stop Chrome from spawning the handler. Its real cause is an unwritable - // HOME, which leaves the crash database path empty; the image keeps HOME - // writable instead (see the Dockerfile). - if inContainer() { - l = l.Set("disable-dev-shm-usage", "") - } + if p.opts.ControlURL != "" { + allocCtx, cancel := chromedp.NewRemoteAllocator(context.Background(), p.opts.ControlURL) + p.allocCtx = allocCtx + p.cancel = cancel + return nil + } - if bin := p.chromeBin(); bin != "" { - l = l.Bin(bin) - } - u, err := l.Launch() - if err != nil { - return nil, fmt.Errorf("launch Chrome: %w", err) - } - controlURL = u - } - - b := rod.New().ControlURL(controlURL) - if err := b.Connect(); err != nil { - return nil, fmt.Errorf("connect Chrome: %w", err) - } - - // kage never wants Chrome to write a file to disk. Every asset is fetched - // through kage's own downloader, which applies the size and media policy, so a - // Chrome-initiated download is only ever an accident: navigating an link - // that turns out to be a binary (a zip, an installer, a CSV) makes Chrome save - // it to the user's Downloads folder, a surprise side effect of a crawl - // (issue #32). Denying downloads browser-wide stops that. The navigation is - // aborted instead, and Render's non-HTML detection reroutes the URL through the - // asset downloader, where the asset policy decides its fate. This is - // best-effort: if the call is unsupported, the non-HTML detection still keeps - // the binary out of the saved mirror. - _ = proto.BrowserSetDownloadBehavior{ - Behavior: proto.BrowserSetDownloadBehaviorBehaviorDeny, - }.Call(b) - - p.browser = b - return b, nil + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.Flag("disable-blink-features", "AutomationControlled"), + chromedp.Flag("disable-gpu", true), + chromedp.Flag("enable-automation", false), + ) + if p.opts.Headless { + opts = append(opts, chromedp.Headless) + } else { + opts = append(opts, chromedp.Flag("headless", false)) + } + + // Chrome's sandbox is the main line of defense when rendering pages from + // the open web, so kage keeps it on by default (issue #10). It is dropped + // only where it genuinely cannot initialize: inside a container, or when + // running as root, where Chrome otherwise refuses to start. + if off, reason := disableSandbox(); off { + opts = append(opts, chromedp.NoSandbox) + warnSandboxDisabled(reason) + } + // In a container, the default /dev/shm is only 64 MB, too small for + // Chrome's renderer on large pages (issue #7 notes related container pain). + if inContainer() { + opts = append(opts, chromedp.Flag("disable-dev-shm-usage", true)) + } + if bin := p.chromeBin(); bin != "" { + opts = append(opts, chromedp.ExecPath(bin)) + } + + allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) + // Touch the browser once so launch failures surface here, not on first page. + browserCtx, browserCancel := chromedp.NewContext(allocCtx) + if err := chromedp.Run(browserCtx); err != nil { + browserCancel() + cancel() + return fmt.Errorf("launch Chrome: %w", err) + } + browserCancel() + + p.allocCtx = allocCtx + p.cancel = cancel + return nil } // Close shuts down the managed Chrome process. @@ -231,36 +237,39 @@ func (p *Pool) Close() error { p.mu.Lock() defer p.mu.Unlock() p.closed = true - if p.browser == nil { - return nil + if p.cancel != nil { + p.cancel() + p.cancel = nil + p.allocCtx = nil } - err := p.browser.Close() - p.browser = nil - return err + return nil } // LookChrome reports the path of a usable Chrome/Chromium binary and whether one -// was found, checking KAGE_CHROME, CHROME_BIN, rod's own lookup, and the common -// system install locations. Tests use it to skip when no browser is present. +// was found, checking KAGE_CHROME, CHROME_BIN, and the common system install +// locations. Tests use it to skip when no browser is present. func LookChrome() (string, bool) { for _, env := range []string{"KAGE_CHROME", "CHROME_BIN"} { if v := os.Getenv(env); v != "" { return v, true } } - if bin, ok := launcher.LookPath(); ok { - return bin, true - } for _, c := range systemChromeCandidates() { if _, err := os.Stat(c); err == nil { return c, true } } + // chromedp's default lookup (google-chrome, chromium, …) on PATH. + for _, name := range []string{"google-chrome", "google-chrome-stable", "chromium", "chromium-browser", "chrome"} { + if p, err := exec.LookPath(name); err == nil && p != "" { + return p, true + } + } return "", false } // chromeBin returns an explicit Chrome path from options or the environment, or -// "" to let the launcher find/download one. +// "" to let the allocator find one. func (p *Pool) chromeBin() string { if p.opts.ChromeBin != "" { return p.opts.ChromeBin @@ -328,10 +337,6 @@ func warnSandboxDisabled(reason string) { // inContainer reports whether kage is running inside a container, where Chrome // needs container-specific flags. It honors IN_DOCKER (set it in your image) // and the /.dockerenv marker that Docker writes into every container. -// -// Keeping the sandbox on by default and dropping it only here was prompted by -// Dimitrios Prasakis (issue #10); the IN_DOCKER opt-in was suggested on Hacker -// News (https://news.ycombinator.com/item?id=48534865). Thanks to both. func inContainer() bool { if envTrue("IN_DOCKER") { return true @@ -377,31 +382,23 @@ func envBool(name string) (val, ok bool) { // watchMainDocument subscribes to network responses and returns an accessor for // the main document's content type. The first Document-type response is the main // frame's navigation; later Document responses are sub-frames (iframes), whose -// type kage does not police, so only the first is kept. The accessor is safe to -// call from another goroutine. Any setup error leaves the accessor returning "", -// which the caller reads as "unknown, render normally". -func watchMainDocument(page *rod.Page) func() string { +// type kage does not police, so only the first is kept. +func watchMainDocument(ctx context.Context) func() string { var ( mu sync.Mutex ct string ) - if err := (proto.NetworkEnable{}).Call(page); err != nil { - return func() string { return "" } - } - wait := page.EachEvent(func(e *proto.NetworkResponseReceived) { - if e.Type != proto.NetworkResourceTypeDocument || e.Response == nil { + chromedp.ListenTarget(ctx, func(ev interface{}) { + e, ok := ev.(*network.EventResponseReceived) + if !ok || e.Type != network.ResourceTypeDocument || e.Response == nil { return } mu.Lock() if ct == "" { - ct = e.Response.MIMEType + ct = e.Response.MimeType } mu.Unlock() }) - // EachEvent's wait blocks until the page context ends, draining events as they - // arrive; run it for the page's lifetime. The deferred page.Close in Render - // cancels the context and unblocks it. - go wait() return func() string { mu.Lock() defer mu.Unlock() @@ -410,10 +407,7 @@ func watchMainDocument(page *rod.Page) func() string { } // waitFor polls get until it returns a non-empty value, the deadline passes, or -// the context is cancelled, then returns whatever it last saw. It exists because -// the network response is processed on another goroutine, so the value may not be -// set the instant Navigate returns; an HTML page sets it within a few -// milliseconds, while a never-arriving response simply waits out the deadline. +// the context is cancelled, then returns whatever it last saw. func waitFor(ctx context.Context, get func() string, deadline time.Duration) string { const step = 20 * time.Millisecond for waited := time.Duration(0); waited < deadline; waited += step { @@ -431,9 +425,8 @@ func waitFor(ctx context.Context, get func() string, deadline time.Duration) str // isHTML reports whether a document content type is one kage renders and saves as // a page. HTML and XHTML qualify; an empty type is treated as HTML so an unlabelled -// response still renders. Anything else (a zip, a CSV, a PDF, a bare image or -// JSON) is an asset that reached the page worker because its link carried no file -// extension to classify it by. +// response still renders. Anything else is an asset that reached the page worker +// because its link carried no file extension to classify it by. func isHTML(contentType string) bool { mt := strings.ToLower(strings.TrimSpace(contentType)) if i := strings.IndexByte(mt, ';'); i >= 0 { @@ -451,29 +444,29 @@ func isObjRefChainError(err error) bool { return err != nil && strings.Contains(err.Error(), "Object reference chain is too long") } -// settle waits for the network to go quiet for d, recovering from any rod -// panic and capping the wait so a chatty page can never hang the worker. -func settle(page *rod.Page, d time.Duration) { +// settle waits a fixed quiet window d after load so late-arriving DOM changes +// land in the snapshot. It approximates network idle with a plain sleep — +// chromedp has no built-in equivalent of rod's WaitRequestIdle — bounded by +// ctx so a cancelled or timed-out render never hangs the worker. +func settle(ctx context.Context, d time.Duration) { if d <= 0 { return } - defer func() { _ = recover() }() - done := make(chan struct{}) - go func() { - defer func() { _ = recover(); close(done) }() - wait := page.WaitRequestIdle(d, nil, nil, []proto.NetworkResourceType{}) - wait() - }() select { - case <-done: - case <-time.After(d + 5*time.Second): + case <-ctx.Done(): + case <-time.After(d): } } -// autoScroll scrolls to the bottom in steps to trigger lazy-loaded images. -func autoScroll(page *rod.Page) { - defer func() { _ = recover() }() - _, _ = page.Eval(`() => new Promise((resolve) => { +// autoScroll scrolls to the bottom in steps to trigger lazy-loaded images. The +// evaluation awaits the scroll promise — chromedp's Evaluate does not await +// promises unless asked, unlike rod's Eval — so Render only continues once the +// page has been walked to the bottom and back. +func autoScroll(ctx context.Context) { + await := func(p *cdpruntime.EvaluateParams) *cdpruntime.EvaluateParams { + return p.WithAwaitPromise(true) + } + _ = chromedp.Run(ctx, chromedp.Evaluate(`(() => new Promise((resolve) => { let total = 0; const step = 800; const timer = setInterval(() => { @@ -485,5 +478,5 @@ func autoScroll(page *rod.Page) { resolve(true); } }, 100); - })`) + }))()`, nil, await)) } diff --git a/browser/pool_test.go b/browser/pool_test.go index 4c14a78..00674f4 100644 --- a/browser/pool_test.go +++ b/browser/pool_test.go @@ -6,7 +6,6 @@ import ( "net/http" "net/http/httptest" "os" - "runtime" "strings" "testing" "time" @@ -84,14 +83,6 @@ func TestDisableSandboxContainer(t *testing.T) { } } -func TestLauncherLeaklessDisabledOnWindows(t *testing.T) { - got := launcherLeakless() - want := runtime.GOOS != "windows" - if got != want { - t.Errorf("launcherLeakless() = %v on %s; want %v", got, runtime.GOOS, want) - } -} - func TestRenderCapturesFinalDOM(t *testing.T) { if testing.Short() { t.Skip("render test drives Chrome; skipped under -short") @@ -208,3 +199,78 @@ func TestRenderRoutesNonHTML(t *testing.T) { } } } + +func TestRenderAbortsOnCallerCancel(t *testing.T) { + if testing.Short() { + t.Skip("render test drives Chrome; skipped under -short") + } + if _, ok := LookChrome(); !ok { + t.Skip("no Chrome/Chromium found; skipping render test") + } + + // A server that never answers, so the render can only end by cancellation. + // The caller's context must abort the in-flight navigation (Ctrl-C during a + // clone), not wait out the render timeout. + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + })) + defer func() { close(release); srv.Close() }() + + p := New(Options{Headless: true, Workers: 1, RenderTimeout: 30 * time.Second}) + defer func() { _ = p.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + start := time.Now() + _, err := p.Render(ctx, srv.URL) + if err == nil { + t.Error("Render against a hanging server: got nil error, want a cancellation error") + } + if el := time.Since(start); el > 10*time.Second { + t.Errorf("Render blocked %v with a 2s caller context and 30s render timeout; want a prompt abort", el) + } +} + +func TestRenderScrollCapturesLazyContent(t *testing.T) { + if testing.Short() { + t.Skip("render test drives Chrome; skipped under -short") + } + if _, ok := LookChrome(); !ok { + t.Skip("no Chrome/Chromium found; skipping render test") + } + + // A tall page that injects content only once the visitor scrolls far down. + // The snapshot must wait for the scroll to finish, or the node is missed. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(` +
+ +`)) + })) + defer srv.Close() + + p := New(Options{Headless: true, Workers: 1, Settle: 300 * time.Millisecond, RenderTimeout: 20 * time.Second, Scroll: true}) + defer func() { _ = p.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + res, err := p.Render(ctx, srv.URL) + if err != nil { + t.Fatalf("render: %v", err) + } + if !strings.Contains(res.HTML, "loaded-on-scroll") { + t.Errorf("scroll-triggered content missing from the snapshot:\n%s", res.HTML) + } +} diff --git a/docs/content/getting-started/installation.md b/docs/content/getting-started/installation.md index 6152047..1c3f5db 100644 --- a/docs/content/getting-started/installation.md +++ b/docs/content/getting-started/installation.md @@ -85,7 +85,7 @@ kage clone example.com --chrome /path/to/chromium export KAGE_CHROME=/path/to/chromium ``` -If no browser is found, kage's launcher can download a private copy of Chromium -on first use. +kage does not download Chromium for you: install Chrome or Chromium, or use the +container image above. Next: [the quick start](/getting-started/quick-start/). diff --git a/docs/content/reference/configuration.md b/docs/content/reference/configuration.md index e271a89..0d3cb51 100644 --- a/docs/content/reference/configuration.md +++ b/docs/content/reference/configuration.md @@ -16,7 +16,8 @@ locating the browser. | `CHROME_BIN` | Fallback Chrome path, read if `KAGE_CHROME` is unset. | If neither is set and no system Chrome is found in the usual install locations, -kage's launcher can download a private copy of Chromium on first use. +kage does not download Chromium for you; install a system browser or use the +container image. ## Output layout diff --git a/docs/content/reference/release-notes.md b/docs/content/reference/release-notes.md index a26b83a..775575e 100644 --- a/docs/content/reference/release-notes.md +++ b/docs/content/reference/release-notes.md @@ -6,11 +6,15 @@ weight: 40 The authoritative, commit-level history lives in [`CHANGELOG.md`](https://github.com/tamnd/kage/blob/main/CHANGELOG.md) and on the [releases page](https://github.com/tamnd/kage/releases). This page summarises each version. +## Unreleased + +- **`go install` works again.** The module no longer uses a `replace` for leakless. Chrome is driven with chromedp instead of go-rod, so the antivirus-flagged helper is not linked and Windows package installs stay clean ([#72](https://github.com/tamnd/kage/issues/72), [#68](https://github.com/tamnd/kage/issues/68)). + ## v0.3.9 A fix for the antivirus warning some Windows users hit when installing kage. -- **The Windows build no longer ships the leakless helper antivirus flags.** kage renders pages with [go-rod](https://github.com/go-rod/rod), whose launcher pulls in [leakless](https://github.com/ysmood/leakless), a small watchdog that force-kills Chrome if kage exits. leakless carries a prebuilt helper binary for every platform and links the Windows one straight into `kage.exe`. Windows Defender recognises that helper as `Trojan:Win32/Kepavll!rfn` and quarantines it, so a fresh `scoop install` failed with a virus warning on `leakless.exe` ([#68](https://github.com/tamnd/kage/issues/68)). kage already launches Chrome with leakless switched off, so the helper never ran anyway. It is now replaced with a stub that carries no embedded binary, which drops about 1.28 MB from the Windows build and clears the warning. Thanks to John Pywtorak for the report. `go install`, unaffected before, stays clean. +- **The Windows build no longer ships the leakless helper antivirus flags.** kage used to pull in [leakless](https://github.com/ysmood/leakless) via go-rod's launcher. A stub replace cleared the Defender false positive on install ([#68](https://github.com/tamnd/kage/issues/68)). That replace later broke `go install` ([#72](https://github.com/tamnd/kage/issues/72)); the Unreleased chromedp migration removes leakless entirely. ## v0.3.4 diff --git a/go.mod b/go.mod index 1c8ca32..e7f84c5 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.26.5 require ( charm.land/lipgloss/v2 v2.0.0-beta.3.0.20251106193318-19329a3e8410 github.com/charmbracelet/fang v1.0.0 - github.com/go-rod/rod v0.116.2 - github.com/go-rod/stealth v0.4.9 + github.com/chromedp/cdproto v0.0.0-20260719223732-95f6af754cfe + github.com/chromedp/chromedp v0.16.0 github.com/google/uuid v1.6.0 github.com/klauspost/compress v1.18.6 github.com/parquet-go/parquet-go v0.30.1 @@ -26,9 +26,14 @@ require ( github.com/charmbracelet/x/term v0.2.2 // indirect github.com/charmbracelet/x/termios v0.1.1 // indirect github.com/charmbracelet/x/windows v0.2.2 // indirect + github.com/chromedp/sysutil v1.1.0 // indirect github.com/clipperhouse/displaywidth v0.4.1 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.3.0 // indirect + github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.4.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect @@ -44,21 +49,8 @@ require ( github.com/spf13/pflag v1.0.9 // indirect github.com/twpayne/go-geom v1.6.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - github.com/ysmood/fetchup v0.2.3 // indirect - github.com/ysmood/goob v0.4.0 // indirect - github.com/ysmood/got v0.40.0 // indirect - github.com/ysmood/gson v0.7.3 // indirect - github.com/ysmood/leakless v0.9.0 // indirect golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.46.0 // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.38.0 // indirect google.golang.org/protobuf v1.34.2 // indirect ) - -// go-rod's launcher imports github.com/ysmood/leakless, which base64/gzip-embeds -// a prebuilt leakless.exe into the Windows build. Antivirus engines flag that -// embedded helper as malware and quarantine kage on install (issue #68). kage -// always launches Chrome with leakless disabled (browser/leakless.go), so the -// guard is dead weight; this replace swaps in an API-compatible stub that -// carries no embedded binary. -replace github.com/ysmood/leakless => ./third_party/leakless diff --git a/go.sum b/go.sum index 15a5bc9..271ee80 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,12 @@ github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8 github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= github.com/charmbracelet/x/windows v0.2.2 h1:IofanmuvaxnKHuV04sC0eBy/smG6kIKrWG2/jYn2GuM= github.com/charmbracelet/x/windows v0.2.2/go.mod h1:/8XtdKZzedat74NQFn0NGlGL4soHB0YQZrETF96h75k= +github.com/chromedp/cdproto v0.0.0-20260719223732-95f6af754cfe h1:PmhRwLZ8qLtldQCBiydwdPFJI8WVQ936ux1cpgHLRb8= +github.com/chromedp/cdproto v0.0.0-20260719223732-95f6af754cfe/go.mod h1:RwFsSODCtFExll+GhHM6R92SARHR3Z3oipaxLHj46C0= +github.com/chromedp/chromedp v0.16.0 h1:rOO4deOm4CbZgBCa8mD9g2rDyIoNs0BkgvNrlbp5ouk= +github.com/chromedp/chromedp v0.16.0/go.mod h1:rbuGKFT1vMcFcFqKfPIO1GpX/N+2s8onm2qMxZLbU5U= +github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= +github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= github.com/clipperhouse/displaywidth v0.4.1 h1:uVw9V8UDfnggg3K2U84VWY1YLQ/x2aKSCtkRyYozfoU= github.com/clipperhouse/displaywidth v0.4.1/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o= github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= @@ -37,11 +43,14 @@ github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsV github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-rod/rod v0.113.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= -github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA= -github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg= -github.com/go-rod/stealth v0.4.9 h1:X2PmQk4DUF2wzw6GOsWjW/glb8K5ebnftbEvLh7MlZ4= -github.com/go-rod/stealth v0.4.9/go.mod h1:eAzyvw8c0iAd5nJJsSWeh0fQ5z94vCIfdi1hUmYDimc= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -52,6 +61,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= @@ -66,6 +77,8 @@ github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0= github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8= github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA= github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs= github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU= @@ -93,20 +106,6 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ= -github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns= -github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= -github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18= -github.com/ysmood/gop v0.0.2/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk= -github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg= -github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk= -github.com/ysmood/got v0.34.1/go.mod h1:yddyjq/PmAf08RMLSwDjPyCvHvYed+WjHnQxpH851LM= -github.com/ysmood/got v0.40.0 h1:ZQk1B55zIvS7zflRrkGfPDrPG3d7+JOza1ZkNxcc74Q= -github.com/ysmood/got v0.40.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg= -github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY= -github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= -github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE= -github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= @@ -116,8 +115,9 @@ golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= diff --git a/third_party/leakless/go.mod b/third_party/leakless/go.mod deleted file mode 100644 index 309d7cf..0000000 --- a/third_party/leakless/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/ysmood/leakless - -go 1.26.4 diff --git a/third_party/leakless/leakless.go b/third_party/leakless/leakless.go deleted file mode 100644 index 07c3011..0000000 --- a/third_party/leakless/leakless.go +++ /dev/null @@ -1,55 +0,0 @@ -// Package leakless is kage's drop-in replacement for -// github.com/ysmood/leakless, wired in through a replace directive in the root -// go.mod. -// -// The upstream package guards a child process by extracting a small helper -// executable that force-kills the child when the parent dies. It ships that -// helper by base64/gzip-embedding a prebuilt binary for every target -// (bin_amd64_windows.go and friends), so the packed leakless.exe ends up linked -// into any program that imports the package, kage included. Antivirus engines -// flag that embedded Windows helper as malware, so a fresh install of kage got -// quarantined before it ever ran (issue #68). -// -// kage already launches Chrome with leakless disabled (see -// browser/leakless.go), so the guard is never used. This stub keeps the exact -// public surface go-rod's launcher depends on (New, Support, LockPort, and the -// Launcher type's Command/Pid/Err) while carrying no embedded binary, which -// removes the false positive entirely. Support reports no guard is available, -// so go-rod's launcher never takes the leakless path even if a caller asked -// for it. -package leakless - -import "os/exec" - -// Launcher mirrors the upstream type. The channel is left unbuffered and is -// never written to, matching the "may never receive the pid" contract go-rod -// already tolerates. -type Launcher struct { - pid chan int -} - -// New returns a Launcher. It allocates nothing beyond the pid channel. -func New() *Launcher { - return &Launcher{pid: make(chan int)} -} - -// Command builds the command without a guard wrapper. Because Support returns -// false, go-rod never calls this in practice; if some other caller did, running -// the target directly is the correct no-guard behaviour. -func (l *Launcher) Command(name string, arg ...string) *exec.Cmd { - return exec.Command(name, arg...) -} - -// Pid returns the (never-signalled) pid channel. -func (l *Launcher) Pid() chan int { return l.pid } - -// Err returns the guard error, always empty here since there is no guard. -func (l *Launcher) Err() string { return "" } - -// Support reports whether a guard binary is available. It always returns false -// so callers skip leakless entirely. -func Support() bool { return false } - -// LockPort is the cross-process mutex the upstream guard uses to serialise -// extraction. With no guard there is nothing to serialise, so it is a no-op. -func LockPort(port int) func() { return func() {} }