Skip to content

fix: consume and clean up copied Chrome profile in authenticated browser fetch (#161) #247

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
josephkehan-prog wants to merge 7 commits into
base: main
Choose a base branch
from
Open

fix: consume and clean up copied Chrome profile in authenticated browser fetch (#161) #247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1d32338
fix: consume and clean up copied Chrome profile in authenticated brow…
josephkehan-prog Jul 18, 2026
804fd9c
merge: update from main (v0.2.1)
josephkehan-prog Jul 19, 2026
3696ca4
fix(cache): parse zone-less UTC timestamps as UTC in expiry checks (#…
josephkehan-prog Jul 19, 2026
bb4ed3a
test(fetch): add page.evaluate mock to persistent-profile tests
josephkehan-prog Jul 23, 2026
90ef577
Merge branch 'main' into fix/161-chrome-profile-consume-cleanup
josephkehan-prog Jul 23, 2026
88ba74b
Merge remote-tracking branch 'origin/main' into fix/161-chrome-profil…
josephkehan-prog Jul 25, 2026
0865723
perf(fetch): copy and remove the temp Chrome profile without blocking
josephkehan-prog Jul 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions src/fetch/auth.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import { existsSync, cpSync, mkdtempSync } from 'node:fs';
import { existsSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { getConfig } from '../config.js';
import { createLogger } from '../logger.js';
import { discoverSessions, isCDPReachable } from './cdp-client.js';
import { copyProfileToTemp } from './profile-copy.js';
import type { CDPSession } from '../types.js';

export interface AuthOptions {
Expand All @@ -30,10 +30,10 @@ export async function getAuthOptions(): Promise<AuthOptions | null> {
profilePath: config.chromeProfilePath,
});
}
const tempDir = mkdtempSync(join(tmpdir(), 'wigolo-chrome-'));
cpSync(config.chromeProfilePath, tempDir, { recursive: true });
logger.debug('copied Chrome profile to temp directory', { from: config.chromeProfilePath, to: tempDir });
return { userDataDir: tempDir };
// Single-use temp copy: consumed by the browser tier's persistent-context
// launch and removed by the router (removeTempProfile) once the fetch
// settles — success, failure, or abort.
return { userDataDir: await copyProfileToTemp(config.chromeProfilePath) };
}

if (config.cdpUrl) {
Expand Down
43 changes: 41 additions & 2 deletions src/fetch/browser-pool.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,13 @@ export class ChallengeBlockedError extends Error {
export interface BrowserFetchOptions {
timeoutMs?: number;
storageStatePath?: string;
/**
* Temp Chrome-profile copy (made by getAuthOptions via profile-copy.ts).
* Consumed as a DEDICATED `launchPersistentContext` so the profile's
* cookies/logins are presented to the site. The context is closed at
* end-of-fetch; the CALLER owns the copy and removes it afterwards.
* `cdpUrl` takes precedence when both are set.
*/
userDataDir?: string;
headers?: Record<string, string>;
screenshot?: boolean;
Expand Down Expand Up @@ -411,8 +418,9 @@ export class MultiBrowserPool {
let advertisedUa: string | null = null;

// Stealth applies only to the launch path — the CDP path connects to an
// external browser that owns its own fingerprint.
const useStealth = options.stealth === true && !options.cdpUrl;
// external browser that owns its own fingerprint, and the persistent-profile
// path must present the profile's own fingerprint, not a hardened one.
const useStealth = options.stealth === true && !options.cdpUrl && !options.userDataDir;

if (options.cdpUrl) {
// CDP is always Chromium
Expand All @@ -429,6 +437,37 @@ export class MultiBrowserPool {
});
ctx = await this.acquireForType(resolvedType);
}
} else if (options.userDataDir) {
// Authenticated profile fetch (WIGOLO_CHROME_PROFILE_PATH): launch a
// DEDICATED persistent context from the temp profile copy so the
// profile's cookies/logins are actually presented to the site. A
// Chrome-format profile is Chromium-only. launchPersistentContext owns
// its browser process (no separate Browser handle), so closing the
// context in the finally tears everything down; the CALLER owns the temp
// copy and removes it after this fetch settles (see profile-copy.ts).
// Bounded by the same dedicated-path semaphore as stealth so a burst of
// authenticated fetches cannot exceed the browser cap.
resolvedType = 'chromium';
await this.acquireStealthSlot();
stealthSlotHeld = true;
dedicated = true;
log.debug('fetching with browser (persistent profile context)', { url, userDataDir: options.userDataDir });
try {
const cfgProfile = getConfig();
const proxy = playwrightProxyOption(cfgProfile.proxyUrl, cfgProfile.useProxy);
ctx = await chromium.launchPersistentContext(options.userDataDir, {
headless: true,
acceptDownloads: true,
env: sanitizedChildEnv({ stripProxy: true }),
...(proxy ? { proxy } : {}),
});
} catch (err) {
// Free the slot before rethrowing so N launch failures cannot exhaust
// the semaphore (mirrors the stealth setup error path below).
this.releaseStealthSlot();
stealthSlotHeld = false;
throw err;
}
} else if (useStealth) {
resolvedType = this.resolveType(options.browserType, url);
// Bound concurrency BEFORE launching so a burst cannot exceed the cap.
Expand Down
49 changes: 49 additions & 0 deletions src/fetch/profile-copy.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import { cp, mkdtemp, rm } from 'node:fs/promises';
import { basename, join } from 'node:path';
import { tmpdir } from 'node:os';
import { createLogger } from '../logger.js';

const logger = createLogger('fetch');

/**
* Prefix for temp Chrome-profile copies. Doubles as the deletion guard in
* `removeTempProfile` so cleanup can never touch a user-configured directory.
*/
export const TEMP_PROFILE_PREFIX = 'wigolo-chrome-';

/**
* Copy the user's Chrome profile into a fresh temp directory so the browser
* tier can open it without touching (or locking) the live profile. The copy is
* SINGLE-USE and caller-owned: whoever triggers the copy MUST remove it with
* `removeTempProfile` once the fetch settles (success, failure, or abort) —
* a surviving copy is a full-profile privacy leak in tmp.
*/
export async function copyProfileToTemp(profilePath: string): Promise<string> {
const tempDir = await mkdtemp(join(tmpdir(), TEMP_PROFILE_PREFIX));
await cp(profilePath, tempDir, { recursive: true });
logger.debug('copied Chrome profile to temp directory', { from: profilePath, to: tempDir });
return tempDir;
}
Comment on lines +21 to +26

@coderabbitai coderabbitai Bot Jul 28, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟡 Minor | ⚡ Quick win

A cp failure leaks the empty temp directory.

mkdtemp creates tempDir before cp runs. If cp throws (e.g. bad/misconfigured WIGOLO_CHROME_PROFILE_PATH), the function rejects before returning tempDir to the caller, so nothing ever calls removeTempProfile on it — the empty directory is orphaned in /tmp on every failed copy attempt.

🧹 Proposed fix: clean up tempDir on copy failure 
 export async function copyProfileToTemp(profilePath: string): Promise<string> {
   const tempDir = await mkdtemp(join(tmpdir(), TEMP_PROFILE_PREFIX));
-  await cp(profilePath, tempDir, { recursive: true });
+  try {
+    await cp(profilePath, tempDir, { recursive: true });
+  } catch (err) {
+    await rm(tempDir, { recursive: true, force: true }).catch(() => {});
+    throw err;
+  }
   logger.debug('copied Chrome profile to temp directory', { from: profilePath, to: tempDir });
   return tempDir;
 }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
export async function copyProfileToTemp(profilePath: string): Promise<string> {
const tempDir = await mkdtemp(join(tmpdir(), TEMP_PROFILE_PREFIX));
await cp(profilePath, tempDir, { recursive: true });
logger.debug('copied Chrome profile to temp directory', { from: profilePath, to: tempDir });
return tempDir;
}
export async function copyProfileToTemp(profilePath: string): Promise<string> {
const tempDir = await mkdtemp(join(tmpdir(), TEMP_PROFILE_PREFIX));
try {
await cp(profilePath, tempDir, { recursive: true });
} catch (err) {
await rm(tempDir, { recursive: true, force: true }).catch(() => {});
throw err;
}
logger.debug('copied Chrome profile to temp directory', { from: profilePath, to: tempDir });
return tempDir;
}
🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/fetch/profile-copy.ts` around lines 21 - 26, Update copyProfileToTemp so
failures from cp are caught after mkdtemp succeeds, remove the newly created
tempDir using the existing cleanup mechanism, and rethrow the original error;
preserve the current logging and successful return behavior.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "== files =="
git ls-files | rg '(^|/)profile-copy\.ts$|profile-copy|auth\.(ts|js)$|browser-pool' || true

echo "== profile-copy.ts =="
if [ -f src/fetch/profile-copy.ts ]; then
  cat -n src/fetch/profile-copy.ts
fi

echo "== related searches =="
rg -n "copyProfileToTemp|removeTempProfile|launchPersistentContext|Chrome appears to be running|Singleton" src || true

Repository: KnockOutEZ/wigolo

Length of output: 4930

🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "== browser-pool relevant =="
sed -n '430,475p' src/fetch/browser-pool.ts
sed -n '65,85p' src/fetch/browser-pool.ts

echo "== auth relevant =="
sed -n '1,50p' src/fetch/auth.ts
sed -n '925,970p' src/fetch/router.ts

echo "== profile tests =="
sed -n '1,220p' tests/unit/fetch/profile-copy.test.ts || true

echo "== deterministic node fs cp behavior probe =="
node - <<'JS'
(async () => {
  const fs = await import('node:fs');
  const fsp = await import('node:fs/promises');
  const path = await import('node:path');
  const os = await import('node:os');
  const tmp = await fsp.mkdtemp(path.join(os.tmpdir(), 'wigolo-verify-cp-'));
  const src = path.join(tmp, 'src');
  const dst = path.join(tmp, 'dst');
  await fsp.mkdir(src);
  const lock = path.join(src, 'SingletonLock');
  await fsp.writeFile(lock, 'fake lock');
  try {
    await fsp.cp(src, dst, { recursive: true });
  } catch (e) {
    console.log('cp_threw:', e && e.message);
  }
  const copies = await fsp.readdir(dst);
  console.log('copied_files:', copies.sort().join(','));
  console.log('copied_lock_content:', await fsp.readFile(path.join(dst, 'SingletonLock'), 'utf8'));
  await fsp.rm(tmp, { recursive: true, force: true });
})().then(() => process.exit(0)).catch(e => { console.error(e); process.exit(1); });
JS

Repository: KnockOutEZ/wigolo

Length of output: 9620

🌐 Web query:

Chrome ProcessSingleton SingletonLock File exists profile copy launchPersistentContext Chromium

💡 Result:

The "SingletonLock" error in Chromium-based browsers occurs because the browser uses a set of lock files (specifically SingletonLock, SingletonSocket, and SingletonCookie) within the user data directory to ensure that only one browser process manages a single profile at a time [1][2][3]. When the browser starts, it creates these files—often as symlinks pointing to a target containing the hostname and process ID—to claim ownership [4][5][2]. If these files exist when a new process attempts to launch, the browser assumes the profile is already in use and aborts to prevent data corruption [6][7][3]. This error frequently manifests in automated environments like Playwright (e.g., launchPersistentContext) or CI/CD pipelines under several common scenarios: 1. Stale Lock Files: If a browser process crashes or is terminated abruptly, it may fail to clean up the lock files, leaving them in a "stale" state that blocks future launches [1][8][9]. 2. Race Conditions: When multiple automated tests or processes attempt to launch using the same user data directory simultaneously, they race to create the lock [6][10]. Only one succeeds, causing others to fail [6]. 3. Environment Mismatches: Changes to system configuration, such as a hostname change or moving a profile directory between machines, can cause the stored lock file (which contains the old hostname/PID) to remain and block the browser from starting [11][12]. To address this: - Graceful Shutdown: Ensure that all browser contexts and processes are closed cleanly using standard API methods (e.g., browserContext.close() or browser.close()) to allow the browser to remove its own lock files [8]. - Stale File Cleanup: If a browser is known to be closed but the error persists, you can manually remove the stale lock files (SingletonLock, SingletonCookie, and SingletonSocket) from the profile directory [13][11][3]. Note that this should only be done when you are certain no other browser instance is using that profile [2][3]. - Serialization: In automated testing, ensure that launches using the same profile directory are serialized (e.g., using a lock or shared promise) to prevent concurrent access [6][10]. - Profile Isolation: For parallel testing, assign a unique user data directory to each parallel process to avoid resource contention entirely [6]. If you are using launchPersistentContext and encounter persistent issues, investigate whether the browser is truly crashing during initialization, as misleading "already in use" errors can sometimes mask other failures like sandbox configuration issues [14].

Citations:

Strip Chrome process singleton locks before launching the persistent context.

copyProfileToTemp() copies the Chrome profile byte-for-byte, including SingletonLock/SingletonCookie/SingletonSocket, and then launchPersistentContext() may open that temp directory. Those lock files can make Chromium refuse to launch with SingletonLock: File exists, the same behavior warned about for the live profile. Remove the copied singleton files before returning the temp profile path.

🔒️ Proposed fix: remove copied `Singleton*` files from the temp profile 
 export async function copyProfileToTemp(profilePath: string): Promise<string> {
   const tempDir = await mkdtemp(join(tmpdir(), TEMP_PROFILE_PREFIX));
   await cp(profilePath, tempDir, { recursive: true });
+  await Promise.all(
+    ['SingletonLock', 'SingletonCookie', 'SingletonSocket'].map((name) =>
+      rm(join(tempDir, name), { force: true }).catch(() => {}),
+    ),
+  );
   logger.debug('copied Chrome profile to temp directory', { from: profilePath, to: tempDir });
   return tempDir;
 }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
export async function copyProfileToTemp(profilePath: string): Promise<string> {
const tempDir = await mkdtemp(join(tmpdir(), TEMP_PROFILE_PREFIX));
await cp(profilePath, tempDir, { recursive: true });
logger.debug('copied Chrome profile to temp directory', { from: profilePath, to: tempDir });
return tempDir;
}
export async function copyProfileToTemp(profilePath: string): Promise<string> {
const tempDir = await mkdtemp(join(tmpdir(), TEMP_PROFILE_PREFIX));
await cp(profilePath, tempDir, { recursive: true });
await Promise.all(
['SingletonLock', 'SingletonCookie', 'SingletonSocket'].map((name) =>
rm(join(tempDir, name), { force: true }).catch(() => {}),
),
);
logger.debug('copied Chrome profile to temp directory', { from: profilePath, to: tempDir });
return tempDir;
}
🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/fetch/profile-copy.ts` around lines 21 - 26, Update copyProfileToTemp to
remove all copied Chrome singleton lock files (including SingletonLock,
SingletonCookie, and SingletonSocket) from the temporary profile before logging
and returning tempDir. Ensure the cleanup applies within the copied profile
while preserving the existing copy behavior.


/**
* Remove a temp Chrome-profile copy created by `copyProfileToTemp`. Guarded to
* wigolo-owned temp copies only (the `wigolo-chrome-` prefix) and best-effort:
* a cleanup failure is logged, never thrown, so it cannot mask the fetch's own
* outcome. No-op when no copy was made (`userDataDir` undefined).
*/
export async function removeTempProfile(userDataDir: string | undefined): Promise<void> {
if (!userDataDir) return;
if (!basename(userDataDir).startsWith(TEMP_PROFILE_PREFIX)) {
logger.warn('refusing to remove a directory that is not a wigolo temp profile copy', { userDataDir });
return;
}
try {
await rm(userDataDir, { recursive: true, force: true });
logger.debug('removed temp Chrome profile copy', { userDataDir });
} catch (err) {
logger.warn('failed to remove temp Chrome profile copy', {
userDataDir,
error: err instanceof Error ? err.message : String(err),
});
}
}
49 changes: 31 additions & 18 deletions src/fetch/router.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { getConfig, type Config } from '../config.js';
import { createLogger } from '../logger.js';
import { contentAppearsEmpty } from './content-check.js';
import { getAuthOptions } from './auth.js';
import { getAuthOptions, type AuthOptions } from './auth.js';
import { removeTempProfile } from './profile-copy.js';
import { fetchWithPlaywright, shouldEscalate } from './playwright-tier.js';
import { describeFetchError } from './error-describe.js';
import {
Expand Down Expand Up @@ -920,32 +921,44 @@ export class SmartRouter {
// Actions always force Playwright --- actions need a live browser page
if (actions && actions.length > 0) {
if (!this.browserPool) throw new Error('SmartRouter: browserPool not configured');
const authOptions = useAuth ? (await getAuthOptions() ?? {}) : {};
const authOptions: AuthOptions = useAuth ? (await getAuthOptions() ?? {}) : {};
logger.debug('routing to playwright', { url, reason: 'actions present' });
return this.browserFetch(url, {
headers,
screenshot,
actions,
...authOptions,
signal,
stealth: stealthForBrowser(config, { antiBotEscalation: false }),
});
try {
return await this.browserFetch(url, {
headers,
screenshot,
actions,
...authOptions,
signal,
stealth: stealthForBrowser(config, { antiBotEscalation: false }),
});
} finally {
// The temp Chrome-profile copy (if getAuthOptions made one) is
// single-use — remove it once the fetch settles (success, failure, or
// abort) so no full-profile copy survives in tmp.
await removeTempProfile(authOptions.userDataDir);
}
}

// Always Playwright for auth or explicit override
if (renderJs === 'always' || useAuth) {
if (!this.browserPool) throw new Error('SmartRouter: browserPool not configured');
const authOptions = useAuth ? (await getAuthOptions() ?? {}) : {};
const authOptions: AuthOptions = useAuth ? (await getAuthOptions() ?? {}) : {};
logger.debug('routing to playwright', { url, reason: useAuth ? 'auth' : 'render_js=always' });
// Explicit browser request (auth / render_js:always) — not an anti-bot
// escalation, so 'auto' leaves it unhardened; 'on' still hardens.
return this.browserFetch(url, {
headers,
screenshot,
...authOptions,
signal,
stealth: stealthForBrowser(config, { antiBotEscalation: false }),
});
try {
return await this.browserFetch(url, {
headers,
screenshot,
...authOptions,
signal,
stealth: stealthForBrowser(config, { antiBotEscalation: false }),
});
} finally {
// Single-use temp Chrome-profile copy — see the actions path above.
await removeTempProfile(authOptions.userDataDir);
}
}

// HTTP only, no fallback
Expand Down
Loading
Loading