From 0304040a7bd4cc0b8e589a8c66bb8be104f9eff7 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Sat, 8 Aug 2026 08:53:02 +0530 Subject: [PATCH 1/3] fix(adapters,output): repair site drift and markdown/console output bugs Core: - escape `|` in markdown table cells so values containing pipes no longer break the rendered table layout - stop `consoleMessages('error')` from also returning warnings - keep a row-less `` from throwing inside turndown-plugin-gfm and taking the whole article download with it Twitter/X: - read profile counts and bio from the current response containers after X moved them out of `legacy`; stats no longer report 0 - render images from atomic article blocks, handling the current array-shaped `entityMap` - add resumable `--all` archives for `likes` and `bookmarks`, with JSONL output and a resume file so interrupted runs continue - add `twitter collection`: a bounded user-timeline fetch with relationship facts and a completion receipt Facebook: - recover the display name from the current profile-avatar header and scope friends/followers links to the profile region - keep scrolling the feed until the extractor has enough valid rows rather than stopping on raw article-marker counts - preserve query identity in search result URLs (permalink/story/photo/watch) while stripping per-render tracking nonces, and drop l./lm. redirect shims TikTok: - use the current /api/explore/item_list/ endpoint, keeping the recommend feed as a fallback Amazon: - honor the marketplace named by the input URL instead of rewriting every request and emitted URL to amazon.com; bare ASINs still default to the US store, and look-alike hosts are rejected against an explicit domain list - name the landed URL when a product or review page exposes no content Co-Authored-By: Claude Opus 5 --- plugins/amazon/README.md | 4 + plugins/amazon/discussion.js | 9 +- plugins/amazon/product.js | 3 +- plugins/amazon/shared.js | 61 ++++- plugins/amazon/test/discussion.test.js | 62 +++++ plugins/amazon/test/shared.test.js | 29 ++ plugins/facebook/feed.js | 34 ++- plugins/facebook/profile.js | 47 +++- plugins/facebook/search.js | 29 +- plugins/facebook/test/feed.test.js | 18 ++ plugins/facebook/test/profile.test.js | 81 ++++++ plugins/facebook/test/search.test.js | 70 +++++ plugins/tiktok/explore.js | 60 +++-- plugins/twitter/README.md | 1 + plugins/twitter/article.js | 38 ++- plugins/twitter/bookmarks.js | 297 ++++++++++++++++++--- plugins/twitter/collection.js | 339 ++++++++++++++++++++++++ plugins/twitter/likes.js | 272 +++++++++++++++++-- plugins/twitter/profile.js | 22 +- plugins/twitter/test/article.test.js | 43 +++ plugins/twitter/test/bookmarks.test.js | 277 ++++++++++++++++++- plugins/twitter/test/collection.test.js | 276 +++++++++++++++++++ plugins/twitter/test/likes.test.js | 255 ++++++++++++++++++ plugins/twitter/test/profile.test.js | 93 +++++++ plugins/twitter/user-timeline.js | 217 +++++++++++++++ src/browser/cdp.ts | 2 +- src/download/article-download.test.ts | 22 ++ src/download/article-download.ts | 9 + src/output.test.ts | 4 +- src/output.ts | 2 +- 30 files changed, 2557 insertions(+), 119 deletions(-) create mode 100644 plugins/facebook/test/profile.test.js create mode 100644 plugins/twitter/collection.js create mode 100644 plugins/twitter/test/collection.test.js create mode 100644 plugins/twitter/user-timeline.js diff --git a/plugins/amazon/README.md b/plugins/amazon/README.md index cb2f028c..eddebe68 100644 --- a/plugins/amazon/README.md +++ b/plugins/amazon/README.md @@ -21,3 +21,7 @@ webcmd plugin install github:agentrhq/webcmd/amazon | `webcmd amazon product` | Amazon product page facts for candidate validation | | `webcmd amazon search` | Amazon search results for product discovery and coarse filtering | | `webcmd amazon whoami` | Show the current logged-in amazon account | + +## Notes + +- A product or review URL from a sibling marketplace (`amazon.co.uk`, `amazon.de`, `amazon.com.au`, …) is read on that marketplace, and the emitted URLs stay on it. A bare ASIN still defaults to `amazon.com`. diff --git a/plugins/amazon/discussion.js b/plugins/amazon/discussion.js index 45a81f95..de2b00f9 100644 --- a/plugins/amazon/discussion.js +++ b/plugins/amazon/discussion.js @@ -1,6 +1,6 @@ import { AuthRequiredError, CommandExecutionError } from '@agentrhq/webcmd/errors'; import { cli, Strategy } from '@agentrhq/webcmd/registry'; -import { buildProductUrl, buildDiscussionUrl, buildProvenance, cleanText, extractAsin, normalizeProductUrl, parseRatingValue, parseReviewCount, trimRatingPrefix, uniqueNonEmpty, assertUsableState, gotoAndReadState, } from './shared.js'; +import { DOMAIN, amazonHostFromInput, buildProductUrl, buildDiscussionUrl, buildProvenance, cleanText, extractAsin, normalizeProductUrl, parseRatingValue, parseReviewCount, trimRatingPrefix, uniqueNonEmpty, assertUsableState, gotoAndReadState, } from './shared.js'; function normalizeDiscussionPayload(payload) { const sourceUrl = cleanText(payload.href) || buildDiscussionUrl(payload.href ?? ''); const asin = extractAsin(payload.href ?? '') ?? null; @@ -9,7 +9,7 @@ function normalizeDiscussionPayload(payload) { const provenance = buildProvenance(sourceUrl); return { asin, - product_url: asin ? normalizeProductUrl(asin) : null, + product_url: asin ? normalizeProductUrl(sourceUrl) : null, discussion_url: sourceUrl, ...provenance, average_rating_text: averageRatingText, @@ -71,7 +71,7 @@ async function readDiscussionPayload(page, input, limit) { const productState = await gotoAndReadState(page, productUrl, 2500, 'discussion'); assertUsableState(productState, 'discussion'); if (isSignInState(reviewState) && isSignInState(productState)) { - throw new AuthRequiredError('amazon.com', 'Amazon review discussion requires an active signed-in Amazon session in the shared Chrome profile.'); + throw new AuthRequiredError(amazonHostFromInput(input) ?? DOMAIN, 'Amazon review discussion requires an active signed-in Amazon session in the shared Chrome profile.'); } const productPayload = await readCurrentDiscussionPayload(page, limit); if (hasDiscussionSummary(productPayload)) { @@ -111,7 +111,8 @@ cli({ const payload = await readDiscussionPayload(page, input, limit); const normalized = normalizeDiscussionPayload(payload); if (!normalized.average_rating_text && !normalized.total_review_count_text) { - throw new CommandExecutionError('amazon discussion page did not expose review summary', 'The review page may have changed or hit a robot check. Open the review page in Chrome and retry.'); + const landedUrl = cleanText(payload.href) || buildDiscussionUrl(input); + throw new CommandExecutionError(`amazon discussion page did not expose review summary (landed on ${landedUrl})`, 'The review page may have changed or hit a robot check. Open the review page in Chrome and retry.'); } return [normalized]; }, diff --git a/plugins/amazon/product.js b/plugins/amazon/product.js index 845a7a44..58313452 100644 --- a/plugins/amazon/product.js +++ b/plugins/amazon/product.js @@ -83,7 +83,8 @@ cli({ const input = String(kwargs.input ?? ''); const payload = await readProductPayload(page, input); if (!cleanText(payload.product_title)) { - throw new CommandExecutionError('amazon product page did not expose product content', 'The product page may have changed or hit a robot check. Open the product page in Chrome and retry.'); + const landedUrl = cleanText(payload.href) || buildProductUrl(input); + throw new CommandExecutionError(`amazon product page did not expose product content (landed on ${landedUrl})`, 'The product page may have changed or hit a robot check. Open the product page in Chrome and retry.'); } return [normalizeProductPayload(payload)]; }, diff --git a/plugins/amazon/shared.js b/plugins/amazon/shared.js index 518ea99d..d5b81880 100644 --- a/plugins/amazon/shared.js +++ b/plugins/amazon/shared.js @@ -20,6 +20,42 @@ export const PRIMARY_PRICE_SELECTORS = [ '#priceblock_dealprice', '#tp_price_block_total_price_ww', ]; +// Keep this explicit because these hosts are navigation targets in the user's +// signed-in browser. A shape-only `amazon.` pattern also accepts unrelated +// registrable domains such as amazon.shop or amazon.zip. +const MARKETPLACE_DOMAINS = new Set([ + 'amazon.com', + 'amazon.ca', + 'amazon.com.mx', + 'amazon.com.br', + 'amazon.co.uk', + 'amazon.de', + 'amazon.fr', + 'amazon.it', + 'amazon.es', + 'amazon.nl', + 'amazon.pl', + 'amazon.se', + 'amazon.com.be', + 'amazon.ie', + 'amazon.com.tr', + 'amazon.ae', + 'amazon.sa', + 'amazon.eg', + 'amazon.co.za', + 'amazon.in', + 'amazon.co.jp', + 'amazon.com.au', + 'amazon.sg', +]); +function isAmazonMarketplaceHost(hostname) { + const normalized = cleanText(hostname).toLowerCase().replace(/\.$/, ''); + for (const domain of MARKETPLACE_DOMAINS) { + if (normalized === domain || normalized.endsWith(`.${domain}`)) + return true; + } + return false; +} const ROBOT_TEXT_PATTERNS = [ 'Sorry, we just need to make sure you\'re not a robot', 'Enter the characters you see below', @@ -91,19 +127,33 @@ export function extractAsin(input) { const match = normalized.match(/\/(?:dp|gp\/product|product-reviews)\/([A-Z0-9]{10})/i); return match ? match[1].toUpperCase() : null; } +export function amazonHostFromInput(input) { + const normalized = cleanText(input); + if (!normalized) + return null; + try { + const url = new URL(normalized); + return isAmazonMarketplaceHost(url.hostname) ? url.hostname : null; + } + catch { + return null; + } +} export function buildProductUrl(input) { const asin = extractAsin(input); if (!asin) { throw new ArgumentError('amazon product expects an ASIN or product URL', 'Example: webcmd amazon product B0FJS72893'); } - return `${PRODUCT_URL_PREFIX}${asin}`; + const host = amazonHostFromInput(input); + return host ? `https://${host}/dp/${asin}` : `${PRODUCT_URL_PREFIX}${asin}`; } export function buildDiscussionUrl(input) { const asin = extractAsin(input); if (!asin) { throw new ArgumentError('amazon discussion expects an ASIN or product URL', 'Example: webcmd amazon discussion B0FJS72893'); } - return `${DISCUSSION_URL_PREFIX}${asin}`; + const host = amazonHostFromInput(input); + return host ? `https://${host}/product-reviews/${asin}` : `${DISCUSSION_URL_PREFIX}${asin}`; } function getRankingSpec(listType) { return AMAZON_RANKING_SPECS[listType]; @@ -206,7 +256,7 @@ export function resolveBestsellersUrl(input) { export function canonicalizeAmazonUrl(input) { try { const url = new URL(input); - if (!url.hostname.endsWith(DOMAIN)) { + if (!isAmazonMarketplaceHost(url.hostname)) { throw new Error('not-amazon'); } return url.toString(); @@ -230,7 +280,7 @@ export function normalizeProductUrl(value) { const normalized = cleanText(value); const asin = extractAsin(normalized); if (asin) - return buildProductUrl(asin); + return buildProductUrl(normalized); return toAbsoluteAmazonUrl(normalized); } export function parsePriceText(text) { @@ -347,8 +397,11 @@ export function assertUsableState(state, action) { export const __test__ = { buildSearchUrl, extractAsin, + amazonHostFromInput, buildProductUrl, buildDiscussionUrl, + normalizeProductUrl, + canonicalizeAmazonUrl, resolveBestsellersUrl, resolveRankingUrl, isSupportedRankingPath, diff --git a/plugins/amazon/test/discussion.test.js b/plugins/amazon/test/discussion.test.js index 050483de..c4522dc0 100644 --- a/plugins/amazon/test/discussion.test.js +++ b/plugins/amazon/test/discussion.test.js @@ -41,6 +41,68 @@ describe('amazon discussion normalization', () => { ]); }); + it('keeps the review marketplace in every emitted url', () => { + const result = __test__.normalizeDiscussionPayload({ + href: 'https://www.amazon.co.uk/product-reviews/B0FGCPFY9L', + average_rating_text: '4.4 out of 5', + total_review_count_text: '40 global ratings', + qa_links: [], + review_samples: [], + }); + + expect(result.asin).toBe('B0FGCPFY9L'); + expect(result.discussion_url).toBe('https://www.amazon.co.uk/product-reviews/B0FGCPFY9L'); + expect(result.product_url).toBe('https://www.amazon.co.uk/dp/B0FGCPFY9L'); + }); + + it('requests the review page on the marketplace the input names', async () => { + const command = getRegistry().get('amazon/discussion'); + const page = createPageMock([ + { + href: 'https://www.amazon.co.uk/product-reviews/B0FGCPFY9L', + title: 'Amazon.co.uk: Example product', + body_text: 'Customer reviews', + }, + { + href: 'https://www.amazon.co.uk/product-reviews/B0FGCPFY9L', + average_rating_text: '4.4 out of 5', + total_review_count_text: '40 global ratings', + review_samples: [], + }, + ]); + + await command.func(page, { input: 'https://www.amazon.co.uk/product-reviews/B0FGCPFY9L', limit: 1 }); + + expect(page.goto.mock.calls[0][0]).toBe('https://www.amazon.co.uk/product-reviews/B0FGCPFY9L'); + }); + + it('names the loaded url when neither page exposes a review summary', async () => { + const command = getRegistry().get('amazon/discussion'); + const emptyPayload = { href: 'https://www.amazon.co.uk/dp/B0FGCPFY9L', average_rating_text: '', total_review_count_text: '', review_samples: [] }; + const page = createPageMock([ + { href: 'https://www.amazon.co.uk/product-reviews/B0FGCPFY9L', title: 'Amazon.co.uk', body_text: 'Customer reviews' }, + emptyPayload, + { href: 'https://www.amazon.co.uk/dp/B0FGCPFY9L', title: 'Amazon.co.uk', body_text: 'Product' }, + emptyPayload, + ]); + + await expect(command.func(page, { input: 'https://www.amazon.co.uk/dp/B0FGCPFY9L', limit: 1 })) + .rejects.toThrow('landed on https://www.amazon.co.uk/dp/B0FGCPFY9L'); + }); + + it('points a gated non-US review page at that marketplace, not the US store', async () => { + const command = getRegistry().get('amazon/discussion'); + const signIn = { href: 'https://www.amazon.co.uk/ap/signin', title: 'Amazon Sign-In', body_text: 'Sign in Create account' }; + const page = createPageMock([ + signIn, + { href: signIn.href, average_rating_text: '', total_review_count_text: '', review_samples: [] }, + signIn, + ]); + + await expect(command.func(page, { input: 'https://www.amazon.co.uk/dp/B0FGCPFY9L', limit: 1 })) + .rejects.toMatchObject({ domain: 'www.amazon.co.uk' }); + }); + it('falls back to the product page when the review page redirects to sign-in', async () => { const command = getRegistry().get('amazon/discussion'); const page = createPageMock([ diff --git a/plugins/amazon/test/shared.test.js b/plugins/amazon/test/shared.test.js index ec040cb9..628da2f8 100644 --- a/plugins/amazon/test/shared.test.js +++ b/plugins/amazon/test/shared.test.js @@ -6,6 +6,35 @@ describe('amazon shared helpers', () => { expect(__test__.buildProductUrl('https://www.amazon.com/dp/B0FJS72893/ref=something')).toBe('https://www.amazon.com/dp/B0FJS72893'); expect(__test__.buildDiscussionUrl('https://www.amazon.com/dp/B0FJS72893')).toBe('https://www.amazon.com/product-reviews/B0FJS72893'); }); + it('keeps the input marketplace instead of rewriting it to the US store', () => { + expect(__test__.buildProductUrl('https://www.amazon.co.uk/dp/B0FGCPFY9L')).toBe('https://www.amazon.co.uk/dp/B0FGCPFY9L'); + expect(__test__.buildProductUrl('https://www.amazon.de/dp/B0FJS72893/ref=something')).toBe('https://www.amazon.de/dp/B0FJS72893'); + expect(__test__.buildProductUrl('https://www.amazon.com.au/dp/B0FJS72893')).toBe('https://www.amazon.com.au/dp/B0FJS72893'); + expect(__test__.buildDiscussionUrl('https://www.amazon.co.uk/product-reviews/B0FGCPFY9L?pageNumber=1')).toBe('https://www.amazon.co.uk/product-reviews/B0FGCPFY9L'); + expect(__test__.normalizeProductUrl('https://www.amazon.co.uk/dp/B0FGCPFY9L')).toBe('https://www.amazon.co.uk/dp/B0FGCPFY9L'); + }); + it('defaults to the US store for bare ASINs and non-marketplace hosts', () => { + expect(__test__.amazonHostFromInput('B0FJS72893')).toBeNull(); + expect(__test__.buildProductUrl('B0FJS72893')).toBe('https://www.amazon.com/dp/B0FJS72893'); + expect(__test__.buildDiscussionUrl('B0FJS72893')).toBe('https://www.amazon.com/product-reviews/B0FJS72893'); + expect(__test__.normalizeProductUrl('B0FJS72893')).toBe('https://www.amazon.com/dp/B0FJS72893'); + }); + it('accepts sibling marketplaces but rejects look-alike hosts', () => { + expect(__test__.amazonHostFromInput('https://www.amazon.co.uk/dp/B0FJS72893')).toBe('www.amazon.co.uk'); + expect(__test__.amazonHostFromInput('https://amazon.de/dp/B0FJS72893')).toBe('amazon.de'); + expect(__test__.amazonHostFromInput('https://amazon.com.au/dp/B0FJS72893')).toBe('amazon.com.au'); + expect(__test__.amazonHostFromInput('https://smile.amazon.com.be/dp/B0FJS72893')).toBe('smile.amazon.com.be'); + expect(__test__.amazonHostFromInput('https://evilamazon.com/dp/B0FJS72893')).toBeNull(); + expect(__test__.amazonHostFromInput('https://amazon.com.evil.com/dp/B0FJS72893')).toBeNull(); + expect(__test__.amazonHostFromInput('https://amazon.evil.com/dp/B0FJS72893')).toBeNull(); + expect(__test__.amazonHostFromInput('https://x.amazon.evil.com/dp/B0FJS72893')).toBeNull(); + expect(__test__.amazonHostFromInput('https://amazon.attacker.io/dp/B0FJS72893')).toBeNull(); + expect(__test__.amazonHostFromInput('https://amazon.shop/dp/B0FJS72893')).toBeNull(); + expect(__test__.amazonHostFromInput('https://amazon.zip/dp/B0FJS72893')).toBeNull(); + expect(() => __test__.canonicalizeAmazonUrl('https://amazon.evil.com/gp/bestsellers')).toThrow('Invalid Amazon URL'); + expect(__test__.canonicalizeAmazonUrl('https://www.amazon.co.uk/gp/bestsellers/books')).toBe('https://www.amazon.co.uk/gp/bestsellers/books'); + expect(() => __test__.canonicalizeAmazonUrl('https://evilamazon.com/gp/bestsellers')).toThrow('Invalid Amazon URL'); + }); it('parses price, rating, and review-count text', () => { expect(__test__.parsePriceText('1 offer from $34.11')).toEqual({ price_text: '$34.11', diff --git a/plugins/facebook/feed.js b/plugins/facebook/feed.js index d91357a7..59175045 100644 --- a/plugins/facebook/feed.js +++ b/plugins/facebook/feed.js @@ -306,12 +306,35 @@ async function loadFeedPosts(page, limit) { + Array.from(document.querySelectorAll('[aria-label]')).filter((el) => /^(Actions for this post)$/i.test((el.getAttribute('aria-label') || '').trim())).length : 0; })()`; - let prev = -1; + const extractStep = buildFeedExtractScript(limit); + let prevMarkerCount = -1; + let prevRowCount = -1; + let stalledPasses = 0; for (let i = 0; i < 8; i += 1) { - let count = 0; - try { count = Number(unwrapBrowserResult(await page.evaluate(scrollStep))) || 0; } catch { break; } - if (count >= limit || count === prev) break; - prev = count; + let markerCount = 0; + try { markerCount = Number(unwrapBrowserResult(await page.evaluate(scrollStep))) || 0; } catch { break; } + + // Raw article/menu counts include comments, suggestions, and other chrome. + // Do not stop merely because those markers reached --limit; stop + // when the actual feed extractor has enough valid rows. + let rowCount = 0; + try { + const payload = unwrapBrowserResult(await page.evaluate(extractStep)); + rowCount = Array.isArray(payload && payload.rows) ? payload.rows.length : 0; + } catch { + // The final extraction below owns error classification. A transient + // observation failure here should only make the bounded scroll continue. + } + if (rowCount >= limit) break; + + if (markerCount === prevMarkerCount && rowCount === prevRowCount) { + stalledPasses += 1; + if (stalledPasses >= 2) break; + } else { + stalledPasses = 0; + } + prevMarkerCount = markerCount; + prevRowCount = rowCount; } } @@ -387,5 +410,6 @@ export const __test__ = { buildFeedExtractScript, command, getFacebookFeed, + loadFeedPosts, requireLimit, }; diff --git a/plugins/facebook/profile.js b/plugins/facebook/profile.js index 14b065d0..3c5eb2f0 100644 --- a/plugins/facebook/profile.js +++ b/plugins/facebook/profile.js @@ -17,19 +17,48 @@ cli({ pipeline: [ { navigate: { url: 'https://www.facebook.com/${{ args.username }}', settleMs: 3000 } }, { evaluate: `(() => { - const h1 = document.querySelector('h1'); - let name = h1 ? h1.textContent.trim() : ''; + const username = \${{ args.username | json }}; + const main = document.querySelector('[role="main"]') || document; + const clean = (value) => String(value || '').replace(/\\s+/g, ' ').trim(); + const h1 = main.querySelector('h1'); + let name = clean(h1 && h1.textContent); - // Find friends/followers links - const links = Array.from(document.querySelectorAll('a')); - const friendsLink = links.find(a => a.href && a.href.includes('/friends')); - const followersLink = links.find(a => a.href && a.href.includes('/followers')); + // Facebook's current profile header no longer uses an h1. The profile-avatar + // link carries the display name on both the link and its role=img child. + // Scope this fallback to the requested profile path so page chrome cannot win. + if (!name) { + const requestedPrefix = '/' + String(username).toLowerCase() + '/'; + const profileMediaLink = Array.from(main.querySelectorAll('a[aria-label][href]')).find((link) => { + const href = link.getAttribute('href') || ''; + let path = ''; + try { path = new URL(href, window.location.href).pathname.toLowerCase(); } catch {} + const image = link.querySelector('[role="img"][aria-label]'); + return path.startsWith(requestedPrefix) + && image + && clean(link.getAttribute('aria-label')) === clean(image.getAttribute('aria-label')); + }); + name = clean(profileMediaLink && profileMediaLink.getAttribute('aria-label')); + } + + // Scope relationship links to the profile main region. Global navigation also + // contains /friends and previously produced empty or generic chrome text. + const links = Array.from(main.querySelectorAll('a[href]')); + const hasProfilePath = (link, suffix) => { + try { + const path = new URL(link.getAttribute('href') || '', window.location.href).pathname + .replace(/\\/+$/, '') + .toLowerCase(); + return path === '/' + String(username).toLowerCase() + suffix; + } catch { return false; } + }; + const friendsLink = links.find((a) => hasProfilePath(a, '/friends')); + const followersLink = links.find((a) => hasProfilePath(a, '/followers')); return [{ name: name, - username: \${{ args.username | json }}, - friends: friendsLink ? friendsLink.textContent.trim() : '-', - followers: followersLink ? followersLink.textContent.trim() : '-', + username, + friends: friendsLink ? clean(friendsLink.textContent) : '-', + followers: followersLink ? clean(followersLink.textContent) : '-', url: window.location.href, }]; })() diff --git a/plugins/facebook/search.js b/plugins/facebook/search.js index 0a01c107..42ee83f9 100644 --- a/plugins/facebook/search.js +++ b/plugins/facebook/search.js @@ -54,11 +54,34 @@ function buildSearchExtractScript(limit) { try { u = new URL(href, 'https://www.facebook.com'); } catch (e) { return false; } // drop hidden-domain .com spam — real results stay on facebook.com if (!/(^|\\.)facebook\\.com$/i.test(u.hostname)) return false; + // l.facebook.com / lm.facebook.com only wrap outbound links (/l.php?u=…); + // they are redirect shims, never search entities. + if (/^lm?\\.facebook\\.com$/i.test(u.hostname)) return false; const p = u.pathname; if (/^\\/search(\\/|$)/i.test(p)) return false; // decoy links back to search (incl. bare /search) // chrome / non-result destinations that the catch-all below would keep if (/^\\/(login|checkpoint|help|policies|privacy|settings|bookmarks|messages|notifications|marketplace|gaming|friends|requests|saved|me)\\b/i.test(p)) return false; - return /^\\/(profile\\.php|groups\\/|events\\/|watch\\/|reel\\/|pages\\/|permalink\\.php|story\\.php|[^/]+\\/posts\\/|[^/]+\\/videos\\/|[A-Za-z0-9.\\-]{2,}\\/?$)/i.test(p); + return /^\\/(profile\\.php|photo\\.php|groups\\/|events\\/|watch\\/|reel\\/|pages\\/|permalink\\.php|story\\.php|[^/]+\\/posts\\/|[^/]+\\/videos\\/|[A-Za-z0-9.\\-]{2,}\\/?$)/i.test(p); + } + + // Query-identity destinations (permalink.php?story_fbid=…, story.php, + // photo.php?fbid=…, watch/?v=…) collapse into a single row when the query is + // dropped — different posts/videos share the same pathname. Keep the identity + // params, but only those: FB appends per-render tracking nonces (__cft__, + // __tn__, ref) that would otherwise defeat dedup by making the same post look + // unique on every render. + function entityKey(u) { + const p = u.pathname.toLowerCase(); + let identityParams = []; + if (p === '/profile.php') identityParams = ['id']; + else if (p === '/permalink.php' || p === '/story.php') { + identityParams = ['story_fbid', 'story_id', 'fbid', 'id']; + } else if (p === '/photo.php') identityParams = ['fbid', 'id']; + else if (p === '/watch' || p === '/watch/') identityParams = ['v']; + const ids = identityParams + .filter((k) => u.searchParams.has(k)) + .map((k) => k + '=' + u.searchParams.get(k)); + return ids.length ? (u.origin + u.pathname + '?' + ids.join('&')) : (u.origin + u.pathname); } function isAuthPage() { @@ -81,8 +104,8 @@ function buildSearchExtractScript(limit) { const rawHref = a.href || a.getAttribute('href') || ''; if (!isEntityHref(rawHref)) continue; let key; - try { const u = new URL(rawHref, 'https://www.facebook.com'); key = u.origin + u.pathname; } - catch (e) { key = rawHref.split('?')[0].split('#')[0]; } + try { const u = new URL(rawHref, 'https://www.facebook.com'); key = entityKey(u); } + catch (e) { key = rawHref.split('#')[0]; } if (seen.has(key)) continue; const title = clean(a.textContent).substring(0, 80); diff --git a/plugins/facebook/test/feed.test.js b/plugins/facebook/test/feed.test.js index e274e44b..30a3d33f 100644 --- a/plugins/facebook/test/feed.test.js +++ b/plugins/facebook/test/feed.test.js @@ -156,6 +156,24 @@ describe('facebook feed', () => { }]); }); + it('keeps scrolling when raw article markers reach the limit but valid rows do not', async () => { + const page = { + evaluate: vi.fn() + .mockResolvedValueOnce(4) + .mockResolvedValueOnce({ status: 'no_rows', rows: [] }) + .mockResolvedValueOnce(5) + .mockResolvedValueOnce({ + status: 'ok', + rows: [{ index: 1, author: 'A', content: 'Body', likes: '-', comments: '-', shares: '-' }], + }), + }; + + await __test__.loadFeedPosts(page, 1); + + expect(page.evaluate).toHaveBeenCalledTimes(4); + expect(String(page.evaluate.mock.calls[1][0])).toContain('primaryContainers'); + }); + it('maps auth, real empty, parser drift, and malformed payloads to typed errors', async () => { await expect(__test__.command.func(createPage({ status: 'auth', rows: [] }), { limit: 1 })) .rejects.toBeInstanceOf(AuthRequiredError); diff --git a/plugins/facebook/test/profile.test.js b/plugins/facebook/test/profile.test.js new file mode 100644 index 00000000..39b28000 --- /dev/null +++ b/plugins/facebook/test/profile.test.js @@ -0,0 +1,81 @@ +import { describe, expect, it } from 'vitest'; +import { JSDOM } from 'jsdom'; +import { getRegistry } from '@agentrhq/webcmd/registry'; +import '../profile.js'; + +function runExtract(html, username = 'zuck') { + const dom = new JSDOM(html, { url: `https://www.facebook.com/${username}` }); + const command = getRegistry().get('facebook/profile'); + const script = command.pipeline.find((step) => step.evaluate).evaluate + .replace('${{ args.username | json }}', JSON.stringify(username)); + return Function('window', 'document', `return ${script};`)(dom.window, dom.window.document); +} + +describe('facebook profile', () => { + it('keeps the existing profile row contract', () => { + const command = getRegistry().get('facebook/profile'); + expect(command).toBeDefined(); + expect(command.columns).toEqual(['name', 'username', 'friends', 'followers', 'url']); + }); + + it('extracts the display name from the current profile-avatar header', () => { + const rows = runExtract(` + +
+ + + + + + +
Mark Zuckerberg
+ 1.2M followers + 1,234 friends +
+ `); + + expect(rows).toEqual([{ + name: 'Mark Zuckerberg', + username: 'zuck', + friends: '1,234 friends', + followers: '1.2M followers', + url: 'https://www.facebook.com/zuck', + }]); + }); + + it('prefers an h1 when Facebook still renders the legacy header', () => { + const rows = runExtract(` +
+

Legacy Profile Name

+ + + +
+ `, 'legacy'); + + expect(rows[0]).toMatchObject({ + name: 'Legacy Profile Name', + username: 'legacy', + friends: '-', + followers: '-', + }); + }); + + it('does not take a display name or friends label from global page chrome', () => { + const rows = runExtract(` + +
+ `); + + expect(rows[0]).toMatchObject({ + name: '', + friends: '-', + followers: '-', + }); + }); +}); diff --git a/plugins/facebook/test/search.test.js b/plugins/facebook/test/search.test.js index 53145b85..89631081 100644 --- a/plugins/facebook/test/search.test.js +++ b/plugins/facebook/test/search.test.js @@ -80,6 +80,76 @@ describe('facebook search', () => { expect(rows[0].url).toBe('https://www.facebook.com/carol.page'); }); + it('keeps distinct permalink.php posts apart by their identity query', async () => { + // permalink.php / story.php / watch encode identity in the query string; + // deduping on pathname alone collapses different posts into one row. + const page = createDomPage(` +
+
First distinct post about AI research
+
Second distinct post about AI safety
+
Video one about AI agents here
+
Video two about AI agents here
+
+ `); + + const rows = await searchCommand().func(page, { query: 'ai', limit: 10 }); + expect(rows.map((row) => row.url)).toEqual([ + 'https://www.facebook.com/permalink.php?story_fbid=1001&id=50', + 'https://www.facebook.com/permalink.php?story_fbid=2002&id=50', + 'https://www.facebook.com/watch/?v=111', + 'https://www.facebook.com/watch/?v=222', + ]); + }); + + it('keeps profile and photo identities without treating arbitrary query params as identity', async () => { + const page = createDomPage(` +
+
First profile result with details
+
Second profile result with details
+
Photo result with useful details
+
Same vanity page first render
+
Same vanity page second render
+
+ `); + + const rows = await searchCommand().func(page, { query: 'ai', limit: 10 }); + expect(rows.map((row) => row.url)).toEqual([ + 'https://www.facebook.com/profile.php?id=1001', + 'https://www.facebook.com/profile.php?id=2002', + 'https://www.facebook.com/photo.php?fbid=3003&id=1001', + 'https://www.facebook.com/realpage', + ]); + }); + + it('dedupes one post rendered with different per-render tracking nonces', async () => { + // FB appends __cft__ / __tn__ nonces that differ on every render; keeping + // them in the key would make the same post appear as multiple rows. + const page = createDomPage(` +
+
Same post rendered once here now
+
Same post rendered twice here now
+
+ `); + + const rows = await searchCommand().func(page, { query: 'ai', limit: 10 }); + expect(rows.map((row) => row.url)).toEqual(['https://www.facebook.com/permalink.php?story_fbid=1001&id=50']); + }); + + it('drops l.facebook.com / lm.facebook.com outbound-redirect shims', async () => { + // /l.php?u=… wrappers are external-link redirects, not search entities; their + // pathname would otherwise slip through the vanity catch-all. + const page = createDomPage(` +
+
Real Page result here
+ External article link in a post + Another external redirect link here +
+ `); + + const rows = await searchCommand().func(page, { query: 'ai', limit: 10 }); + expect(rows.map((row) => row.url)).toEqual(['https://www.facebook.com/realpage']); + }); + it('validates query and limit before navigation', async () => { const page = createPage({ status: 'ok', rows: [] }); await expect(searchCommand().func(page, { query: ' ', limit: 3 })).rejects.toBeInstanceOf(ArgumentError); diff --git a/plugins/tiktok/explore.js b/plugins/tiktok/explore.js index ff000cb6..262fb32b 100644 --- a/plugins/tiktok/explore.js +++ b/plugins/tiktok/explore.js @@ -60,32 +60,54 @@ function buildExploreScript(limit) { if (row && !dedup.has(row.id)) dedup.set(row.id, row); } - const msToken = getCookie('msToken'); - let apiFailure = null; - if (dedup.size < limit) { - let cursor = 0; + async function collectEndpoint(path, label, extraParams, initialCursor) { + let cursor = initialCursor; for (let page = 0; page < maxPages && dedup.size < limit; page += 1) { const params = new URLSearchParams({ aid, count: String(pageSize), - from_page: 'fyp', - cursor: String(cursor), + ...extraParams, }); + if (cursor !== undefined) params.set('cursor', String(cursor)); if (msToken) params.set('msToken', msToken); - try { - const data = await fetchJson('/api/recommend/item_list/?' + params.toString()); + const data = await fetchJson(path + '?' + params.toString()); + if (label === 'explore') { + assertTikTokApiSuccess(data, 'explore'); + } else { assertTikTokApiSuccess(data, 'recommend'); - const items = Array.isArray(data.itemList) ? data.itemList : []; - for (const item of items) { - const row = normalizeVideoItem(item, dedup.size + 1); - if (row && !dedup.has(row.id)) dedup.set(row.id, row); - } - if (data.hasMore !== true && items.length === 0) break; - cursor = asNumber(data.cursor) ?? cursor + items.length; - } catch (error) { - apiFailure = error instanceof Error ? error.message : String(error); - break; } + const items = Array.isArray(data.itemList) + ? data.itemList + : (Array.isArray(data.items) ? data.items : []); + for (const item of items) { + const row = normalizeVideoItem(item, dedup.size + 1); + if (row && !dedup.has(row.id)) dedup.set(row.id, row); + } + const nextCursor = asNumber(data.cursor); + const hasMore = data.hasMore === true || data.has_more === true; + if (!hasMore || items.length === 0 || (cursor !== undefined && nextCursor === cursor)) break; + cursor = nextCursor ?? ((cursor ?? 0) + items.length); + } + } + + const msToken = getCookie('msToken'); + const apiFailures = []; + if (dedup.size < limit) { + try { + // TikTok's current /explore page uses this endpoint. categoryType=120 + // is the "All" tab shown by default. + await collectEndpoint('/api/explore/item_list/', 'explore', { categoryType: '120' }); + } catch (error) { + apiFailures.push(error instanceof Error ? error.message : String(error)); + } + } + if (dedup.size < limit) { + try { + // Keep the previous recommend feed as a compatibility fallback for + // regions where /explore still hydrates from the For You endpoint. + await collectEndpoint('/api/recommend/item_list/', 'recommend', { from_page: 'fyp' }, 0); + } catch (error) { + apiFailures.push(error instanceof Error ? error.message : String(error)); } } @@ -94,7 +116,7 @@ function buildExploreScript(limit) { .map((row, index) => ({ ...row, index: index + 1 })); if (rows.length === 0) { - const suffix = apiFailure ? ' (recommend API failed: ' + apiFailure + ')' : ''; + const suffix = apiFailures.length ? ' (explore APIs failed: ' + apiFailures.join('; ') + ')' : ''; throw new Error('No videos found on /explore' + suffix); } return rows; diff --git a/plugins/twitter/README.md b/plugins/twitter/README.md index a6f9208a..d8d4f167 100644 --- a/plugins/twitter/README.md +++ b/plugins/twitter/README.md @@ -19,6 +19,7 @@ webcmd plugin install github:agentrhq/webcmd/twitter | `webcmd twitter bookmark-folder` | Read the tweets inside a single Twitter/X bookmark folder. Get the folder id from `webcmd twitter bookmark-folders`. | | `webcmd twitter bookmark-folders` | List your Twitter/X bookmark folders (the user-created collections under Bookmarks). Returns folder id, name, item count, and created_at. | | `webcmd twitter bookmarks` | Fetch your Twitter/X bookmarks (the logged-in user's saved tweets, newest first) | +| `webcmd twitter collection` | Fetch a user timeline with relationship facts and a bounded completion receipt. | | `webcmd twitter delete` | Delete a specific tweet by URL | | `webcmd twitter device-follow` | Read the /i/timeline device-follow notification stream (tweets aggregated under a bell-icon "new posts from @userA and N others" notification) | | `webcmd twitter download` | Download Twitter/X media (images and videos). Provide either to fetch every media item from their profile via the GraphQL UserMedia endpoint with cursor pagination, or --tweet-url to download a single tweet. | diff --git a/plugins/twitter/article.js b/plugins/twitter/article.js index 71821f0b..a5bdabc0 100644 --- a/plugins/twitter/article.js +++ b/plugins/twitter/article.js @@ -173,6 +173,32 @@ cli({ if (!Array.isArray(blocks)) { return {error: 'Twitter API response article blocks were malformed'}; } + // The current GraphQL response serializes Draft.js entityMap as an + // unordered [{key, value}] array, not an object keyed by entity id. + // Normalize both representations before resolving atomic blocks. + const rawEntityMap = contentState.entityMap || {}; + const entityByKey = {}; + if (Array.isArray(rawEntityMap)) { + for (const entry of rawEntityMap) { + if (entry && entry.key != null && entry.value) { + entityByKey[String(entry.key)] = entry.value; + } + } + } else { + for (const [key, entry] of Object.entries(rawEntityMap)) { + entityByKey[String(key)] = entry?.value || entry; + } + } + + // Build media_id -> original_img_url lookup from media_entities. + const mediaEntities = articleResults.media_entities || []; + const mediaUrlById = {}; + for (const me of Object.values(mediaEntities)) { + const url = me?.media_info?.original_img_url; + if (typeof url === 'string' && me?.media_id != null) { + mediaUrlById[String(me.media_id)] = url; + } + } // Convert draft.js blocks to Markdown const parts = []; @@ -180,7 +206,17 @@ cli({ for (const block of blocks) { if (!block || typeof block !== 'object' || Array.isArray(block)) continue; const blockType = block.type || 'unstyled'; - if (blockType === 'atomic') continue; + if (blockType === 'atomic') { + const entityKey = block.entityRanges?.[0]?.key; + const entity = entityKey == null ? null : entityByKey[String(entityKey)]; + if (entity?.type === 'MEDIA') { + const mediaId = entity.data?.mediaItems?.[0]?.mediaId; + const imgUrl = mediaId == null ? null : mediaUrlById[String(mediaId)]; + const caption = String(entity.data?.caption || 'Image').replaceAll(']', ']'); + if (imgUrl) parts.push('![' + caption + '](' + imgUrl + ')'); + } + continue; + } const text = block.text || ''; if (!text) continue; if (blockType !== 'ordered-list-item') orderedCounter = 0; diff --git a/plugins/twitter/bookmarks.js b/plugins/twitter/bookmarks.js index c1538105..e6b07ade 100644 --- a/plugins/twitter/bookmarks.js +++ b/plugins/twitter/bookmarks.js @@ -1,9 +1,13 @@ +import fs from 'node:fs'; +import path from 'node:path'; import { cli, Strategy } from '@agentrhq/webcmd/registry'; -import { AuthRequiredError, CommandExecutionError } from '@agentrhq/webcmd/errors'; -import { extractMedia, describeTwitterApiError } from './shared.js'; +import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors'; +import { extractMedia, describeTwitterApiError, resolveTwitterQueryId, unwrapBrowserResult } from './shared.js'; import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js'; const BOOKMARKS_QUERY_ID = 'Fy0QMy4q_aZCpkO0PnyLYw'; -const MAX_PAGINATION_PAGES = 100; +// Safety cap only. Full-archive runs can set a higher page budget via --max-pages. +const DEFAULT_MAX_PAGINATION_PAGES = 100; +const HARD_MAX_PAGINATION_PAGES = 100000; const FEATURES = { rweb_video_screen_enabled: false, profile_label_improvements_pcf_label_in_post_enabled: true, @@ -100,6 +104,137 @@ export function parseBookmarks(data, seen) { } return { tweets, nextCursor }; } +function resolveOptionalFilePath(raw, label) { + if (raw === undefined || raw === null || raw === '') + return ''; + const value = String(raw).trim(); + if (!value) + throw new ArgumentError(`${label} cannot be empty`); + return path.resolve(value); +} +function readResumeFile(filePath, expected = null) { + if (!filePath || !fs.existsSync(filePath)) + return null; + let parsed; + try { + parsed = JSON.parse(fs.readFileSync(filePath, 'utf8')); + } + catch (error) { + throw new CommandExecutionError(`Could not parse Twitter bookmarks resume file ${filePath}: ${error instanceof Error ? error.message : String(error)}`); + } + const count = parsed?.count; + const cursor = parsed?.cursor == null ? null : String(parsed.cursor); + const outputFile = parsed?.outputFile ? path.resolve(String(parsed.outputFile)) : null; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed) + || !Number.isInteger(count) || count < 0 + || (parsed.cursor != null && typeof parsed.cursor !== 'string') + || (cursor !== null && !cursor.trim())) { + throw new CommandExecutionError(`Twitter bookmarks resume file ${filePath} has an invalid shape`); + } + if (expected) { + if (parsed.source !== expected.source) + throw new ArgumentError(`Resume file source mismatch: expected ${expected.source}, found ${parsed.source || 'unknown'}`); + if (outputFile !== expected.outputFile) + throw new ArgumentError(`Resume file output mismatch: expected ${expected.outputFile || 'in-memory mode'}, found ${outputFile || 'in-memory mode'}`); + if (!expected.outputFile && !Array.isArray(parsed.tweets)) + throw new CommandExecutionError(`Twitter bookmarks resume file ${filePath} is missing in-memory tweets`); + if (!expected.outputFile && parsed.tweets.length !== count) + throw new CommandExecutionError(`Twitter bookmarks resume file ${filePath} count does not match its in-memory tweets`); + if (parsed.complete) + throw new CommandExecutionError(`Twitter bookmarks resume file ${filePath} is already marked complete`); + } + return { + cursor, + count, + tweets: Array.isArray(parsed.tweets) ? parsed.tweets : [], + complete: Boolean(parsed.complete), + source: parsed.source || null, + outputFile, + updatedAt: parsed.updatedAt || null, + }; +} +function ensureParentDir(filePath) { + if (!filePath) + return; + fs.mkdirSync(path.dirname(filePath), { recursive: true }); +} +function removeFile(filePath) { + if (!filePath) + return; + try { + fs.rmSync(filePath, { force: true }); + } + catch { + } +} +function loadJsonlArchiveState(filePath) { + const seen = new Set(); + let count = 0; + if (!filePath || !fs.existsSync(filePath)) + return { seen, count }; + const text = fs.readFileSync(filePath, 'utf8'); + for (const [index, line] of text.split('\n').entries()) { + const trimmed = line.trim(); + if (!trimmed) + continue; + try { + const row = JSON.parse(trimmed); + if (!row?.id) + throw new Error('missing id'); + const id = String(row.id); + if (seen.has(id)) + throw new Error(`duplicate id ${id}`); + seen.add(id); + count += 1; + } + catch (error) { + throw new CommandExecutionError(`Invalid JSONL record in ${filePath} at line ${index + 1}: ${error instanceof Error ? error.message : String(error)}`); + } + } + return { seen, count }; +} +function appendJsonlRows(filePath, rows) { + if (!filePath || !Array.isArray(rows) || rows.length === 0) + return; + ensureParentDir(filePath); + // Escape LS/PS so JSONL stays one physical line even when tweet text contains them. + const text = rows + .map((row) => JSON.stringify(row).replace(/\u2028/g, '\\u2028').replace(/\u2029/g, '\\u2029')) + .join('\n') + '\n'; + fs.appendFileSync(filePath, text, 'utf8'); +} +function writeResumeFile(filePath, payload) { + if (!filePath) + return; + ensureParentDir(filePath); + const temporaryPath = `${filePath}.tmp-${process.pid}`; + try { + fs.writeFileSync(temporaryPath, JSON.stringify(payload, null, 2) + '\n'); + fs.renameSync(temporaryPath, filePath); + } + catch (error) { + try { + fs.rmSync(temporaryPath, { force: true }); + } + catch { + } + throw new CommandExecutionError(`Could not persist Twitter bookmarks resume state: ${error instanceof Error ? error.message : String(error)}`); + } +} +function removeResumeFile(filePath) { + removeFile(filePath); +} +function resolveMaxPages(kwargs, fetchAll) { + const raw = kwargs['max-pages']; + if (raw === undefined || raw === null || raw === '') { + return fetchAll ? HARD_MAX_PAGINATION_PAGES : DEFAULT_MAX_PAGINATION_PAGES; + } + const value = Number(raw); + if (!Number.isInteger(value) || value < 1 || value > HARD_MAX_PAGINATION_PAGES) { + throw new ArgumentError(`--max-pages must be an integer between 1 and ${HARD_MAX_PAGINATION_PAGES}`); + } + return value; +} cli({ site: 'twitter', name: 'bookmarks', @@ -109,73 +244,147 @@ cli({ strategy: Strategy.COOKIE, browser: true, args: [ - { name: 'limit', type: 'int', default: 20, help: 'Maximum number of bookmarks to return (default 20).' }, - { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the bookmarks by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API\'s native (saved-time) ordering.' }, + { name: 'limit', type: 'int', default: 20, help: 'Maximum number of bookmarks to return (default 20). Ignored when --all is set.' }, + { name: 'all', type: 'bool', default: false, help: 'Fetch all bookmark pages until exhausted. Prefer --output-file for large archives.' }, + { name: 'resume-file', type: 'string', help: 'Resume file for long-running all-pages bookmark syncs.' }, + { name: 'output-file', type: 'string', help: 'Write all-page results to JSONL. Requires --all and --resume-file.' }, + { name: 'max-pages', type: 'int', help: `Optional pagination safety cap (default ${DEFAULT_MAX_PAGINATION_PAGES}; raised automatically with --all).` }, + { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the bookmarks by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API\'s native (saved-time) ordering. Incompatible with --output-file.' }, ], columns: ['id', 'author', 'text', 'likes', 'retweets', 'bookmarks', 'created_at', 'url', 'has_media', 'media_urls', 'media_posters'], func: async (page, kwargs) => { - const limit = kwargs.limit || 20; + const fetchAll = Boolean(kwargs.all); + const limit = fetchAll ? Number.POSITIVE_INFINITY : (kwargs.limit || 20); + const resumeFile = resolveOptionalFilePath(kwargs['resume-file'], '--resume-file'); + const outputFile = resolveOptionalFilePath(kwargs['output-file'], '--output-file'); + const useOutputFile = Boolean(fetchAll && outputFile); + const maxPages = resolveMaxPages(kwargs, fetchAll); + const topByEngagement = Number(kwargs['top-by-engagement'] || 0); + if (useOutputFile && topByEngagement > 0) { + throw new ArgumentError('--top-by-engagement cannot be combined with --output-file'); + } + if (outputFile && !fetchAll) { + throw new ArgumentError('--output-file requires --all'); + } + if (resumeFile && !fetchAll) { + throw new ArgumentError('--resume-file requires --all'); + } + if (outputFile && !resumeFile) { + throw new ArgumentError('--output-file requires --resume-file so partial archives remain resumable'); + } const cookies = await page.getCookies({ url: 'https://x.com' }); const ct0 = cookies.find((c) => c.name === 'ct0')?.value || null; if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)'); - const queryId = await page.evaluate(`async () => { - try { - const ghResp = await fetch('https://raw.githubusercontent.com/fa0311/twitter-openapi/refs/heads/main/src/config/placeholder.json'); - if (ghResp.ok) { - const data = await ghResp.json(); - const entry = data['Bookmarks']; - if (entry && entry.queryId) return entry.queryId; - } - } catch {} - try { - const scripts = performance.getEntriesByType('resource') - .filter(r => r.name.includes('client-web') && r.name.endsWith('.js')) - .map(r => r.name); - for (const scriptUrl of scripts.slice(0, 15)) { - try { - const text = await (await fetch(scriptUrl)).text(); - const re = /queryId:"([A-Za-z0-9_-]+)"[^}]{0,200}operationName:"Bookmarks"/; - const m = text.match(re); - if (m) return m[1]; - } catch {} - } - } catch {} - return null; - }`) || BOOKMARKS_QUERY_ID; + const queryId = await resolveTwitterQueryId(page, 'Bookmarks', BOOKMARKS_QUERY_ID); const headers = JSON.stringify({ 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, 'X-Csrf-Token': ct0, 'X-Twitter-Auth-Type': 'OAuth2Session', 'X-Twitter-Active-User': 'yes', }); - const allTweets = []; - const seen = new Set(); - let cursor = null; - // Runaway guard only; --limit and cursor exhaustion control normal pagination. - for (let i = 0; i < MAX_PAGINATION_PAGES && allTweets.length < limit; i++) { - const fetchCount = Math.min(100, limit - allTweets.length + 10); + const resumed = fetchAll ? readResumeFile(resumeFile, { + source: 'bookmarks', + outputFile: useOutputFile ? outputFile : null, + }) : null; + if (useOutputFile && resumed && resumed.count > 0 && !fs.existsSync(outputFile)) { + throw new CommandExecutionError(`Twitter bookmarks output file is missing for resume state: ${outputFile}`); + } + if (useOutputFile && !resumed && fs.existsSync(outputFile)) { + throw new ArgumentError(`Refusing to overwrite existing Twitter bookmarks output file: ${outputFile}`); + } + const allTweets = useOutputFile ? [] : (resumed?.tweets ? [...resumed.tweets] : []); + const jsonlState = useOutputFile ? loadJsonlArchiveState(outputFile) : null; + const seen = useOutputFile + ? jsonlState.seen + : new Set(allTweets.map((tweet) => tweet?.id).filter(Boolean)); + if (useOutputFile && resumed && jsonlState.count !== resumed.count) { + throw new CommandExecutionError(`Twitter bookmarks output file has ${jsonlState.count} record(s), expected resume count ${resumed.count}`); + } + let outputCount = useOutputFile ? jsonlState.count : 0; + let cursor = resumed?.cursor || null; + let pages = 0; + let exhausted = false; + // Runaway guard only; --limit/--all and cursor exhaustion control normal pagination. + while (pages < maxPages && (fetchAll || allTweets.length < limit)) { + pages += 1; + const currentCount = useOutputFile ? outputCount : allTweets.length; + const remaining = fetchAll ? 100 : (limit - currentCount + 10); + const fetchCount = Math.min(100, remaining); const apiUrl = buildBookmarksUrl(fetchCount, cursor).replace(BOOKMARKS_QUERY_ID, queryId); - const data = await page.evaluate(`async () => { - const r = await fetch("${apiUrl}", { headers: ${headers}, credentials: 'include' }); + const data = unwrapBrowserResult(await page.evaluate(`async () => { + const r = await fetch(${JSON.stringify(apiUrl)}, { headers: ${headers}, credentials: 'include' }); return r.ok ? await r.json() : { error: r.status }; - }`); + }`)); if (data?.error) { - if (allTweets.length === 0) + if ((useOutputFile ? outputCount : allTweets.length) === 0) throw new CommandExecutionError(describeTwitterApiError('Bookmarks', data.error)); break; } + const hasInstructions = Array.isArray(data?.data?.bookmark_timeline_v2?.timeline?.instructions) + || Array.isArray(data?.data?.bookmark_timeline?.timeline?.instructions); + if (!hasInstructions) { + throw new CommandExecutionError('twitter_bookmarks_protocol_error: missing Bookmarks timeline instructions'); + } const { tweets, nextCursor } = parseBookmarks(data, seen); - allTweets.push(...tweets); - if (!nextCursor || nextCursor === cursor) + if (useOutputFile) { + appendJsonlRows(outputFile, tweets); + outputCount += tweets.length; + } + else { + allTweets.push(...tweets); + } + const pageComplete = !nextCursor; + writeResumeFile(resumeFile, { + cursor: pageComplete ? null : nextCursor, + count: useOutputFile ? outputCount : allTweets.length, + tweets: useOutputFile ? undefined : allTweets, + updatedAt: new Date().toISOString(), + complete: pageComplete, + source: 'bookmarks', + outputFile: useOutputFile ? outputFile : null, + }); + if (pageComplete) { + exhausted = true; break; + } + if (nextCursor === cursor) { + throw new CommandExecutionError('twitter_bookmarks_repeated_cursor: archive completion cannot be proven; resume state was retained'); + } cursor = nextCursor; } - const trimmed = allTweets.slice(0, limit); - return applyTopByEngagement(trimmed, kwargs['top-by-engagement']); + const finalCount = useOutputFile ? outputCount : allTweets.length; + if (finalCount === 0) { + throw new EmptyResultError('twitter bookmarks', 'No bookmarks found for the logged-in account'); + } + // Resume is only removed after the timeline is truly exhausted. Hitting + // --max-pages, partial API errors after some rows, or an interrupt must + // leave the resume file so the next run can continue. + if (exhausted) + removeResumeFile(resumeFile); + if (useOutputFile) { + return { + outputFile, + count: outputCount, + source: 'bookmarks', + complete: exhausted, + pages, + ...(exhausted ? {} : { cursor, resumeFile: resumeFile || null }), + }; + } + if (fetchAll && !exhausted) { + throw new CommandExecutionError( + `twitter_bookmarks_archive_incomplete: stopped after ${pages} page(s); completion cannot be proven`, + resumeFile ? `Resume with --resume-file ${resumeFile}` : 'Rerun with --resume-file to preserve continuation state.', + ); + } + const trimmed = fetchAll ? allTweets : allTweets.slice(0, limit); + return applyTopByEngagement(trimmed, topByEngagement); }, }); export const __test__ = { parseBookmarks, extractBookmarkTweet, + appendJsonlRows, + readResumeFile, }; diff --git a/plugins/twitter/collection.js b/plugins/twitter/collection.js new file mode 100644 index 00000000..ff2f56ce --- /dev/null +++ b/plugins/twitter/collection.js @@ -0,0 +1,339 @@ +import { cli, Strategy } from '@agentrhq/webcmd/registry'; +import { ArgumentError, CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors'; +import { + extractMedia, + extractQuotedTweet, + looksLikePrivateTwitterTimeline, + normalizeTwitterScreenName, +} from './shared.js'; +import { + DEFAULT_USER_TWEETS_PAGE_DELAY_SECONDS, + MAX_USER_TWEETS_LIMIT, + MAX_USER_TWEETS_PAGES, + USER_TWEETS_PAGE_SIZE, + fetchUserTimelinePage, + resolveUserTimelineContext, +} from './user-timeline.js'; + +const RFC3339_TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(Z|([+-])(\d{2}):(\d{2}))$/; + +function isLeapYear(year) { + return year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); +} + +function normalizeUntil(raw) { + const value = String(raw ?? '').trim(); + const match = value.match(RFC3339_TIMESTAMP); + if (!match) { + throw new ArgumentError( + 'twitter collection --until must be an RFC3339 timestamp', + 'Example: webcmd twitter collection @jack --until 2026-07-23T00:00:00Z', + ); + } + const [, yearRaw, monthRaw, dayRaw, hourRaw, minuteRaw, secondRaw, , zone, , offsetHourRaw, offsetMinuteRaw] = match; + const year = Number(yearRaw); + const month = Number(monthRaw); + const day = Number(dayRaw); + const hour = Number(hourRaw); + const minute = Number(minuteRaw); + const second = Number(secondRaw); + const offsetHour = zone === 'Z' ? 0 : Number(offsetHourRaw); + const offsetMinute = zone === 'Z' ? 0 : Number(offsetMinuteRaw); + const daysInMonth = month === 2 + ? (isLeapYear(year) ? 29 : 28) + : ([4, 6, 9, 11].includes(month) ? 30 : 31); + if ( + month < 1 || month > 12 + || day < 1 || day > daysInMonth + || hour > 23 || minute > 59 || second > 59 + || offsetHour > 23 || offsetMinute > 59 + ) { + throw new ArgumentError( + 'twitter collection --until must be an RFC3339 timestamp', + 'Example: webcmd twitter collection @jack --until 2026-07-23T00:00:00Z', + ); + } + const parsed = new Date(value); + if (Number.isNaN(parsed.getTime())) { + throw new ArgumentError( + 'twitter collection --until must be an RFC3339 timestamp', + 'Example: webcmd twitter collection @jack --until 2026-07-23T00:00:00Z', + ); + } + return parsed; +} + +function normalizeCollectionLimit(rawLimit) { + const limit = rawLimit ?? MAX_USER_TWEETS_LIMIT; + if (!Number.isInteger(limit) || limit < 1 || limit > MAX_USER_TWEETS_LIMIT) { + throw new ArgumentError( + `twitter collection --limit must be an integer between 1 and ${MAX_USER_TWEETS_LIMIT}`, + 'Example: webcmd twitter collection @jack --until 2026-07-23T00:00:00Z --limit 250', + ); + } + return limit; +} + +function normalizeCollectionPageDelaySeconds(rawDelay) { + const delay = rawDelay ?? DEFAULT_USER_TWEETS_PAGE_DELAY_SECONDS; + if (!Number.isInteger(delay) || delay < 0 || delay > 60) { + throw new ArgumentError( + 'twitter collection --page-delay must be an integer between 0 and 60 seconds', + 'Example: webcmd twitter collection @jack --until 2026-07-23T00:00:00Z --page-delay 2', + ); + } + return delay; +} + +function unwrapTweetResult(result) { + if (!result) return null; + if (result.__typename === 'TweetWithVisibilityResults' && result.tweet) return result.tweet; + return result.tweet || result; +} + +function relationshipTarget(result, fallbackId = null, contextStatus = 'complete') { + const tweet = unwrapTweetResult(result); + const user = tweet?.core?.user_results?.result; + const rawHandle = user?.legacy?.screen_name || user?.core?.screen_name || null; + const authorHandle = typeof rawHandle === 'string' && normalizeTwitterScreenName(rawHandle) + ? normalizeTwitterScreenName(rawHandle) + : null; + const rawAuthorId = user?.rest_id || user?.legacy?.id_str || null; + const authorId = typeof rawAuthorId === 'string' && rawAuthorId.trim() ? rawAuthorId : null; + const rawPostId = tweet?.rest_id || fallbackId || null; + const postId = typeof rawPostId === 'string' && rawPostId.trim() ? rawPostId : null; + const hasVisibleContext = Boolean(tweet?.note_tweet?.note_tweet_results?.result?.text || tweet?.legacy?.full_text); + const resolvedContextStatus = contextStatus === 'complete' && !hasVisibleContext + ? (postId ? 'unavailable' : 'unknown') + : contextStatus; + return { + post_id: postId, + author_handle: authorHandle, + author_id: authorId, + url: postId && authorHandle ? `https://x.com/${authorHandle}/status/${postId}` : null, + context_status: resolvedContextStatus, + }; +} + +function extractRelationship(result) { + const tweet = unwrapTweetResult(result); + const legacy = tweet?.legacy || {}; + const repostResult = tweet?.retweeted_status_result?.result || legacy.retweeted_status_result?.result || null; + const repostId = legacy.retweeted_status_id_str || null; + if (repostResult || repostId) { + const target = relationshipTarget(repostResult, repostId, repostResult ? 'complete' : 'unknown'); + if (!repostResult || !target.post_id) { + throw new CommandExecutionError('twitter_collection_unresolved_relationship: repost target is unavailable'); + } + return { kind: 'repost', target }; + } + const quoteResult = tweet?.quoted_status_result?.result || legacy.quoted_status_result?.result || null; + const quoteId = legacy.quoted_status_id_str || null; + if (legacy.is_quote_status || quoteResult || quoteId) { + return { + kind: 'quote', + target: relationshipTarget(quoteResult, quoteId, quoteResult ? 'complete' : 'unavailable'), + }; + } + const replyId = legacy.in_reply_to_status_id_str || null; + const replyHandle = normalizeTwitterScreenName(legacy.in_reply_to_screen_name || '') || null; + const replyAuthorId = typeof legacy.in_reply_to_user_id_str === 'string' && legacy.in_reply_to_user_id_str.trim() + ? legacy.in_reply_to_user_id_str + : null; + if (replyId || replyHandle || replyAuthorId) { + return { + kind: 'reply', + target: { + post_id: replyId, + author_handle: replyHandle, + author_id: replyAuthorId, + url: replyId && replyHandle ? `https://x.com/${replyHandle}/status/${replyId}` : null, + context_status: replyId ? 'unavailable' : 'unknown', + }, + }; + } + return { kind: 'original', target: null }; +} + +function extractCollectionPost(result, seen) { + const tweet = unwrapTweetResult(result); + if (!tweet?.rest_id || typeof tweet.rest_id !== 'string') { + throw new CommandExecutionError('twitter_collection_protocol_error: timeline post is missing a stable ID'); + } + if (seen.has(tweet.rest_id)) return null; + seen.add(tweet.rest_id); + const legacy = tweet.legacy || {}; + const user = tweet.core?.user_results?.result; + const author = user?.legacy?.screen_name || user?.core?.screen_name || null; + if (!author || !normalizeTwitterScreenName(author)) { + throw new CommandExecutionError('twitter_collection_protocol_error: timeline post is missing an author handle'); + } + return { + id: tweet.rest_id, + author, + name: user?.legacy?.name || user?.core?.name || '', + text: tweet.note_tweet?.note_tweet_results?.result?.text || legacy.full_text || '', + likes: legacy.favorite_count || 0, + retweets: legacy.retweet_count || 0, + replies: legacy.reply_count || 0, + views: Number(tweet.views?.count) || 0, + is_retweet: Boolean(legacy.retweeted_status_result), + created_at: legacy.created_at || '', + url: `https://x.com/${author}/status/${tweet.rest_id}`, + ...extractMedia(legacy), + quoted_tweet: extractQuotedTweet(tweet), + relationship: extractRelationship(tweet), + }; +} + +function parseCollectionPage(payload, seen) { + if (looksLikePrivateTwitterTimeline(payload)) { + throw new EmptyResultError( + 'twitter collection', + 'Timeline is private or unavailable to the current X account; completion cannot be proven.', + ); + } + const result = payload?.data?.user?.result; + if (!result || typeof result !== 'object') { + throw new CommandExecutionError('twitter_collection_protocol_error: missing UserTweets result'); + } + const instructionSets = [ + result.timeline_v2?.timeline?.instructions, + result.timeline?.timeline?.instructions, + ].filter(Array.isArray); + if (instructionSets.length === 0) { + throw new CommandExecutionError( + 'twitter_collection_protocol_error: missing UserTweets timeline instructions', + ); + } + const posts = []; + let nextCursor = null; + const visit = (value) => { + if (!value || typeof value !== 'object') return; + if (value.type === 'TimelinePinEntry') return; + if (value.tweet_results?.result) { + const post = extractCollectionPost(value.tweet_results.result, seen); + if (post) posts.push(post); + } + if ( + (value.entryType === 'TimelineTimelineCursor' || value.__typename === 'TimelineTimelineCursor') + && (value.cursorType === 'Bottom' || value.cursorType === 'ShowMore') + && value.value + ) { + nextCursor = value.value; + } + if (Array.isArray(value)) { + for (const item of value) visit(item); + return; + } + for (const child of Object.values(value)) { + if (child && typeof child === 'object') visit(child); + } + }; + for (const instructions of instructionSets) visit(instructions); + return [posts, nextCursor]; +} + +function parseCreatedAt(post) { + const parsed = new Date(post.created_at); + if (typeof post.created_at !== 'string' || !post.created_at || Number.isNaN(parsed.getTime())) { + throw new CommandExecutionError(`twitter_collection_invalid_timestamp: post ${post.id}`); + } + return parsed; +} + +function completedReceipt(stopReason, until, pagesFetched, oldestSeenAt) { + return { + completed: true, + stop_reason: stopReason, + requested_until: until.toISOString(), + pages_fetched: pagesFetched, + oldest_seen_at: oldestSeenAt ? oldestSeenAt.toISOString() : null, + }; +} + +async function paginateCollection({ until, limit, maxPages = MAX_USER_TWEETS_PAGES, fetchPage, wait }) { + const seen = new Set(); + const seenCursors = new Set(); + const posts = []; + let cursor = null; + let oldestSeenAt = null; + for (let pageIndex = 0; pageIndex < maxPages; pageIndex++) { + if (pageIndex > 0 && wait) await wait(); + const payload = await fetchPage(cursor, USER_TWEETS_PAGE_SIZE); + if (payload?.error) { + throw new CommandExecutionError(`twitter_collection_request_error: UserTweets returned ${payload.error}`); + } + const [pagePosts, nextCursor] = parseCollectionPage(payload, seen); + for (const post of pagePosts) { + if (posts.length >= limit) { + throw new CommandExecutionError('twitter_collection_limit_reached: pagination cannot prove completion'); + } + const createdAt = parseCreatedAt(post); + if (!oldestSeenAt || createdAt < oldestSeenAt) oldestSeenAt = createdAt; + posts.push(post); + if (createdAt <= until) { + return { + posts, + receipt: completedReceipt('time_boundary_reached', until, pageIndex + 1, oldestSeenAt), + }; + } + } + if (!nextCursor) { + return { + posts, + receipt: completedReceipt('cursor_exhausted', until, pageIndex + 1, oldestSeenAt), + }; + } + if (nextCursor === cursor || seenCursors.has(nextCursor)) { + throw new CommandExecutionError('twitter_collection_repeated_cursor: pagination cannot prove completion'); + } + if (posts.length >= limit) { + throw new CommandExecutionError('twitter_collection_limit_reached: pagination cannot prove completion'); + } + seenCursors.add(nextCursor); + cursor = nextCursor; + } + throw new CommandExecutionError('twitter_collection_page_guard_hit: pagination cannot prove completion'); +} + +cli({ + site: 'twitter', + name: 'collection', + access: 'read', + description: 'Fetch a user timeline with relationship facts and a bounded completion receipt.', + domain: 'x.com', + strategy: Strategy.COOKIE, + browser: true, + args: [ + { name: 'username', type: 'string', positional: true, required: true, help: 'Twitter screen name (with or without @).' }, + { name: 'until', type: 'string', required: true, help: 'RFC3339 lower time boundary that must be reached or exhausted.' }, + { name: 'limit', type: 'int', default: MAX_USER_TWEETS_LIMIT, help: 'Safety ceiling; reaching it is a typed failure.' }, + { name: 'page-delay', type: 'int', default: DEFAULT_USER_TWEETS_PAGE_DELAY_SECONDS, help: 'Seconds to wait between cursor pages.' }, + ], + columns: ['posts', 'receipt'], + func: async (page, kwargs) => { + const until = normalizeUntil(kwargs.until); + const limit = normalizeCollectionLimit(kwargs.limit); + const pageDelaySeconds = normalizeCollectionPageDelaySeconds(kwargs['page-delay']); + const context = await resolveUserTimelineContext(page, kwargs.username, { + allowLoggedInDefault: false, + commandName: 'collection', + }); + return paginateCollection({ + until, + limit, + fetchPage: (cursor, count) => fetchUserTimelinePage(page, context, cursor, count), + wait: pageDelaySeconds > 0 ? () => page.wait(pageDelaySeconds) : null, + }); + }, +}); + +export const __test__ = { + normalizeUntil, + normalizeCollectionLimit, + normalizeCollectionPageDelaySeconds, + extractRelationship, + parseCollectionPage, + paginateCollection, +}; diff --git a/plugins/twitter/likes.js b/plugins/twitter/likes.js index ba23720d..fc98db97 100644 --- a/plugins/twitter/likes.js +++ b/plugins/twitter/likes.js @@ -1,10 +1,14 @@ +import fs from 'node:fs'; +import path from 'node:path'; import { cli, Strategy } from '@agentrhq/webcmd/registry'; import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors'; import { looksLikePrivateTwitterTimeline, normalizeTwitterScreenName, resolveTwitterQueryId, sanitizeQueryId, extractMedia, unwrapBrowserResult, describeTwitterApiError } from './shared.js'; import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js'; const LIKES_QUERY_ID = 'CDWHmpZeSdIJ3HGeRbNm0w'; const USER_BY_SCREEN_NAME_QUERY_ID = 'IGgvgiOx4QZndDHuD3x9TQ'; -const MAX_PAGINATION_PAGES = 100; +// Safety cap only. Full-archive runs can set a higher page budget via --max-pages. +const DEFAULT_MAX_PAGINATION_PAGES = 100; +const HARD_MAX_PAGINATION_PAGES = 100000; const FEATURES = { rweb_video_screen_enabled: false, profile_label_improvements_pcf_label_in_post_enabled: true, @@ -135,6 +139,140 @@ function parseLikes(data, seen) { } return { tweets, nextCursor }; } +function resolveOptionalFilePath(raw, label) { + if (raw === undefined || raw === null || raw === '') + return ''; + const value = String(raw).trim(); + if (!value) + throw new ArgumentError(`${label} cannot be empty`); + return path.resolve(value); +} +function readResumeFile(filePath, expected = null) { + if (!filePath || !fs.existsSync(filePath)) + return null; + let parsed; + try { + parsed = JSON.parse(fs.readFileSync(filePath, 'utf8')); + } + catch (error) { + throw new CommandExecutionError(`Could not parse Twitter likes resume file ${filePath}: ${error instanceof Error ? error.message : String(error)}`); + } + const count = parsed?.count; + const cursor = parsed?.cursor == null ? null : String(parsed.cursor); + const outputFile = parsed?.outputFile ? path.resolve(String(parsed.outputFile)) : null; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed) + || !Number.isInteger(count) || count < 0 + || (parsed.cursor != null && typeof parsed.cursor !== 'string') + || (cursor !== null && !cursor.trim())) { + throw new CommandExecutionError(`Twitter likes resume file ${filePath} has an invalid shape`); + } + if (expected) { + if (parsed.source !== expected.source) + throw new ArgumentError(`Resume file source mismatch: expected ${expected.source}, found ${parsed.source || 'unknown'}`); + if (String(parsed.username || '').toLowerCase() !== String(expected.username).toLowerCase()) + throw new ArgumentError(`Resume file username mismatch: expected @${expected.username}, found @${parsed.username || 'unknown'}`); + if (outputFile !== expected.outputFile) + throw new ArgumentError(`Resume file output mismatch: expected ${expected.outputFile || 'in-memory mode'}, found ${outputFile || 'in-memory mode'}`); + if (!expected.outputFile && !Array.isArray(parsed.tweets)) + throw new CommandExecutionError(`Twitter likes resume file ${filePath} is missing in-memory tweets`); + if (!expected.outputFile && parsed.tweets.length !== count) + throw new CommandExecutionError(`Twitter likes resume file ${filePath} count does not match its in-memory tweets`); + if (parsed.complete) + throw new CommandExecutionError(`Twitter likes resume file ${filePath} is already marked complete`); + } + return { + cursor, + count, + tweets: Array.isArray(parsed.tweets) ? parsed.tweets : [], + username: parsed.username || null, + complete: Boolean(parsed.complete), + source: parsed.source || null, + outputFile, + updatedAt: parsed.updatedAt || null, + }; +} +function ensureParentDir(filePath) { + if (!filePath) + return; + fs.mkdirSync(path.dirname(filePath), { recursive: true }); +} +function removeFile(filePath) { + if (!filePath) + return; + try { + fs.rmSync(filePath, { force: true }); + } + catch { + } +} +function loadJsonlArchiveState(filePath) { + const seen = new Set(); + let count = 0; + if (!filePath || !fs.existsSync(filePath)) + return { seen, count }; + const text = fs.readFileSync(filePath, 'utf8'); + for (const [index, line] of text.split('\n').entries()) { + const trimmed = line.trim(); + if (!trimmed) + continue; + try { + const row = JSON.parse(trimmed); + if (!row?.id) + throw new Error('missing id'); + const id = String(row.id); + if (seen.has(id)) + throw new Error(`duplicate id ${id}`); + seen.add(id); + count += 1; + } + catch (error) { + throw new CommandExecutionError(`Invalid JSONL record in ${filePath} at line ${index + 1}: ${error instanceof Error ? error.message : String(error)}`); + } + } + return { seen, count }; +} +function appendJsonlRows(filePath, rows) { + if (!filePath || !Array.isArray(rows) || rows.length === 0) + return; + ensureParentDir(filePath); + // Escape LS/PS so JSONL stays one physical line even when tweet text contains them. + const text = rows + .map((row) => JSON.stringify(row).replace(/\u2028/g, '\\u2028').replace(/\u2029/g, '\\u2029')) + .join('\n') + '\n'; + fs.appendFileSync(filePath, text, 'utf8'); +} +function writeResumeFile(filePath, payload) { + if (!filePath) + return; + ensureParentDir(filePath); + const temporaryPath = `${filePath}.tmp-${process.pid}`; + try { + fs.writeFileSync(temporaryPath, JSON.stringify(payload, null, 2) + '\n'); + fs.renameSync(temporaryPath, filePath); + } + catch (error) { + try { + fs.rmSync(temporaryPath, { force: true }); + } + catch { + } + throw new CommandExecutionError(`Could not persist Twitter likes resume state: ${error instanceof Error ? error.message : String(error)}`); + } +} +function removeResumeFile(filePath) { + removeFile(filePath); +} +function resolveMaxPages(kwargs, fetchAll) { + const raw = kwargs['max-pages']; + if (raw === undefined || raw === null || raw === '') { + return fetchAll ? HARD_MAX_PAGINATION_PAGES : DEFAULT_MAX_PAGINATION_PAGES; + } + const value = Number(raw); + if (!Number.isInteger(value) || value < 1 || value > HARD_MAX_PAGINATION_PAGES) { + throw new ArgumentError(`--max-pages must be an integer between 1 and ${HARD_MAX_PAGINATION_PAGES}`); + } + return value; +} cli({ site: 'twitter', name: 'likes', @@ -145,12 +283,34 @@ cli({ browser: true, args: [ { name: 'username', type: 'string', positional: true, help: 'Twitter screen name (with or without @). Defaults to the logged-in user when omitted.' }, - { name: 'limit', type: 'int', default: 20, help: 'Maximum number of liked tweets to return (default 20).' }, - { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the liked tweets by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API\'s native (recency) ordering.' }, + { name: 'limit', type: 'int', default: 20, help: 'Maximum number of liked tweets to return (default 20). Ignored when --all is set.' }, + { name: 'all', type: 'bool', default: false, help: 'Fetch all liked-tweet pages until exhausted. Prefer --output-file for large archives.' }, + { name: 'resume-file', type: 'string', help: 'Resume file for long-running all-pages likes syncs.' }, + { name: 'output-file', type: 'string', help: 'Write all-page results to JSONL. Requires --all and --resume-file.' }, + { name: 'max-pages', type: 'int', help: `Optional pagination safety cap (default ${DEFAULT_MAX_PAGINATION_PAGES}; raised automatically with --all).` }, + { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the liked tweets by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API\'s native (recency) ordering. Incompatible with --output-file.' }, ], columns: ['id', 'author', 'name', 'text', 'likes', 'retweets', 'created_at', 'url', 'has_media', 'media_urls', 'media_posters'], func: async (page, kwargs) => { - const limit = kwargs.limit || 20; + const fetchAll = Boolean(kwargs.all); + const limit = fetchAll ? Number.POSITIVE_INFINITY : (kwargs.limit || 20); + const resumeFile = resolveOptionalFilePath(kwargs['resume-file'], '--resume-file'); + const outputFile = resolveOptionalFilePath(kwargs['output-file'], '--output-file'); + const useOutputFile = Boolean(fetchAll && outputFile); + const maxPages = resolveMaxPages(kwargs, fetchAll); + const topByEngagement = Number(kwargs['top-by-engagement'] || 0); + if (useOutputFile && topByEngagement > 0) { + throw new ArgumentError('--top-by-engagement cannot be combined with --output-file'); + } + if (outputFile && !fetchAll) { + throw new ArgumentError('--output-file requires --all'); + } + if (resumeFile && !fetchAll) { + throw new ArgumentError('--resume-file requires --all'); + } + if (outputFile && !resumeFile) { + throw new ArgumentError('--output-file requires --resume-file so partial archives remain resumable'); + } const rawUsername = String(kwargs.username ?? '').trim(); let username = normalizeTwitterScreenName(rawUsername); if (rawUsername && !username) { @@ -199,38 +359,114 @@ cli({ if (!userId) { throw new CommandExecutionError(`Could not find user @${username}`); } - const allTweets = []; - const seen = new Set(); - let cursor = null; + const resumed = fetchAll ? readResumeFile(resumeFile, { + source: 'likes', + username, + outputFile: useOutputFile ? outputFile : null, + }) : null; + if (useOutputFile && resumed && resumed.count > 0 && !fs.existsSync(outputFile)) { + throw new CommandExecutionError(`Twitter likes output file is missing for resume state: ${outputFile}`); + } + if (useOutputFile && !resumed && fs.existsSync(outputFile)) { + throw new ArgumentError(`Refusing to overwrite existing Twitter likes output file: ${outputFile}`); + } + const allTweets = useOutputFile ? [] : (resumed?.tweets ? [...resumed.tweets] : []); + const jsonlState = useOutputFile ? loadJsonlArchiveState(outputFile) : null; + const seen = useOutputFile + ? jsonlState.seen + : new Set(allTweets.map((tweet) => tweet?.id).filter(Boolean)); + if (useOutputFile && resumed && jsonlState.count !== resumed.count) { + throw new CommandExecutionError(`Twitter likes output file has ${jsonlState.count} record(s), expected resume count ${resumed.count}`); + } + let outputCount = useOutputFile ? jsonlState.count : 0; + let cursor = resumed?.cursor || null; let lastRawResponse = null; - // Runaway guard only; --limit and cursor exhaustion control normal pagination. - for (let i = 0; i < MAX_PAGINATION_PAGES && allTweets.length < limit; i++) { - const fetchCount = Math.min(100, limit - allTweets.length + 10); + let pages = 0; + let exhausted = false; + // Runaway guard only; --limit/--all and cursor exhaustion control normal pagination. + while (pages < maxPages && (fetchAll || allTweets.length < limit)) { + pages += 1; + const currentCount = useOutputFile ? outputCount : allTweets.length; + const remaining = fetchAll ? 100 : (limit - currentCount + 10); + const fetchCount = Math.min(100, remaining); const apiUrl = buildLikesUrl(likesQueryId, userId, fetchCount, cursor); const data = unwrapBrowserResult(await page.evaluate(`async () => { - const r = await fetch("${apiUrl}", { headers: ${headers}, credentials: 'include' }); + const r = await fetch(${JSON.stringify(apiUrl)}, { headers: ${headers}, credentials: 'include' }); return r.ok ? await r.json() : { error: r.status }; }`)); if (data?.error) { - if (allTweets.length === 0) + if ((useOutputFile ? outputCount : allTweets.length) === 0) throw new CommandExecutionError(describeTwitterApiError('Likes', data.error)); break; } lastRawResponse = data; + const hasInstructions = Array.isArray(data?.data?.user?.result?.timeline_v2?.timeline?.instructions) + || Array.isArray(data?.data?.user?.result?.timeline?.timeline?.instructions); + if (!hasInstructions) { + if (looksLikePrivateTwitterTimeline(data) && (useOutputFile ? outputCount : allTweets.length) === 0) { + throw new EmptyResultError('twitter likes', `No likes returned for @${username} (Likes are private by default on X; only the account owner can view their own likes)`); + } + throw new CommandExecutionError('twitter_likes_protocol_error: missing Likes timeline instructions'); + } const { tweets, nextCursor } = parseLikes(data, seen); - allTweets.push(...tweets); - if (!nextCursor || nextCursor === cursor) + if (useOutputFile) { + appendJsonlRows(outputFile, tweets); + outputCount += tweets.length; + } + else { + allTweets.push(...tweets); + } + const pageComplete = !nextCursor; + writeResumeFile(resumeFile, { + cursor: pageComplete ? null : nextCursor, + count: useOutputFile ? outputCount : allTweets.length, + tweets: useOutputFile ? undefined : allTweets, + updatedAt: new Date().toISOString(), + complete: pageComplete, + source: 'likes', + username, + outputFile: useOutputFile ? outputFile : null, + }); + if (pageComplete) { + exhausted = true; break; + } + if (nextCursor === cursor) { + throw new CommandExecutionError('twitter_likes_repeated_cursor: archive completion cannot be proven; resume state was retained'); + } cursor = nextCursor; } - if (allTweets.length === 0) { + const finalCount = useOutputFile ? outputCount : allTweets.length; + if (finalCount === 0) { if (looksLikePrivateTwitterTimeline(lastRawResponse)) { throw new EmptyResultError('twitter likes', `No likes returned for @${username} (Likes are private by default on X; only the account owner can view their own likes)`); } throw new EmptyResultError('twitter likes', `No likes found for @${username}`); } - const trimmed = allTweets.slice(0, limit); - return applyTopByEngagement(trimmed, kwargs['top-by-engagement']); + // Resume is only removed after the timeline is truly exhausted. Hitting + // --max-pages, partial API errors after some rows, or an interrupt must + // leave the resume file so the next run can continue. + if (exhausted) + removeResumeFile(resumeFile); + if (useOutputFile) { + return { + outputFile, + count: outputCount, + source: 'likes', + username, + complete: exhausted, + pages, + ...(exhausted ? {} : { cursor, resumeFile: resumeFile || null }), + }; + } + if (fetchAll && !exhausted) { + throw new CommandExecutionError( + `twitter_likes_archive_incomplete: stopped after ${pages} page(s); completion cannot be proven`, + resumeFile ? `Resume with --resume-file ${resumeFile}` : 'Rerun with --resume-file to preserve continuation state.', + ); + } + const trimmed = fetchAll ? allTweets : allTweets.slice(0, limit); + return applyTopByEngagement(trimmed, topByEngagement); }, }); export const __test__ = { @@ -238,4 +474,6 @@ export const __test__ = { buildLikesUrl, buildUserByScreenNameUrl, parseLikes, + appendJsonlRows, + readResumeFile, }; diff --git a/plugins/twitter/profile.js b/plugins/twitter/profile.js index e3574f26..724990a7 100644 --- a/plugins/twitter/profile.js +++ b/plugins/twitter/profile.js @@ -12,6 +12,14 @@ function stringField(value) { return typeof value === 'string' ? value : ''; } +function countField(...values) { + for (const value of values) { + if (typeof value === 'number' && Number.isFinite(value)) + return value; + } + return 0; +} + export function mapTwitterProfileResult(result, screenName) { if (!isPlainObject(result)) { throw new CommandExecutionError(`Twitter profile response for @${screenName} is malformed`); @@ -27,18 +35,18 @@ export function mapTwitterProfileResult(result, screenName) { throw new CommandExecutionError(`Twitter profile response for @${screenName} is missing profile identity fields`); } const location = isPlainObject(result.location) ? result.location : {}; - const expandedUrl = legacy.entities?.url?.urls?.[0]?.expanded_url || ''; + const expandedUrl = stringField(result.website?.url) || stringField(legacy.entities?.url?.urls?.[0]?.expanded_url); return [{ screen_name: stringField(core.screen_name) || stringField(legacy.screen_name) || screenName, name: stringField(core.name) || stringField(legacy.name), - bio: stringField(legacy.description), + bio: stringField(result.profile_bio?.description) || stringField(legacy.description), location: stringField(location.location) || stringField(legacy.location), url: stringField(expandedUrl), - followers: legacy.followers_count || 0, - following: legacy.friends_count || 0, - tweets: legacy.statuses_count || 0, - likes: legacy.favourites_count || 0, - verified: Boolean(result.is_blue_verified || legacy.verified), + followers: countField(result.relationship_counts?.followers, legacy.followers_count, legacy.normal_followers_count), + following: countField(result.relationship_counts?.following, legacy.friends_count), + tweets: countField(result.tweet_counts?.tweets, legacy.statuses_count), + likes: countField(result.action_counts?.favorites_count, legacy.favourites_count), + verified: Boolean(result.is_blue_verified || result.verification?.verified || legacy.verified), created_at: stringField(core.created_at) || stringField(legacy.created_at), }]; } diff --git a/plugins/twitter/test/article.test.js b/plugins/twitter/test/article.test.js index 75254a2d..a5ad065d 100644 --- a/plugins/twitter/test/article.test.js +++ b/plugins/twitter/test/article.test.js @@ -192,6 +192,49 @@ describe('twitter article command', () => { } }); + it('renders atomic media using entity keys rather than entityMap array positions', async () => { + const payload = validArticlePayload({ + article: { + article_results: { + result: { + title: 'Article with image', + content_state: { + blocks: [ + { type: 'unstyled', text: 'before' }, + { type: 'atomic', text: ' ', entityRanges: [{ key: 0, offset: 0, length: 1 }] }, + { type: 'unstyled', text: 'after' }, + ], + entityMap: [ + { key: '7', value: { type: 'LINK', data: { url: 'https://example.com' } } }, + { + key: '0', + value: { + type: 'MEDIA', + data: { + caption: 'architecture diagram', + mediaItems: [{ mediaId: 'media-1' }], + }, + }, + }, + ], + }, + media_entities: [{ + media_id: 'media-1', + media_info: { original_img_url: 'https://pbs.twimg.com/media/example.jpg' }, + }], + }, + }, + }, + }); + const page = createFetchPage(async () => jsonResponse(payload)); + + await expect(command.func(page, { 'tweet-id': TWEET_ID })).resolves.toEqual([ + expect.objectContaining({ + content: 'before\n\n![architecture diagram](https://pbs.twimg.com/media/example.jpg)\n\nafter', + }), + ]); + }); + it('keeps the valid note-tweet fallback', async () => { const payload = validArticlePayload({ article: undefined, diff --git a/plugins/twitter/test/bookmarks.test.js b/plugins/twitter/test/bookmarks.test.js index 29e500e3..1744e6ea 100644 --- a/plugins/twitter/test/bookmarks.test.js +++ b/plugins/twitter/test/bookmarks.test.js @@ -1,4 +1,6 @@ -import { describe, expect, it } from 'vitest'; +import fs from 'node:fs'; +import { describe, expect, it, vi } from 'vitest'; +import { getRegistry } from '@agentrhq/webcmd/registry'; import { __test__ } from '../bookmarks.js'; const { parseBookmarks, extractBookmarkTweet } = __test__; @@ -204,3 +206,276 @@ describe('twitter bookmarks parser', () => { expect(parseBookmarks({}, new Set())).toEqual({ tweets: [], nextCursor: null }); }); }); + +function bookmarksPayload(withBottomCursor = false) { + const entries = [{ + entryId: 'tweet-1', + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '1', + legacy: { + full_text: 'bookmarked post', + favorite_count: 3, + retweet_count: 1, + bookmark_count: 4, + created_at: 'now', + }, + core: { + user_results: { + result: { + legacy: { screen_name: 'alice', name: 'Alice' }, + }, + }, + }, + }, + }, + }, + }, + }]; + if (withBottomCursor) { + entries.push({ + entryId: 'cursor-bottom-1', + content: { + entryType: 'TimelineTimelineCursor', + cursorType: 'Bottom', + value: 'NEXT_CURSOR', + }, + }); + } + return { + data: { + bookmark_timeline_v2: { + timeline: { + instructions: [{ entries }], + }, + }, + }, + }; +} + +describe('twitter bookmarks command', () => { + it('keeps resume state and reports complete=false when --max-pages stops early', async () => { + const command = getRegistry().get('twitter/bookmarks'); + const resumeFile = `/tmp/webcmd-bookmarks-resume-${process.pid}-${Date.now()}.json`; + const outputFile = `/tmp/webcmd-bookmarks-out-${process.pid}-${Date.now()}.jsonl`; + const page = { + getCookies: vi.fn(async () => [{ name: 'ct0', value: 'token' }]), + evaluate: vi.fn(async (script) => { + const text = String(script); + if (text.includes('Bookmarks') && text.includes('queryId')) return null; + if (text.includes('/Bookmarks')) return bookmarksPayload(true); + throw new Error(`Unexpected evaluate: ${text.slice(0, 80)}`); + }), + }; + + try { + const result = await command.func(page, { + all: true, + 'max-pages': 1, + 'resume-file': resumeFile, + 'output-file': outputFile, + }); + + expect(result).toMatchObject({ + outputFile, + count: 1, + source: 'bookmarks', + complete: false, + pages: 1, + cursor: 'NEXT_CURSOR', + resumeFile, + }); + expect(fs.existsSync(resumeFile)).toBe(true); + const resume = __test__.readResumeFile(resumeFile); + expect(resume).toMatchObject({ + cursor: 'NEXT_CURSOR', + count: 1, + complete: false, + source: 'bookmarks', + outputFile, + }); + expect(fs.readFileSync(outputFile, 'utf8').trim().split('\n')).toHaveLength(1); + } + finally { + fs.rmSync(resumeFile, { force: true }); + fs.rmSync(outputFile, { force: true }); + } + }); + + it('removes resume file only after the bookmarks timeline is exhausted', async () => { + const command = getRegistry().get('twitter/bookmarks'); + const resumeFile = `/tmp/webcmd-bookmarks-resume-done-${process.pid}-${Date.now()}.json`; + const outputFile = `/tmp/webcmd-bookmarks-out-done-${process.pid}-${Date.now()}.jsonl`; + const page = { + getCookies: vi.fn(async () => [{ name: 'ct0', value: 'token' }]), + evaluate: vi.fn(async (script) => { + const text = String(script); + if (text.includes('Bookmarks') && text.includes('queryId')) return null; + if (text.includes('/Bookmarks')) return bookmarksPayload(false); + throw new Error(`Unexpected evaluate: ${text.slice(0, 80)}`); + }), + }; + + try { + const result = await command.func(page, { + all: true, + 'max-pages': 1, + 'resume-file': resumeFile, + 'output-file': outputFile, + }); + + expect(result).toMatchObject({ + outputFile, + count: 1, + source: 'bookmarks', + complete: true, + pages: 1, + }); + expect(result.cursor).toBeUndefined(); + expect(fs.existsSync(resumeFile)).toBe(false); + expect(fs.existsSync(outputFile)).toBe(true); + } + finally { + fs.rmSync(resumeFile, { force: true }); + fs.rmSync(outputFile, { force: true }); + } + }); +}); + +describe('twitter bookmarks archive safety', () => { + function pageFor(payload = bookmarksPayload(false), { wrapped = false } = {}) { + return { + getCookies: vi.fn(async () => [{ name: 'ct0', value: 'token' }]), + evaluate: vi.fn(async (script) => { + const text = String(script); + if (text.includes('operationName')) { + return wrapped ? { session: 'site:twitter', data: null } : null; + } + if (text.includes('/Bookmarks')) { + return wrapped ? { session: 'site:twitter', data: payload } : payload; + } + throw new Error(`Unexpected evaluate: ${text.slice(0, 80)}`); + }), + }; + } + + it('rejects --resume-file without --all before touching the browser', async () => { + const command = getRegistry().get('twitter/bookmarks'); + const page = { getCookies: vi.fn(), evaluate: vi.fn() }; + await expect(command.func(page, { 'resume-file': '/tmp/resume.json' })) + .rejects.toThrow(/--resume-file requires --all/); + expect(page.getCookies).not.toHaveBeenCalled(); + }); + + it('unwraps Browser Bridge envelopes for query resolution and bookmark payloads', async () => { + const command = getRegistry().get('twitter/bookmarks'); + const rows = await command.func(pageFor(bookmarksPayload(false), { wrapped: true }), { limit: 1 }); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ id: '1', author: 'alice', text: 'bookmarked post' }); + }); + + it('refuses to overwrite an existing output file without matching resume state', async () => { + const command = getRegistry().get('twitter/bookmarks'); + const outputFile = `/tmp/webcmd-bookmarks-existing-${process.pid}-${Date.now()}.jsonl`; + const resumeFile = `${outputFile}.resume.json`; + fs.writeFileSync(outputFile, 'user-owned\n'); + try { + await expect(command.func(pageFor(), { + all: true, + 'output-file': outputFile, + 'resume-file': resumeFile, + })).rejects.toThrow(/Refusing to overwrite/); + expect(fs.readFileSync(outputFile, 'utf8')).toBe('user-owned\n'); + } + finally { + fs.rmSync(outputFile, { force: true }); + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('rejects cross-source and cross-output resume state', () => { + const resumeFile = `/tmp/webcmd-bookmarks-mismatch-${process.pid}-${Date.now()}.json`; + try { + fs.writeFileSync(resumeFile, JSON.stringify({ + cursor: 'NEXT', + count: 0, + tweets: [], + complete: false, + source: 'likes', + outputFile: null, + })); + expect(() => __test__.readResumeFile(resumeFile, { + source: 'bookmarks', + outputFile: null, + })).toThrow(/source mismatch/); + fs.writeFileSync(resumeFile, JSON.stringify({ + cursor: 'NEXT', + count: 0, + complete: false, + source: 'bookmarks', + outputFile: '/tmp/other.jsonl', + })); + expect(() => __test__.readResumeFile(resumeFile, { + source: 'bookmarks', + outputFile: '/tmp/wanted.jsonl', + })).toThrow(/output mismatch/); + } + finally { + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('rejects output files whose JSONL record count differs from resume state', async () => { + const command = getRegistry().get('twitter/bookmarks'); + const outputFile = `/tmp/webcmd-bookmarks-count-mismatch-${process.pid}-${Date.now()}.jsonl`; + const resumeFile = `${outputFile}.resume.json`; + fs.writeFileSync(outputFile, '{"id":"1"}\n{"id":"2"}\n'); + fs.writeFileSync(resumeFile, JSON.stringify({ + cursor: 'NEXT', + count: 1, + complete: false, + source: 'bookmarks', + outputFile, + })); + try { + await expect(command.func(pageFor(), { + all: true, + 'output-file': outputFile, + 'resume-file': resumeFile, + })).rejects.toThrow(/expected resume count 1/); + } + finally { + fs.rmSync(outputFile, { force: true }); + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('throws for an incomplete in-memory --all run while retaining resume state', async () => { + const command = getRegistry().get('twitter/bookmarks'); + const resumeFile = `/tmp/webcmd-bookmarks-memory-${process.pid}-${Date.now()}.json`; + try { + await expect(command.func(pageFor(bookmarksPayload(true)), { + all: true, + 'max-pages': 1, + 'resume-file': resumeFile, + })).rejects.toThrow(/archive_incomplete/); + expect(__test__.readResumeFile(resumeFile)).toMatchObject({ + cursor: 'NEXT_CURSOR', + count: 1, + source: 'bookmarks', + complete: false, + }); + } + finally { + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('fails closed when the Bookmarks payload has no timeline instructions', async () => { + const command = getRegistry().get('twitter/bookmarks'); + await expect(command.func(pageFor({ data: {} }), { all: true })) + .rejects.toThrow(/missing Bookmarks timeline instructions/); + }); +}); diff --git a/plugins/twitter/test/collection.test.js b/plugins/twitter/test/collection.test.js new file mode 100644 index 00000000..64745f24 --- /dev/null +++ b/plugins/twitter/test/collection.test.js @@ -0,0 +1,276 @@ +import { describe, expect, it, vi } from 'vitest'; +import { getRegistry } from '@agentrhq/webcmd/registry'; +import { ArgumentError, CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors'; +import '../tweets.js'; +import { __test__ } from '../collection.js'; + +function syntheticTweet(id, { + author = 'synth_author', + createdAt = '2026-07-23T12:00:00.000Z', + legacy = {}, + quotedStatusResult, + retweetedStatusResult, +} = {}) { + return { + rest_id: String(id), + legacy: { + full_text: `synthetic post ${id}`, + favorite_count: 0, + retweet_count: 0, + reply_count: 0, + created_at: createdAt, + ...legacy, + }, + core: { + user_results: { + result: { + rest_id: `user-${author}`, + legacy: { screen_name: author, name: author }, + }, + }, + }, + ...(quotedStatusResult ? { quoted_status_result: quotedStatusResult } : {}), + ...(retweetedStatusResult ? { retweeted_status_result: retweetedStatusResult } : {}), + }; +} + +function tweetEntry(tweet) { + return { content: { itemContent: { tweet_results: { result: tweet } } } }; +} + +function collectionPayload(tweets, nextCursor = null) { + const entries = tweets.map(tweetEntry); + if (nextCursor) { + entries.push({ + content: { + entryType: 'TimelineTimelineCursor', + cursorType: 'Bottom', + value: nextCursor, + }, + }); + } + return { + data: { + user: { + result: { + timeline_v2: { timeline: { instructions: [{ entries }] } }, + }, + }, + }, + }; +} + +describe('twitter collection', () => { + it('registers an independent read command with posts and receipt columns', () => { + const command = getRegistry().get('twitter/collection'); + expect(command).toMatchObject({ + access: 'read', + browser: true, + columns: ['posts', 'receipt'], + }); + expect(command?.args?.map((arg) => arg.name)).toEqual([ + 'username', 'until', 'limit', 'page-delay', + ]); + expect(getRegistry().get('twitter/tweets')?.args?.map((arg) => arg.name)) + .not.toContain('collection-receipt'); + }); + + it('classifies original, quote, reply and repost without inventing context', () => { + expect(__test__.extractRelationship(syntheticTweet('10'))).toEqual({ + kind: 'original', + target: null, + }); + expect(__test__.extractRelationship(syntheticTweet('11', { + legacy: { + in_reply_to_status_id_str: '30', + in_reply_to_screen_name: 'parent_author', + in_reply_to_user_id_str: 'user-parent_author', + }, + }))).toMatchObject({ + kind: 'reply', + target: { post_id: '30', context_status: 'unavailable' }, + }); + expect(__test__.extractRelationship(syntheticTweet('12', { + legacy: { is_quote_status: true, quoted_status_id_str: '50' }, + quotedStatusResult: { result: { __typename: 'TweetTombstone' } }, + }))).toMatchObject({ + kind: 'quote', + target: { post_id: '50', context_status: 'unavailable' }, + }); + expect(__test__.extractRelationship(syntheticTweet('13', { + legacy: { retweeted_status_id_str: '60' }, + retweetedStatusResult: { result: syntheticTweet('60', { author: 'repost_target' }) }, + }))).toMatchObject({ + kind: 'repost', + target: { + post_id: '60', + author_handle: 'repost_target', + context_status: 'complete', + }, + }); + }); + + it('rejects an unresolved repost instead of inferring it from text', () => { + expect(() => __test__.extractRelationship(syntheticTweet('14', { + legacy: { full_text: 'RT @someone: synthetic', retweeted_status_id_str: '70' }, + }))).toThrow(CommandExecutionError); + expect(() => __test__.extractRelationship(syntheticTweet('14', { + legacy: { full_text: 'RT @someone: synthetic', retweeted_status_id_str: '70' }, + }))).toThrow(/twitter_collection_unresolved_relationship/); + }); + + it('accepts only RFC3339 lower boundaries', () => { + expect(__test__.normalizeUntil('2026-07-23T00:00:00Z')).toBeInstanceOf(Date); + expect(__test__.normalizeUntil('2026-07-23T00:00:00.123456Z')).toBeInstanceOf(Date); + expect(__test__.normalizeUntil('2024-02-29T00:00:00+08:00')).toBeInstanceOf(Date); + expect(() => __test__.normalizeUntil('2026-07-23')).toThrow(ArgumentError); + expect(() => __test__.normalizeUntil('not-a-date')).toThrow(ArgumentError); + expect(() => __test__.normalizeUntil('2026-02-29T00:00:00Z')).toThrow(ArgumentError); + expect(() => __test__.normalizeUntil('2026-07-23T24:00:00Z')).toThrow(ArgumentError); + expect(() => __test__.normalizeUntil('2026-07-23T00:00:00+24:00')).toThrow(ArgumentError); + }); + + it('fails closed for private, unavailable, and malformed timelines', () => { + expect(() => __test__.parseCollectionPage({ + data: { user: { result: { __typename: 'User', timeline_v2: { timeline: {} } } } }, + }, new Set())).toThrow(EmptyResultError); + expect(() => __test__.parseCollectionPage({ + data: { user: { result: { __typename: 'UserUnavailable' } } }, + }, new Set())).toThrow(/missing UserTweets timeline instructions/); + expect(() => __test__.parseCollectionPage({ + data: { user: { result: { timeline_v2: { timeline: { unexpected: true } } } } }, + }, new Set())).toThrow(/missing UserTweets timeline instructions/); + }); + + it('completes only after the lower boundary is reached', async () => { + const result = await __test__.paginateCollection({ + until: __test__.normalizeUntil('2026-07-23T00:00:00Z'), + limit: 10, + maxPages: 5, + fetchPage: async () => collectionPayload([ + syntheticTweet('20', { createdAt: '2026-07-23T01:00:00.000Z' }), + syntheticTweet('21', { createdAt: '2026-07-22T23:59:59.000Z' }), + ], 'unused-cursor'), + }); + expect(result).toMatchObject({ + posts: [ + { id: '20' }, + { id: '21' }, + ], + receipt: { + completed: true, + stop_reason: 'time_boundary_reached', + requested_until: '2026-07-23T00:00:00.000Z', + pages_fetched: 1, + oldest_seen_at: '2026-07-22T23:59:59.000Z', + }, + }); + }); + + it('completes on cursor exhaustion and exposes no cursor', async () => { + const result = await __test__.paginateCollection({ + until: __test__.normalizeUntil('2026-07-23T00:00:00Z'), + limit: 10, + maxPages: 5, + fetchPage: async () => collectionPayload([ + syntheticTweet('22', { createdAt: '2026-07-23T01:00:00.000Z' }), + ]), + }); + expect(result).toMatchObject({ + receipt: { completed: true, stop_reason: 'cursor_exhausted', pages_fetched: 1 }, + }); + expect(Object.keys(result.receipt)).not.toContain('cursor'); + }); + + it('returns the posts and receipt envelope from the registered command', async () => { + const command = getRegistry().get('twitter/collection'); + const page = { + getCookies: vi.fn(async () => [{ name: 'ct0', value: 'test-only' }]), + wait: vi.fn(async () => undefined), + evaluate: vi.fn(async (script) => { + const source = String(script); + if (source.includes('operationName')) return null; + if (source.includes('/UserByScreenName')) return '42'; + if (source.includes('/UserTweets')) { + return collectionPayload([ + syntheticTweet('27', { createdAt: '2026-07-22T23:59:59.000Z' }), + ]); + } + return null; + }), + }; + const result = await command.func(page, { + username: 'synth_author', + until: '2026-07-23T00:00:00Z', + limit: 10, + 'page-delay': 0, + }); + expect(result).toMatchObject({ + posts: [{ id: '27', relationship: { kind: 'original' } }], + receipt: { completed: true, stop_reason: 'time_boundary_reached' }, + }); + }); + + it('uses collection-specific validation errors before touching the browser', async () => { + const command = getRegistry().get('twitter/collection'); + const page = { + goto: vi.fn(), + getCookies: vi.fn(), + evaluate: vi.fn(), + }; + + await expect(command.func(page, { + username: 'home/extra', + until: '2026-07-23T00:00:00Z', + limit: 10, + 'page-delay': 0, + })).rejects.toThrow(/twitter collection username/); + expect(page.goto).not.toHaveBeenCalled(); + expect(page.getCookies).not.toHaveBeenCalled(); + expect(page.evaluate).not.toHaveBeenCalled(); + }); + + it('fails on repeated cursor, limit, page guard, and malformed timestamps', async () => { + await expect(__test__.paginateCollection({ + until: __test__.normalizeUntil('2026-07-23T00:00:00Z'), + limit: 10, + maxPages: 5, + fetchPage: async () => collectionPayload([ + syntheticTweet('23', { createdAt: '2026-07-23T01:00:00.000Z' }), + ], 'same-cursor'), + })).rejects.toThrow(/twitter_collection_repeated_cursor/); + await expect(__test__.paginateCollection({ + until: __test__.normalizeUntil('2026-07-23T00:00:00Z'), + limit: 1, + maxPages: 5, + fetchPage: async () => collectionPayload([ + syntheticTweet('24', { createdAt: '2026-07-23T01:00:00.000Z' }), + ], 'next-cursor'), + })).rejects.toThrow(/twitter_collection_limit_reached/); + await expect(__test__.paginateCollection({ + until: __test__.normalizeUntil('2026-07-23T00:00:00Z'), + limit: 1, + maxPages: 5, + fetchPage: async () => collectionPayload([ + syntheticTweet('24a', { createdAt: '2026-07-23T01:00:00.000Z' }), + syntheticTweet('24b', { createdAt: '2026-07-22T23:59:59.000Z' }), + ]), + })).rejects.toThrow(/twitter_collection_limit_reached/); + await expect(__test__.paginateCollection({ + until: __test__.normalizeUntil('2026-07-23T00:00:00Z'), + limit: 10, + maxPages: 1, + fetchPage: async () => collectionPayload([ + syntheticTweet('25', { createdAt: '2026-07-23T01:00:00.000Z' }), + ], 'next-cursor'), + })).rejects.toThrow(/twitter_collection_page_guard_hit/); + await expect(__test__.paginateCollection({ + until: __test__.normalizeUntil('2026-07-23T00:00:00Z'), + limit: 10, + maxPages: 5, + fetchPage: async () => collectionPayload([ + syntheticTweet('26', { createdAt: 'not-a-timestamp' }), + ]), + })).rejects.toThrow(/twitter_collection_invalid_timestamp/); + }); +}); diff --git a/plugins/twitter/test/likes.test.js b/plugins/twitter/test/likes.test.js index d7385f7c..2cf91367 100644 --- a/plugins/twitter/test/likes.test.js +++ b/plugins/twitter/test/likes.test.js @@ -1,3 +1,4 @@ +import fs from 'node:fs'; import { describe, expect, it, vi } from 'vitest'; import { getRegistry } from '@agentrhq/webcmd/registry'; import { ArgumentError, AuthRequiredError, EmptyResultError } from '@agentrhq/webcmd/errors'; @@ -154,6 +155,140 @@ describe('twitter likes command', () => { }); }); +describe('twitter likes archive safety', () => { + function pageFor(payload = likesPayload()) { + return { + getCookies: vi.fn(async () => [{ name: 'ct0', value: 'token' }]), + evaluate: vi.fn(async (script) => { + const text = String(script); + if (text.includes('operationName')) return null; + if (text.includes('/UserByScreenName')) return { session: 'site:twitter', data: '42' }; + if (text.includes('/Likes')) return { session: 'site:twitter', data: payload }; + throw new Error(`Unexpected evaluate: ${text.slice(0, 80)}`); + }), + }; + } + + it('rejects --resume-file without --all before touching the browser', async () => { + const command = getRegistry().get('twitter/likes'); + const page = { getCookies: vi.fn(), evaluate: vi.fn() }; + await expect(command.func(page, { username: 'viewer', 'resume-file': '/tmp/resume.json' })) + .rejects.toThrow(/--resume-file requires --all/); + expect(page.getCookies).not.toHaveBeenCalled(); + }); + + it('refuses to overwrite an existing output file without matching resume state', async () => { + const command = getRegistry().get('twitter/likes'); + const outputFile = `/tmp/webcmd-likes-existing-${process.pid}-${Date.now()}.jsonl`; + const resumeFile = `${outputFile}.resume.json`; + fs.writeFileSync(outputFile, 'user-owned\n'); + try { + await expect(command.func(pageFor(), { + username: 'viewer', + all: true, + 'output-file': outputFile, + 'resume-file': resumeFile, + })).rejects.toThrow(/Refusing to overwrite/); + expect(fs.readFileSync(outputFile, 'utf8')).toBe('user-owned\n'); + } + finally { + fs.rmSync(outputFile, { force: true }); + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('rejects cross-source and malformed resume state instead of silently restarting', () => { + const resumeFile = `/tmp/webcmd-likes-mismatch-${process.pid}-${Date.now()}.json`; + try { + fs.writeFileSync(resumeFile, JSON.stringify({ + cursor: 'NEXT', + count: 0, + tweets: [], + complete: false, + source: 'bookmarks', + username: 'viewer', + outputFile: null, + })); + expect(() => __test__.readResumeFile(resumeFile, { + source: 'likes', + username: 'viewer', + outputFile: null, + })).toThrow(/source mismatch/); + fs.writeFileSync(resumeFile, '{broken'); + expect(() => __test__.readResumeFile(resumeFile)).toThrow(/Could not parse/); + } + finally { + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('rejects output files whose JSONL record count differs from resume state', async () => { + const command = getRegistry().get('twitter/likes'); + const outputFile = `/tmp/webcmd-likes-count-mismatch-${process.pid}-${Date.now()}.jsonl`; + const resumeFile = `${outputFile}.resume.json`; + fs.writeFileSync(outputFile, '{"id":"1"}\n{"id":"2"}\n'); + fs.writeFileSync(resumeFile, JSON.stringify({ + cursor: 'NEXT', + count: 1, + complete: false, + source: 'likes', + username: 'viewer', + outputFile, + })); + try { + await expect(command.func(pageFor(), { + username: 'viewer', + all: true, + 'output-file': outputFile, + 'resume-file': resumeFile, + })).rejects.toThrow(/expected resume count 1/); + } + finally { + fs.rmSync(outputFile, { force: true }); + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('throws for an incomplete in-memory --all run while retaining resume state', async () => { + const command = getRegistry().get('twitter/likes'); + const resumeFile = `/tmp/webcmd-likes-memory-${process.pid}-${Date.now()}.json`; + const payload = likesPayload(); + payload.data.user.result.timeline_v2.timeline.instructions[0].entries.push({ + entryId: 'cursor-bottom-1', + content: { + entryType: 'TimelineTimelineCursor', + cursorType: 'Bottom', + value: 'NEXT_CURSOR', + }, + }); + try { + await expect(command.func(pageFor(payload), { + username: 'viewer', + all: true, + 'max-pages': 1, + 'resume-file': resumeFile, + })).rejects.toThrow(/archive_incomplete/); + expect(__test__.readResumeFile(resumeFile)).toMatchObject({ + cursor: 'NEXT_CURSOR', + count: 1, + source: 'likes', + username: 'viewer', + complete: false, + }); + } + finally { + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('fails closed when a non-private Likes payload has no timeline instructions', async () => { + const command = getRegistry().get('twitter/likes'); + await expect(command.func(pageFor({ + data: { user: { result: { __typename: 'UserUnavailable' } } }, + }), { username: 'viewer', all: true })).rejects.toThrow(/missing Likes timeline instructions/); + }); +}); + describe('twitter likes command', () => { it('rejects invalid explicit username before cookies or navigation', async () => { const command = getRegistry().get('twitter/likes'); @@ -217,4 +352,124 @@ describe('twitter likes command', () => { expect(decodeURIComponent(String(likesCall[0]))).toContain('"userId":"42"'); expect(decodeURIComponent(String(likesCall[0]))).not.toContain('[object Object]'); }); + + it('keeps resume state and reports complete=false when --max-pages stops early', async () => { + const command = getRegistry().get('twitter/likes'); + const resumeFile = `/tmp/webcmd-likes-resume-${process.pid}-${Date.now()}.json`; + const outputFile = `/tmp/webcmd-likes-out-${process.pid}-${Date.now()}.jsonl`; + const page = { + goto: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + getCookies: vi.fn(async () => [{ name: 'ct0', value: 'token' }]), + evaluate: vi.fn(async (script) => { + const text = String(script); + if (text.includes('AppTabBar_Profile_Link')) { + return { session: 'site:twitter', data: '/viewer' }; + } + if (text.includes('operationName')) return null; + if (text.includes('/UserByScreenName')) { + return { session: 'site:twitter', data: '42' }; + } + if (text.includes('/Likes')) { + const payload = likesPayload(); + payload.data.user.result.timeline_v2.timeline.instructions[0].entries.push({ + entryId: 'cursor-bottom-1', + content: { + entryType: 'TimelineTimelineCursor', + cursorType: 'Bottom', + value: 'NEXT_CURSOR', + }, + }); + return { session: 'site:twitter', data: payload }; + } + throw new Error(`Unexpected evaluate: ${text.slice(0, 80)}`); + }), + }; + + try { + const result = await command.func(page, { + all: true, + 'max-pages': 1, + 'resume-file': resumeFile, + 'output-file': outputFile, + }); + + expect(result).toMatchObject({ + outputFile, + count: 1, + source: 'likes', + username: 'viewer', + complete: false, + pages: 1, + cursor: 'NEXT_CURSOR', + resumeFile, + }); + expect(fs.existsSync(resumeFile)).toBe(true); + expect(fs.existsSync(outputFile)).toBe(true); + const resume = __test__.readResumeFile(resumeFile); + expect(resume).toMatchObject({ + cursor: 'NEXT_CURSOR', + count: 1, + complete: false, + source: 'likes', + username: 'viewer', + outputFile, + }); + expect(fs.readFileSync(outputFile, 'utf8').trim().split('\n')).toHaveLength(1); + } + finally { + fs.rmSync(resumeFile, { force: true }); + fs.rmSync(outputFile, { force: true }); + } + }); + + it('removes resume file only after the likes timeline is exhausted', async () => { + const command = getRegistry().get('twitter/likes'); + const resumeFile = `/tmp/webcmd-likes-resume-done-${process.pid}-${Date.now()}.json`; + const outputFile = `/tmp/webcmd-likes-out-done-${process.pid}-${Date.now()}.jsonl`; + const page = { + goto: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + getCookies: vi.fn(async () => [{ name: 'ct0', value: 'token' }]), + evaluate: vi.fn(async (script) => { + const text = String(script); + if (text.includes('AppTabBar_Profile_Link')) { + return { session: 'site:twitter', data: '/viewer' }; + } + if (text.includes('operationName')) return null; + if (text.includes('/UserByScreenName')) { + return { session: 'site:twitter', data: '42' }; + } + if (text.includes('/Likes')) { + return { session: 'site:twitter', data: likesPayload() }; + } + throw new Error(`Unexpected evaluate: ${text.slice(0, 80)}`); + }), + }; + + try { + const result = await command.func(page, { + all: true, + 'max-pages': 1, + 'resume-file': resumeFile, + 'output-file': outputFile, + }); + + expect(result).toMatchObject({ + outputFile, + count: 1, + source: 'likes', + username: 'viewer', + complete: true, + pages: 1, + }); + expect(result.cursor).toBeUndefined(); + expect(fs.existsSync(resumeFile)).toBe(false); + expect(fs.existsSync(outputFile)).toBe(true); + } + finally { + fs.rmSync(resumeFile, { force: true }); + fs.rmSync(outputFile, { force: true }); + } + }); }); diff --git a/plugins/twitter/test/profile.test.js b/plugins/twitter/test/profile.test.js index 9d2ec761..bc243de7 100644 --- a/plugins/twitter/test/profile.test.js +++ b/plugins/twitter/test/profile.test.js @@ -68,6 +68,99 @@ describe('twitter profile command', () => { }); }); + it('reads the current UserByScreenName profile containers after X removed result.legacy', () => { + // Captures the field names and containers observed in a live + // UserByScreenName response. X renamed these keys as well as moving them, + // so searching the tree for the old legacy key names cannot recover them. + const rows = __test__.mapTwitterProfileResult({ + core: { + screen_name: 'relocated_user', + name: 'Relocated User', + created_at: 'Sun Mar 20 00:00:00 +0000 2011', + }, + relationship_counts: { followers: 7100000, following: 42 }, + tweet_counts: { tweets: 128 }, + action_counts: { favorites_count: 9 }, + profile_bio: { description: 'current bio text' }, + location: { location: 'Earth' }, + website: { url: 'https://example.com' }, + verification: { verified: true }, + }, 'fallback'); + + expect(rows[0]).toMatchObject({ + screen_name: 'relocated_user', + name: 'Relocated User', + bio: 'current bio text', + location: 'Earth', + url: 'https://example.com', + followers: 7100000, + following: 42, + tweets: 128, + likes: 9, + verified: true, + created_at: 'Sun Mar 20 00:00:00 +0000 2011', + }); + }); + + it('prefers current profile containers while retaining legacy fallbacks', () => { + const rows = __test__.mapTwitterProfileResult({ + core: { screen_name: 'u', name: 'U', created_at: 'now' }, + legacy: { + description: 'old bio', + followers_count: 100, + friends_count: 10, + statuses_count: 5, + favourites_count: 2, + }, + relationship_counts: { followers: 200, following: 20 }, + tweet_counts: { tweets: 15 }, + action_counts: { favorites_count: 12 }, + profile_bio: { description: 'current bio' }, + }, 'fallback'); + + expect(rows[0]).toMatchObject({ + bio: 'current bio', + followers: 200, + following: 20, + tweets: 15, + likes: 12, + }); + }); + + it('does not read same-named values from unrelated nested entities', () => { + const rows = __test__.mapTwitterProfileResult({ + core: { screen_name: 'u', name: 'U', created_at: 'now' }, + pinned_tweet: { + relationship_counts: { followers: 999999, following: 999999 }, + action_counts: { favorites_count: 999999 }, + profile_bio: { description: 'not a user bio' }, + }, + }, 'fallback'); + + expect(rows[0]).toMatchObject({ + bio: '', + followers: 0, + following: 0, + tweets: 0, + likes: 0, + }); + }); + + it('returns 0 / empty string when a count or bio is absent everywhere', () => { + const rows = __test__.mapTwitterProfileResult({ + core: { screen_name: 'sparse', name: 'Sparse', created_at: 'now' }, + legacy: {}, + }, 'fallback'); + + expect(rows[0]).toMatchObject({ + bio: '', + followers: 0, + following: 0, + tweets: 0, + likes: 0, + }); + }); + it('throws typed when the profile result is structurally malformed', () => { expect(() => __test__.mapTwitterProfileResult(null, 'jack')).toThrow(CommandExecutionError); expect(() => __test__.mapTwitterProfileResult([], 'jack')).toThrow(CommandExecutionError); diff --git a/plugins/twitter/user-timeline.js b/plugins/twitter/user-timeline.js new file mode 100644 index 00000000..88eebe92 --- /dev/null +++ b/plugins/twitter/user-timeline.js @@ -0,0 +1,217 @@ +import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@agentrhq/webcmd/errors'; +import { resolveTwitterOperationMetadata, normalizeTwitterGraphqlPayload, unwrapBrowserResult, normalizeTwitterScreenName } from './shared.js'; +import { TWITTER_BEARER_TOKEN } from './utils.js'; + +const USER_TWEETS_QUERY_ID = 'lrMzG9qPQHpqJdP3AbM-bQ'; +const USER_BY_SCREEN_NAME_QUERY_ID = 'IGgvgiOx4QZndDHuD3x9TQ'; + +export const MAX_USER_TWEETS_PAGES = 100; +export const USER_TWEETS_PAGE_SIZE = 100; +export const MAX_USER_TWEETS_LIMIT = MAX_USER_TWEETS_PAGES * USER_TWEETS_PAGE_SIZE; +export const DEFAULT_USER_TWEETS_PAGE_DELAY_SECONDS = 2; + +const USER_TWEETS_FEATURES = { + rweb_video_screen_enabled: true, + rweb_cashtags_enabled: true, + payments_enabled: false, + profile_label_improvements_pcf_label_in_post_enabled: true, + responsive_web_profile_redirect_enabled: true, + rweb_tipjar_consumption_enabled: true, + verified_phone_label_enabled: false, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + premium_content_api_read_enabled: false, + communities_web_enable_tweet_community_results_fetch: true, + c9s_tweet_anatomy_moderator_badge_enabled: true, + responsive_web_grok_analyze_button_fetch_trends_enabled: false, + responsive_web_grok_analyze_post_followups_enabled: true, + rweb_cashtags_composer_attachment_enabled: true, + responsive_web_jetfuel_frame: true, + responsive_web_grok_share_attachment_enabled: true, + responsive_web_grok_annotations_enabled: true, + articles_preview_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: true, + tweet_awards_web_tipping_enabled: false, + content_disclosure_indicator_enabled: true, + content_disclosure_ai_generated_indicator_enabled: true, + responsive_web_grok_show_grok_translated_post: false, + responsive_web_grok_analysis_button_from_backend: true, + post_ctas_fetch_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_grok_image_annotation_enabled: true, + responsive_web_grok_imagine_annotation_enabled: true, + responsive_web_grok_community_note_auto_translation_is_enabled: false, + responsive_web_enhance_cards_enabled: false, +}; + +const USER_TWEETS_FIELD_TOGGLES = { + withPayments: true, + withAuxiliaryUserLabels: true, + withArticleRichContentState: true, + withArticlePlainText: true, + withArticleSummaryText: true, + withArticleVoiceOver: true, + withGrokAnalyze: true, + withDisallowedReplyControls: true, +}; + +const USER_BY_SCREEN_NAME_FEATURES = { + hidden_profile_subscriptions_enabled: true, + profile_label_improvements_pcf_label_in_post_enabled: true, + responsive_web_profile_redirect_enabled: true, + rweb_tipjar_consumption_enabled: true, + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + subscriptions_verification_info_is_identity_verified_enabled: true, + subscriptions_verification_info_verified_since_enabled: true, + highlights_tweets_tab_ui_enabled: true, + responsive_web_twitter_article_notes_tab_enabled: true, + subscriptions_feature_can_gift_premium: true, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + responsive_web_graphql_timeline_navigation_enabled: true, +}; + +const USER_BY_SCREEN_NAME_FIELD_TOGGLES = { + withPayments: true, + withAuxiliaryUserLabels: true, +}; + +const USER_TWEETS_OPERATION = { + queryId: USER_TWEETS_QUERY_ID, + features: USER_TWEETS_FEATURES, + fieldToggles: USER_TWEETS_FIELD_TOGGLES, +}; + +const USER_BY_SCREEN_NAME_OPERATION = { + queryId: USER_BY_SCREEN_NAME_QUERY_ID, + features: USER_BY_SCREEN_NAME_FEATURES, + fieldToggles: USER_BY_SCREEN_NAME_FIELD_TOGGLES, +}; + +function normalizeUserTweetsOperation(operation) { + if (typeof operation === 'string') { + return { queryId: operation, features: USER_TWEETS_FEATURES, fieldToggles: USER_TWEETS_FIELD_TOGGLES }; + } + return { + queryId: operation?.queryId || USER_TWEETS_QUERY_ID, + features: operation?.features || USER_TWEETS_FEATURES, + fieldToggles: operation?.fieldToggles || USER_TWEETS_FIELD_TOGGLES, + }; +} + +function normalizeUserByScreenNameOperation(operation) { + if (typeof operation === 'string') { + return { queryId: operation, features: USER_BY_SCREEN_NAME_FEATURES, fieldToggles: USER_BY_SCREEN_NAME_FIELD_TOGGLES }; + } + return { + queryId: operation?.queryId || USER_BY_SCREEN_NAME_QUERY_ID, + features: operation?.features || USER_BY_SCREEN_NAME_FEATURES, + fieldToggles: operation?.fieldToggles || USER_BY_SCREEN_NAME_FIELD_TOGGLES, + }; +} + +function appendGraphqlParams(path, variables, operation) { + const fieldToggles = operation.fieldToggles || {}; + const params = [ + `variables=${encodeURIComponent(JSON.stringify(variables))}`, + `features=${encodeURIComponent(JSON.stringify(operation.features || {}))}`, + ]; + if (Object.keys(fieldToggles).length > 0) { + params.push(`fieldToggles=${encodeURIComponent(JSON.stringify(fieldToggles))}`); + } + return `${path}?${params.join('&')}`; +} + +export function buildUserTweetsUrl(operation, userId, count, cursor) { + const normalized = normalizeUserTweetsOperation(operation); + const vars = { + userId, + count, + includePromotedContent: false, + withQuickPromoteEligibilityTweetFields: true, + withVoice: true, + }; + if (cursor) vars.cursor = cursor; + return appendGraphqlParams(`/i/api/graphql/${normalized.queryId}/UserTweets`, vars, normalized); +} + +export function buildUserByScreenNameUrl(operation, screenName) { + const normalized = normalizeUserByScreenNameOperation(operation); + const vars = { screen_name: screenName, withSafetyModeUserFields: true }; + return appendGraphqlParams(`/i/api/graphql/${normalized.queryId}/UserByScreenName`, vars, normalized); +} + +export async function resolveUserTimelineContext( + page, + rawUsername, + { allowLoggedInDefault = false, commandName = 'tweets' } = {}, +) { + const raw = String(rawUsername ?? '').trim(); + let username = normalizeTwitterScreenName(raw); + if (raw && !username) { + throw new ArgumentError( + `twitter ${commandName} username must be a valid Twitter/X handle`, + commandName === 'collection' + ? 'Example: webcmd twitter collection @jack --until 2026-07-23T00:00:00Z' + : 'Example: webcmd twitter tweets @jack --limit 20', + ); + } + if (!username && !allowLoggedInDefault) { + throw new ArgumentError('twitter collection username must be a valid Twitter/X handle', 'Example: webcmd twitter collection @jack --until 2026-07-23T00:00:00Z'); + } + if (!username) { + await page.goto('https://x.com/home'); + await page.wait({ selector: '[data-testid="primaryColumn"]' }); + const href = unwrapBrowserResult(await page.evaluate(`() => { + const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]'); + return link ? link.getAttribute('href') : null; + }`)); + if (!href || typeof href !== 'string') { + throw new AuthRequiredError('x.com', 'Could not detect logged-in user. Are you logged in?'); + } + username = normalizeTwitterScreenName(href); + if (!username) { + throw new AuthRequiredError('x.com', 'Could not detect logged-in user. Are you logged in?'); + } + } + + const cookies = await page.getCookies({ url: 'https://x.com' }); + const ct0 = cookies.find((cookie) => cookie.name === 'ct0')?.value || null; + if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)'); + + const userTweetsOperation = await resolveTwitterOperationMetadata(page, 'UserTweets', USER_TWEETS_OPERATION); + const userByScreenNameOperation = await resolveTwitterOperationMetadata(page, 'UserByScreenName', USER_BY_SCREEN_NAME_OPERATION); + const headers = JSON.stringify({ + Authorization: `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, + 'X-Csrf-Token': ct0, + 'X-Twitter-Auth-Type': 'OAuth2Session', + 'X-Twitter-Active-User': 'yes', + }); + const userByScreenNameUrl = buildUserByScreenNameUrl(userByScreenNameOperation, username); + const userId = unwrapBrowserResult(await page.evaluate(`async () => { + const resp = await fetch(${JSON.stringify(userByScreenNameUrl)}, { headers: ${headers}, credentials: 'include' }); + if (!resp.ok) return null; + const data = await resp.json(); + return data?.data?.user?.result?.rest_id || null; + }`)); + if (!userId) throw new CommandExecutionError(`Could not resolve @${username}`); + return { username, userId, headers, userTweetsOperation }; +} + +export async function fetchUserTimelinePage(page, context, cursor, count) { + const url = buildUserTweetsUrl(context.userTweetsOperation, context.userId, count, cursor); + return normalizeTwitterGraphqlPayload(await page.evaluate(`async () => { + const response = await fetch(${JSON.stringify(url)}, { headers: ${context.headers}, credentials: 'include' }); + return response.ok ? await response.json() : { error: response.status }; + }`)); +} diff --git a/src/browser/cdp.ts b/src/browser/cdp.ts index cbe89a54..5291e2cd 100644 --- a/src/browser/cdp.ts +++ b/src/browser/cdp.ts @@ -401,7 +401,7 @@ class CDPPage extends BasePage { } if (level === 'all') return [...this._consoleMessages]; // 'error' level includes both console.error() and uncaught exceptions - if (level === 'error') return this._consoleMessages.filter(m => m.type === 'error' || m.type === 'warning'); + if (level === 'error') return this._consoleMessages.filter(m => m.type === 'error'); return this._consoleMessages.filter(m => m.type === level); } diff --git a/src/download/article-download.test.ts b/src/download/article-download.test.ts index 264e22e1..0431feeb 100644 --- a/src/download/article-download.test.ts +++ b/src/download/article-download.test.ts @@ -70,6 +70,28 @@ describe('downloadArticle', () => { expect(md).toMatch(/\|\s*1\s*\|\s*2\s*\|/); }); + // A caption is the only child that survives inside a row-less table: the parser + // foster-parents any other content out of the table before turndown sees it. + it('keeps the text of a table that carries no rows instead of failing the download', async () => { + const md = await runAndRead( + '

before

' + + '
orphan cell
' + + '

after

', + ); + expect(md).toContain('orphan cell'); + expect(md).toContain('before'); + expect(md).toContain('after'); + }); + + it('drops an empty table and still converts a real one beside it', async () => { + const md = await runAndRead( + '
' + + '
a
1
', + ); + expect(md).toMatch(/\|\s*a\s*\|/); + expect(md).toMatch(/\|\s*1\s*\|/); + }); + it('converts strikethrough and task lists', async () => { const md = await runAndRead( '

gone

' + diff --git a/src/download/article-download.ts b/src/download/article-download.ts index 3cc6f6cd..1d2dd0e7 100644 --- a/src/download/article-download.ts +++ b/src/download/article-download.ts @@ -110,6 +110,15 @@ function createTurndown( }); td.use(gfm); td.remove(STRIPPED_TAGS); + // turndown-plugin-gfm@1.0.2 reads `table.rows[0].parentNode` from both its + // table rule and its keep filter, so a table carrying no `tr` throws before + // either can decide, taking the whole document with it. Claiming those tables + // here bypasses both: `addRule` unshifts onto the rule array, and `forNode` + // exhausts that array before it consults the keep list. + td.addRule('rowlessTable', { + filter: (node) => node.nodeName === 'TABLE' && !(node as HTMLTableElement).rows?.length, + replacement: (content) => (content.trim() ? `\n\n${content.trim()}\n\n` : ''), + }); // turndown-plugin-gfm@1.0.2 emits single-tilde strikethrough (`~x~`), which // is not the canonical GFM form. Override it so exported markdown is // portable across common renderers. diff --git a/src/output.test.ts b/src/output.test.ts index caeeb43d..ea0f5d57 100644 --- a/src/output.test.ts +++ b/src/output.test.ts @@ -45,11 +45,11 @@ describe('formatOutput', () => { ); }); - it('preserves the legacy literal Markdown table cells', () => { + it('escapes pipe characters in Markdown table cells', () => { expect(formatOutput([ { name: 'a|b', note: 'line 1\nline 2' }, ], { fmt: 'md', fmtExplicit: true, columns: ['name', 'note'], isTTY: false })).toBe( - '| name | note |\n| --- | --- |\n| a|b | line 1\nline 2 |\n', + '| name | note |\n| --- | --- |\n| a\\|b | line 1\nline 2 |\n', ); }); diff --git a/src/output.ts b/src/output.ts index f54deeda..bd2f2776 100644 --- a/src/output.ts +++ b/src/output.ts @@ -162,7 +162,7 @@ function formatMarkdown(data: unknown, opts: RenderOptions): string { const output = [ `| ${columns.join(' | ')} |`, `| ${columns.map(() => '---').join(' | ')} |`, - ...rows.map(row => `| ${columns.map(column => String(row[column] ?? '')).join(' | ')} |`), + ...rows.map(row => `| ${columns.map(column => String(row[column] ?? '').replace(/\|/g, '\\|')).join(' | ')} |`), ]; return `${output.join('\n')}\n`; } From a17d32479e3b1c367aa8413ab19233f8ad7908ee Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Sat, 8 Aug 2026 09:08:15 +0530 Subject: [PATCH 2/3] fix(ci): allow additive args on migrated commands, update pinned output bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plugin parity gate compared `args` against the frozen v0.5.3 manifest with strict deep-equality, so a migrated command could never gain a new optional flag — the check was stricter than the guarantee it protects. It now asserts what actually matters: every frozen argument is still present, invocable identically (name, type, default, required, positional), and in the same relative order, so positional invocations keep working. New arguments may be added around them and help text may be reworded. Every other field is still compared strictly. Also update the hosted markdown parity case, which pinned the pre-escaping bytes, and shrink the typed-error lint baseline by the two entries the likes / bookmarks rewrite resolved. Co-Authored-By: Claude Opus 5 --- scripts/check-plugin-command-parity.mjs | 37 ++++++++++++++++++++++++- scripts/typed-error-lint-baseline.json | 20 ++----------- src/hosted/runner.test.ts | 4 +-- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/scripts/check-plugin-command-parity.mjs b/scripts/check-plugin-command-parity.mjs index 610f2f81..cefb15c1 100644 --- a/scripts/check-plugin-command-parity.mjs +++ b/scripts/check-plugin-command-parity.mjs @@ -10,10 +10,15 @@ const core = read('cli-manifest.json'); const frozen = read('test/fixtures/core-cli-manifest-v0.5.3.json'); const coreKeys = new Set(core.map(key)); const pluginByKey = new Map(plugins.map(entry => [key(entry), entry])); +// `args` is checked separately: a migrated command may still gain new optional +// flags, so strict deep-equality would forbid ordinary feature work rather than +// the regression this guards against. const fields = [ - 'aliases', 'access', 'domain', 'strategy', 'browser', 'args', 'columns', 'tags', 'keywords', + 'aliases', 'access', 'domain', 'strategy', 'browser', 'columns', 'tags', 'keywords', 'defaultFormat', 'pipeline', 'navigateBefore', 'siteSession', 'freshPage', ]; +// Help text documents an argument; it does not change how one is invoked. +const argFields = ['name', 'type', 'default', 'required', 'positional']; const issues = []; for (const expected of frozen) { @@ -27,6 +32,7 @@ for (const expected of frozen) { if (JSON.stringify(pick(actual)) !== JSON.stringify(pick(expected))) { issues.push(`${command} executable metadata differs from frozen core manifest`); } + issues.push(...argIssues(command, actual.args ?? [], expected.args ?? [])); } if (issues.length) { @@ -47,3 +53,32 @@ function key(entry) { function pick(entry) { return Object.fromEntries(fields.map(field => [field, entry[field]])); } + +function pickArg(arg) { + return Object.fromEntries(argFields.map(field => [field, arg[field]])); +} + +// Every frozen argument must still be invocable exactly as it was in v0.5.3: +// present, same type / default / required / positional, and in the same relative +// order so positional invocations keep working. New arguments may be added +// around them, and help text may be reworded. +function argIssues(command, actual, expected) { + const found = []; + const actualByName = new Map(actual.map(arg => [arg.name, arg])); + for (const arg of expected) { + const match = actualByName.get(arg.name); + if (!match) { + found.push(`${command} dropped the frozen argument --${arg.name}`); + continue; + } + if (JSON.stringify(pickArg(match)) !== JSON.stringify(pickArg(arg))) { + found.push(`${command} changed how --${arg.name} is invoked`); + } + } + const frozenOrder = expected.map(arg => arg.name); + const actualOrder = actual.map(arg => arg.name).filter(name => frozenOrder.includes(name)); + if (JSON.stringify(actualOrder) !== JSON.stringify(frozenOrder.filter(n => actualByName.has(n)))) { + found.push(`${command} reordered the frozen arguments`); + } + return found; +} diff --git a/scripts/typed-error-lint-baseline.json b/scripts/typed-error-lint-baseline.json index 5d8ec0f2..81287d69 100644 --- a/scripts/typed-error-lint-baseline.json +++ b/scripts/typed-error-lint-baseline.json @@ -327,14 +327,6 @@ "text": "const fetchCount = Math.min(100, limit - allTweets.length + 10);", "occurrence": 0 }, - { - "rule": "silent-clamp", - "command": "twitter/bookmarks", - "file": "plugins/twitter/bookmarks.js", - "line": 157, - "text": "const fetchCount = Math.min(100, limit - allTweets.length + 10);", - "occurrence": 0 - }, { "rule": "silent-clamp", "command": "twitter/following", @@ -343,14 +335,6 @@ "text": "const fetchCount = Math.min(50, limit - allUsers.length + 10);", "occurrence": 0 }, - { - "rule": "silent-clamp", - "command": "twitter/likes", - "file": "plugins/twitter/likes.js", - "line": 208, - "text": "const fetchCount = Math.min(100, limit - allTweets.length + 10);", - "occurrence": 0 - }, { "rule": "silent-clamp", "command": "twitter/list-tweets", @@ -491,7 +475,7 @@ "rule": "silent-sentinel", "command": "twitter/bookmarks", "file": "plugins/twitter/bookmarks.js", - "line": 55, + "line": 59, "text": "const screenName = user?.legacy?.screen_name || user?.core?.screen_name || 'unknown';", "occurrence": 0 }, @@ -499,7 +483,7 @@ "rule": "silent-sentinel", "command": "twitter/likes", "file": "plugins/twitter/likes.js", - "line": 91, + "line": 95, "text": "const screenName = user?.legacy?.screen_name || user?.core?.screen_name || 'unknown';", "occurrence": 0 }, diff --git a/src/hosted/runner.test.ts b/src/hosted/runner.test.ts index f5823b90..44745841 100644 --- a/src/hosted/runner.test.ts +++ b/src/hosted/runner.test.ts @@ -1356,10 +1356,10 @@ describe('runHostedCli', () => { expected: 'username\n"a,""b\nline 2"\n', }, { - name: 'literal Markdown cells', + name: 'escaped Markdown cells', result: [{ username: 'a|b\nline 2' }], argv: ['-f', 'md'], - expected: '| username |\n| --- |\n| a|b\nline 2 |\n', + expected: '| username |\n| --- |\n| a\\|b\nline 2 |\n', }, ])('renders hosted $name with canonical literal bytes', async ({ result, argv, expected }) => { const stdout = sink(true); From 7efddc38a4d113aa5cf0ac968b3822c8cef89eef Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Sat, 8 Aug 2026 20:48:22 +0530 Subject: [PATCH 3/3] fix(twitter tests): build archive fixture paths with os.tmpdir() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The likes / bookmarks archive tests hardcoded POSIX `/tmp/...` paths. The command resolves whatever it is given, so on Windows those became `D:\tmp\...` — a directory that does not exist. Fixture writes failed with ENOENT, and the assertions compared the raw `/tmp` string against the resolved one. Building the paths from `os.tmpdir()` keeps them absolute and platform-correct, so `path.resolve` is a no-op on them. Co-Authored-By: Claude Opus 5 --- plugins/twitter/test/bookmarks.test.js | 18 ++++++++++-------- plugins/twitter/test/likes.test.js | 18 ++++++++++-------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/plugins/twitter/test/bookmarks.test.js b/plugins/twitter/test/bookmarks.test.js index 1744e6ea..a4d7ddc1 100644 --- a/plugins/twitter/test/bookmarks.test.js +++ b/plugins/twitter/test/bookmarks.test.js @@ -1,4 +1,6 @@ import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { describe, expect, it, vi } from 'vitest'; import { getRegistry } from '@agentrhq/webcmd/registry'; import { __test__ } from '../bookmarks.js'; @@ -258,8 +260,8 @@ function bookmarksPayload(withBottomCursor = false) { describe('twitter bookmarks command', () => { it('keeps resume state and reports complete=false when --max-pages stops early', async () => { const command = getRegistry().get('twitter/bookmarks'); - const resumeFile = `/tmp/webcmd-bookmarks-resume-${process.pid}-${Date.now()}.json`; - const outputFile = `/tmp/webcmd-bookmarks-out-${process.pid}-${Date.now()}.jsonl`; + const resumeFile = path.join(os.tmpdir(), `webcmd-bookmarks-resume-${process.pid}-${Date.now()}.json`); + const outputFile = path.join(os.tmpdir(), `webcmd-bookmarks-out-${process.pid}-${Date.now()}.jsonl`); const page = { getCookies: vi.fn(async () => [{ name: 'ct0', value: 'token' }]), evaluate: vi.fn(async (script) => { @@ -306,8 +308,8 @@ describe('twitter bookmarks command', () => { it('removes resume file only after the bookmarks timeline is exhausted', async () => { const command = getRegistry().get('twitter/bookmarks'); - const resumeFile = `/tmp/webcmd-bookmarks-resume-done-${process.pid}-${Date.now()}.json`; - const outputFile = `/tmp/webcmd-bookmarks-out-done-${process.pid}-${Date.now()}.jsonl`; + const resumeFile = path.join(os.tmpdir(), `webcmd-bookmarks-resume-done-${process.pid}-${Date.now()}.json`); + const outputFile = path.join(os.tmpdir(), `webcmd-bookmarks-out-done-${process.pid}-${Date.now()}.jsonl`); const page = { getCookies: vi.fn(async () => [{ name: 'ct0', value: 'token' }]), evaluate: vi.fn(async (script) => { @@ -378,7 +380,7 @@ describe('twitter bookmarks archive safety', () => { it('refuses to overwrite an existing output file without matching resume state', async () => { const command = getRegistry().get('twitter/bookmarks'); - const outputFile = `/tmp/webcmd-bookmarks-existing-${process.pid}-${Date.now()}.jsonl`; + const outputFile = path.join(os.tmpdir(), `webcmd-bookmarks-existing-${process.pid}-${Date.now()}.jsonl`); const resumeFile = `${outputFile}.resume.json`; fs.writeFileSync(outputFile, 'user-owned\n'); try { @@ -396,7 +398,7 @@ describe('twitter bookmarks archive safety', () => { }); it('rejects cross-source and cross-output resume state', () => { - const resumeFile = `/tmp/webcmd-bookmarks-mismatch-${process.pid}-${Date.now()}.json`; + const resumeFile = path.join(os.tmpdir(), `webcmd-bookmarks-mismatch-${process.pid}-${Date.now()}.json`); try { fs.writeFileSync(resumeFile, JSON.stringify({ cursor: 'NEXT', @@ -429,7 +431,7 @@ describe('twitter bookmarks archive safety', () => { it('rejects output files whose JSONL record count differs from resume state', async () => { const command = getRegistry().get('twitter/bookmarks'); - const outputFile = `/tmp/webcmd-bookmarks-count-mismatch-${process.pid}-${Date.now()}.jsonl`; + const outputFile = path.join(os.tmpdir(), `webcmd-bookmarks-count-mismatch-${process.pid}-${Date.now()}.jsonl`); const resumeFile = `${outputFile}.resume.json`; fs.writeFileSync(outputFile, '{"id":"1"}\n{"id":"2"}\n'); fs.writeFileSync(resumeFile, JSON.stringify({ @@ -454,7 +456,7 @@ describe('twitter bookmarks archive safety', () => { it('throws for an incomplete in-memory --all run while retaining resume state', async () => { const command = getRegistry().get('twitter/bookmarks'); - const resumeFile = `/tmp/webcmd-bookmarks-memory-${process.pid}-${Date.now()}.json`; + const resumeFile = path.join(os.tmpdir(), `webcmd-bookmarks-memory-${process.pid}-${Date.now()}.json`); try { await expect(command.func(pageFor(bookmarksPayload(true)), { all: true, diff --git a/plugins/twitter/test/likes.test.js b/plugins/twitter/test/likes.test.js index 2cf91367..7911b7ea 100644 --- a/plugins/twitter/test/likes.test.js +++ b/plugins/twitter/test/likes.test.js @@ -1,4 +1,6 @@ import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { describe, expect, it, vi } from 'vitest'; import { getRegistry } from '@agentrhq/webcmd/registry'; import { ArgumentError, AuthRequiredError, EmptyResultError } from '@agentrhq/webcmd/errors'; @@ -179,7 +181,7 @@ describe('twitter likes archive safety', () => { it('refuses to overwrite an existing output file without matching resume state', async () => { const command = getRegistry().get('twitter/likes'); - const outputFile = `/tmp/webcmd-likes-existing-${process.pid}-${Date.now()}.jsonl`; + const outputFile = path.join(os.tmpdir(), `webcmd-likes-existing-${process.pid}-${Date.now()}.jsonl`); const resumeFile = `${outputFile}.resume.json`; fs.writeFileSync(outputFile, 'user-owned\n'); try { @@ -198,7 +200,7 @@ describe('twitter likes archive safety', () => { }); it('rejects cross-source and malformed resume state instead of silently restarting', () => { - const resumeFile = `/tmp/webcmd-likes-mismatch-${process.pid}-${Date.now()}.json`; + const resumeFile = path.join(os.tmpdir(), `webcmd-likes-mismatch-${process.pid}-${Date.now()}.json`); try { fs.writeFileSync(resumeFile, JSON.stringify({ cursor: 'NEXT', @@ -224,7 +226,7 @@ describe('twitter likes archive safety', () => { it('rejects output files whose JSONL record count differs from resume state', async () => { const command = getRegistry().get('twitter/likes'); - const outputFile = `/tmp/webcmd-likes-count-mismatch-${process.pid}-${Date.now()}.jsonl`; + const outputFile = path.join(os.tmpdir(), `webcmd-likes-count-mismatch-${process.pid}-${Date.now()}.jsonl`); const resumeFile = `${outputFile}.resume.json`; fs.writeFileSync(outputFile, '{"id":"1"}\n{"id":"2"}\n'); fs.writeFileSync(resumeFile, JSON.stringify({ @@ -251,7 +253,7 @@ describe('twitter likes archive safety', () => { it('throws for an incomplete in-memory --all run while retaining resume state', async () => { const command = getRegistry().get('twitter/likes'); - const resumeFile = `/tmp/webcmd-likes-memory-${process.pid}-${Date.now()}.json`; + const resumeFile = path.join(os.tmpdir(), `webcmd-likes-memory-${process.pid}-${Date.now()}.json`); const payload = likesPayload(); payload.data.user.result.timeline_v2.timeline.instructions[0].entries.push({ entryId: 'cursor-bottom-1', @@ -355,8 +357,8 @@ describe('twitter likes command', () => { it('keeps resume state and reports complete=false when --max-pages stops early', async () => { const command = getRegistry().get('twitter/likes'); - const resumeFile = `/tmp/webcmd-likes-resume-${process.pid}-${Date.now()}.json`; - const outputFile = `/tmp/webcmd-likes-out-${process.pid}-${Date.now()}.jsonl`; + const resumeFile = path.join(os.tmpdir(), `webcmd-likes-resume-${process.pid}-${Date.now()}.json`); + const outputFile = path.join(os.tmpdir(), `webcmd-likes-out-${process.pid}-${Date.now()}.jsonl`); const page = { goto: vi.fn().mockResolvedValue(undefined), wait: vi.fn().mockResolvedValue(undefined), @@ -425,8 +427,8 @@ describe('twitter likes command', () => { it('removes resume file only after the likes timeline is exhausted', async () => { const command = getRegistry().get('twitter/likes'); - const resumeFile = `/tmp/webcmd-likes-resume-done-${process.pid}-${Date.now()}.json`; - const outputFile = `/tmp/webcmd-likes-out-done-${process.pid}-${Date.now()}.jsonl`; + const resumeFile = path.join(os.tmpdir(), `webcmd-likes-resume-done-${process.pid}-${Date.now()}.json`); + const outputFile = path.join(os.tmpdir(), `webcmd-likes-out-done-${process.pid}-${Date.now()}.jsonl`); const page = { goto: vi.fn().mockResolvedValue(undefined), wait: vi.fn().mockResolvedValue(undefined),