diff --git a/plugins/amazon/README.md b/plugins/amazon/README.md index cb2f028c..eddebe68 100644 --- a/plugins/amazon/README.md +++ b/plugins/amazon/README.md @@ -21,3 +21,7 @@ webcmd plugin install github:agentrhq/webcmd/amazon | `webcmd amazon product` | Amazon product page facts for candidate validation | | `webcmd amazon search` | Amazon search results for product discovery and coarse filtering | | `webcmd amazon whoami` | Show the current logged-in amazon account | + +## Notes + +- A product or review URL from a sibling marketplace (`amazon.co.uk`, `amazon.de`, `amazon.com.au`, …) is read on that marketplace, and the emitted URLs stay on it. A bare ASIN still defaults to `amazon.com`. diff --git a/plugins/amazon/discussion.js b/plugins/amazon/discussion.js index 45a81f95..de2b00f9 100644 --- a/plugins/amazon/discussion.js +++ b/plugins/amazon/discussion.js @@ -1,6 +1,6 @@ import { AuthRequiredError, CommandExecutionError } from '@agentrhq/webcmd/errors'; import { cli, Strategy } from '@agentrhq/webcmd/registry'; -import { buildProductUrl, buildDiscussionUrl, buildProvenance, cleanText, extractAsin, normalizeProductUrl, parseRatingValue, parseReviewCount, trimRatingPrefix, uniqueNonEmpty, assertUsableState, gotoAndReadState, } from './shared.js'; +import { DOMAIN, amazonHostFromInput, buildProductUrl, buildDiscussionUrl, buildProvenance, cleanText, extractAsin, normalizeProductUrl, parseRatingValue, parseReviewCount, trimRatingPrefix, uniqueNonEmpty, assertUsableState, gotoAndReadState, } from './shared.js'; function normalizeDiscussionPayload(payload) { const sourceUrl = cleanText(payload.href) || buildDiscussionUrl(payload.href ?? ''); const asin = extractAsin(payload.href ?? '') ?? null; @@ -9,7 +9,7 @@ function normalizeDiscussionPayload(payload) { const provenance = buildProvenance(sourceUrl); return { asin, - product_url: asin ? normalizeProductUrl(asin) : null, + product_url: asin ? normalizeProductUrl(sourceUrl) : null, discussion_url: sourceUrl, ...provenance, average_rating_text: averageRatingText, @@ -71,7 +71,7 @@ async function readDiscussionPayload(page, input, limit) { const productState = await gotoAndReadState(page, productUrl, 2500, 'discussion'); assertUsableState(productState, 'discussion'); if (isSignInState(reviewState) && isSignInState(productState)) { - throw new AuthRequiredError('amazon.com', 'Amazon review discussion requires an active signed-in Amazon session in the shared Chrome profile.'); + throw new AuthRequiredError(amazonHostFromInput(input) ?? DOMAIN, 'Amazon review discussion requires an active signed-in Amazon session in the shared Chrome profile.'); } const productPayload = await readCurrentDiscussionPayload(page, limit); if (hasDiscussionSummary(productPayload)) { @@ -111,7 +111,8 @@ cli({ const payload = await readDiscussionPayload(page, input, limit); const normalized = normalizeDiscussionPayload(payload); if (!normalized.average_rating_text && !normalized.total_review_count_text) { - throw new CommandExecutionError('amazon discussion page did not expose review summary', 'The review page may have changed or hit a robot check. Open the review page in Chrome and retry.'); + const landedUrl = cleanText(payload.href) || buildDiscussionUrl(input); + throw new CommandExecutionError(`amazon discussion page did not expose review summary (landed on ${landedUrl})`, 'The review page may have changed or hit a robot check. Open the review page in Chrome and retry.'); } return [normalized]; }, diff --git a/plugins/amazon/product.js b/plugins/amazon/product.js index 845a7a44..58313452 100644 --- a/plugins/amazon/product.js +++ b/plugins/amazon/product.js @@ -83,7 +83,8 @@ cli({ const input = String(kwargs.input ?? ''); const payload = await readProductPayload(page, input); if (!cleanText(payload.product_title)) { - throw new CommandExecutionError('amazon product page did not expose product content', 'The product page may have changed or hit a robot check. Open the product page in Chrome and retry.'); + const landedUrl = cleanText(payload.href) || buildProductUrl(input); + throw new CommandExecutionError(`amazon product page did not expose product content (landed on ${landedUrl})`, 'The product page may have changed or hit a robot check. Open the product page in Chrome and retry.'); } return [normalizeProductPayload(payload)]; }, diff --git a/plugins/amazon/shared.js b/plugins/amazon/shared.js index 518ea99d..d5b81880 100644 --- a/plugins/amazon/shared.js +++ b/plugins/amazon/shared.js @@ -20,6 +20,42 @@ export const PRIMARY_PRICE_SELECTORS = [ '#priceblock_dealprice', '#tp_price_block_total_price_ww', ]; +// Keep this explicit because these hosts are navigation targets in the user's +// signed-in browser. A shape-only `amazon.` pattern also accepts unrelated +// registrable domains such as amazon.shop or amazon.zip. +const MARKETPLACE_DOMAINS = new Set([ + 'amazon.com', + 'amazon.ca', + 'amazon.com.mx', + 'amazon.com.br', + 'amazon.co.uk', + 'amazon.de', + 'amazon.fr', + 'amazon.it', + 'amazon.es', + 'amazon.nl', + 'amazon.pl', + 'amazon.se', + 'amazon.com.be', + 'amazon.ie', + 'amazon.com.tr', + 'amazon.ae', + 'amazon.sa', + 'amazon.eg', + 'amazon.co.za', + 'amazon.in', + 'amazon.co.jp', + 'amazon.com.au', + 'amazon.sg', +]); +function isAmazonMarketplaceHost(hostname) { + const normalized = cleanText(hostname).toLowerCase().replace(/\.$/, ''); + for (const domain of MARKETPLACE_DOMAINS) { + if (normalized === domain || normalized.endsWith(`.${domain}`)) + return true; + } + return false; +} const ROBOT_TEXT_PATTERNS = [ 'Sorry, we just need to make sure you\'re not a robot', 'Enter the characters you see below', @@ -91,19 +127,33 @@ export function extractAsin(input) { const match = normalized.match(/\/(?:dp|gp\/product|product-reviews)\/([A-Z0-9]{10})/i); return match ? match[1].toUpperCase() : null; } +export function amazonHostFromInput(input) { + const normalized = cleanText(input); + if (!normalized) + return null; + try { + const url = new URL(normalized); + return isAmazonMarketplaceHost(url.hostname) ? url.hostname : null; + } + catch { + return null; + } +} export function buildProductUrl(input) { const asin = extractAsin(input); if (!asin) { throw new ArgumentError('amazon product expects an ASIN or product URL', 'Example: webcmd amazon product B0FJS72893'); } - return `${PRODUCT_URL_PREFIX}${asin}`; + const host = amazonHostFromInput(input); + return host ? `https://${host}/dp/${asin}` : `${PRODUCT_URL_PREFIX}${asin}`; } export function buildDiscussionUrl(input) { const asin = extractAsin(input); if (!asin) { throw new ArgumentError('amazon discussion expects an ASIN or product URL', 'Example: webcmd amazon discussion B0FJS72893'); } - return `${DISCUSSION_URL_PREFIX}${asin}`; + const host = amazonHostFromInput(input); + return host ? `https://${host}/product-reviews/${asin}` : `${DISCUSSION_URL_PREFIX}${asin}`; } function getRankingSpec(listType) { return AMAZON_RANKING_SPECS[listType]; @@ -206,7 +256,7 @@ export function resolveBestsellersUrl(input) { export function canonicalizeAmazonUrl(input) { try { const url = new URL(input); - if (!url.hostname.endsWith(DOMAIN)) { + if (!isAmazonMarketplaceHost(url.hostname)) { throw new Error('not-amazon'); } return url.toString(); @@ -230,7 +280,7 @@ export function normalizeProductUrl(value) { const normalized = cleanText(value); const asin = extractAsin(normalized); if (asin) - return buildProductUrl(asin); + return buildProductUrl(normalized); return toAbsoluteAmazonUrl(normalized); } export function parsePriceText(text) { @@ -347,8 +397,11 @@ export function assertUsableState(state, action) { export const __test__ = { buildSearchUrl, extractAsin, + amazonHostFromInput, buildProductUrl, buildDiscussionUrl, + normalizeProductUrl, + canonicalizeAmazonUrl, resolveBestsellersUrl, resolveRankingUrl, isSupportedRankingPath, diff --git a/plugins/amazon/test/discussion.test.js b/plugins/amazon/test/discussion.test.js index 050483de..c4522dc0 100644 --- a/plugins/amazon/test/discussion.test.js +++ b/plugins/amazon/test/discussion.test.js @@ -41,6 +41,68 @@ describe('amazon discussion normalization', () => { ]); }); + it('keeps the review marketplace in every emitted url', () => { + const result = __test__.normalizeDiscussionPayload({ + href: 'https://www.amazon.co.uk/product-reviews/B0FGCPFY9L', + average_rating_text: '4.4 out of 5', + total_review_count_text: '40 global ratings', + qa_links: [], + review_samples: [], + }); + + expect(result.asin).toBe('B0FGCPFY9L'); + expect(result.discussion_url).toBe('https://www.amazon.co.uk/product-reviews/B0FGCPFY9L'); + expect(result.product_url).toBe('https://www.amazon.co.uk/dp/B0FGCPFY9L'); + }); + + it('requests the review page on the marketplace the input names', async () => { + const command = getRegistry().get('amazon/discussion'); + const page = createPageMock([ + { + href: 'https://www.amazon.co.uk/product-reviews/B0FGCPFY9L', + title: 'Amazon.co.uk: Example product', + body_text: 'Customer reviews', + }, + { + href: 'https://www.amazon.co.uk/product-reviews/B0FGCPFY9L', + average_rating_text: '4.4 out of 5', + total_review_count_text: '40 global ratings', + review_samples: [], + }, + ]); + + await command.func(page, { input: 'https://www.amazon.co.uk/product-reviews/B0FGCPFY9L', limit: 1 }); + + expect(page.goto.mock.calls[0][0]).toBe('https://www.amazon.co.uk/product-reviews/B0FGCPFY9L'); + }); + + it('names the loaded url when neither page exposes a review summary', async () => { + const command = getRegistry().get('amazon/discussion'); + const emptyPayload = { href: 'https://www.amazon.co.uk/dp/B0FGCPFY9L', average_rating_text: '', total_review_count_text: '', review_samples: [] }; + const page = createPageMock([ + { href: 'https://www.amazon.co.uk/product-reviews/B0FGCPFY9L', title: 'Amazon.co.uk', body_text: 'Customer reviews' }, + emptyPayload, + { href: 'https://www.amazon.co.uk/dp/B0FGCPFY9L', title: 'Amazon.co.uk', body_text: 'Product' }, + emptyPayload, + ]); + + await expect(command.func(page, { input: 'https://www.amazon.co.uk/dp/B0FGCPFY9L', limit: 1 })) + .rejects.toThrow('landed on https://www.amazon.co.uk/dp/B0FGCPFY9L'); + }); + + it('points a gated non-US review page at that marketplace, not the US store', async () => { + const command = getRegistry().get('amazon/discussion'); + const signIn = { href: 'https://www.amazon.co.uk/ap/signin', title: 'Amazon Sign-In', body_text: 'Sign in Create account' }; + const page = createPageMock([ + signIn, + { href: signIn.href, average_rating_text: '', total_review_count_text: '', review_samples: [] }, + signIn, + ]); + + await expect(command.func(page, { input: 'https://www.amazon.co.uk/dp/B0FGCPFY9L', limit: 1 })) + .rejects.toMatchObject({ domain: 'www.amazon.co.uk' }); + }); + it('falls back to the product page when the review page redirects to sign-in', async () => { const command = getRegistry().get('amazon/discussion'); const page = createPageMock([ diff --git a/plugins/amazon/test/shared.test.js b/plugins/amazon/test/shared.test.js index ec040cb9..628da2f8 100644 --- a/plugins/amazon/test/shared.test.js +++ b/plugins/amazon/test/shared.test.js @@ -6,6 +6,35 @@ describe('amazon shared helpers', () => { expect(__test__.buildProductUrl('https://www.amazon.com/dp/B0FJS72893/ref=something')).toBe('https://www.amazon.com/dp/B0FJS72893'); expect(__test__.buildDiscussionUrl('https://www.amazon.com/dp/B0FJS72893')).toBe('https://www.amazon.com/product-reviews/B0FJS72893'); }); + it('keeps the input marketplace instead of rewriting it to the US store', () => { + expect(__test__.buildProductUrl('https://www.amazon.co.uk/dp/B0FGCPFY9L')).toBe('https://www.amazon.co.uk/dp/B0FGCPFY9L'); + expect(__test__.buildProductUrl('https://www.amazon.de/dp/B0FJS72893/ref=something')).toBe('https://www.amazon.de/dp/B0FJS72893'); + expect(__test__.buildProductUrl('https://www.amazon.com.au/dp/B0FJS72893')).toBe('https://www.amazon.com.au/dp/B0FJS72893'); + expect(__test__.buildDiscussionUrl('https://www.amazon.co.uk/product-reviews/B0FGCPFY9L?pageNumber=1')).toBe('https://www.amazon.co.uk/product-reviews/B0FGCPFY9L'); + expect(__test__.normalizeProductUrl('https://www.amazon.co.uk/dp/B0FGCPFY9L')).toBe('https://www.amazon.co.uk/dp/B0FGCPFY9L'); + }); + it('defaults to the US store for bare ASINs and non-marketplace hosts', () => { + expect(__test__.amazonHostFromInput('B0FJS72893')).toBeNull(); + expect(__test__.buildProductUrl('B0FJS72893')).toBe('https://www.amazon.com/dp/B0FJS72893'); + expect(__test__.buildDiscussionUrl('B0FJS72893')).toBe('https://www.amazon.com/product-reviews/B0FJS72893'); + expect(__test__.normalizeProductUrl('B0FJS72893')).toBe('https://www.amazon.com/dp/B0FJS72893'); + }); + it('accepts sibling marketplaces but rejects look-alike hosts', () => { + expect(__test__.amazonHostFromInput('https://www.amazon.co.uk/dp/B0FJS72893')).toBe('www.amazon.co.uk'); + expect(__test__.amazonHostFromInput('https://amazon.de/dp/B0FJS72893')).toBe('amazon.de'); + expect(__test__.amazonHostFromInput('https://amazon.com.au/dp/B0FJS72893')).toBe('amazon.com.au'); + expect(__test__.amazonHostFromInput('https://smile.amazon.com.be/dp/B0FJS72893')).toBe('smile.amazon.com.be'); + expect(__test__.amazonHostFromInput('https://evilamazon.com/dp/B0FJS72893')).toBeNull(); + expect(__test__.amazonHostFromInput('https://amazon.com.evil.com/dp/B0FJS72893')).toBeNull(); + expect(__test__.amazonHostFromInput('https://amazon.evil.com/dp/B0FJS72893')).toBeNull(); + expect(__test__.amazonHostFromInput('https://x.amazon.evil.com/dp/B0FJS72893')).toBeNull(); + expect(__test__.amazonHostFromInput('https://amazon.attacker.io/dp/B0FJS72893')).toBeNull(); + expect(__test__.amazonHostFromInput('https://amazon.shop/dp/B0FJS72893')).toBeNull(); + expect(__test__.amazonHostFromInput('https://amazon.zip/dp/B0FJS72893')).toBeNull(); + expect(() => __test__.canonicalizeAmazonUrl('https://amazon.evil.com/gp/bestsellers')).toThrow('Invalid Amazon URL'); + expect(__test__.canonicalizeAmazonUrl('https://www.amazon.co.uk/gp/bestsellers/books')).toBe('https://www.amazon.co.uk/gp/bestsellers/books'); + expect(() => __test__.canonicalizeAmazonUrl('https://evilamazon.com/gp/bestsellers')).toThrow('Invalid Amazon URL'); + }); it('parses price, rating, and review-count text', () => { expect(__test__.parsePriceText('1 offer from $34.11')).toEqual({ price_text: '$34.11', diff --git a/plugins/facebook/feed.js b/plugins/facebook/feed.js index d91357a7..59175045 100644 --- a/plugins/facebook/feed.js +++ b/plugins/facebook/feed.js @@ -306,12 +306,35 @@ async function loadFeedPosts(page, limit) { + Array.from(document.querySelectorAll('[aria-label]')).filter((el) => /^(Actions for this post)$/i.test((el.getAttribute('aria-label') || '').trim())).length : 0; })()`; - let prev = -1; + const extractStep = buildFeedExtractScript(limit); + let prevMarkerCount = -1; + let prevRowCount = -1; + let stalledPasses = 0; for (let i = 0; i < 8; i += 1) { - let count = 0; - try { count = Number(unwrapBrowserResult(await page.evaluate(scrollStep))) || 0; } catch { break; } - if (count >= limit || count === prev) break; - prev = count; + let markerCount = 0; + try { markerCount = Number(unwrapBrowserResult(await page.evaluate(scrollStep))) || 0; } catch { break; } + + // Raw article/menu counts include comments, suggestions, and other chrome. + // Do not stop merely because those markers reached --limit; stop + // when the actual feed extractor has enough valid rows. + let rowCount = 0; + try { + const payload = unwrapBrowserResult(await page.evaluate(extractStep)); + rowCount = Array.isArray(payload && payload.rows) ? payload.rows.length : 0; + } catch { + // The final extraction below owns error classification. A transient + // observation failure here should only make the bounded scroll continue. + } + if (rowCount >= limit) break; + + if (markerCount === prevMarkerCount && rowCount === prevRowCount) { + stalledPasses += 1; + if (stalledPasses >= 2) break; + } else { + stalledPasses = 0; + } + prevMarkerCount = markerCount; + prevRowCount = rowCount; } } @@ -387,5 +410,6 @@ export const __test__ = { buildFeedExtractScript, command, getFacebookFeed, + loadFeedPosts, requireLimit, }; diff --git a/plugins/facebook/profile.js b/plugins/facebook/profile.js index 14b065d0..3c5eb2f0 100644 --- a/plugins/facebook/profile.js +++ b/plugins/facebook/profile.js @@ -17,19 +17,48 @@ cli({ pipeline: [ { navigate: { url: 'https://www.facebook.com/${{ args.username }}', settleMs: 3000 } }, { evaluate: `(() => { - const h1 = document.querySelector('h1'); - let name = h1 ? h1.textContent.trim() : ''; + const username = \${{ args.username | json }}; + const main = document.querySelector('[role="main"]') || document; + const clean = (value) => String(value || '').replace(/\\s+/g, ' ').trim(); + const h1 = main.querySelector('h1'); + let name = clean(h1 && h1.textContent); - // Find friends/followers links - const links = Array.from(document.querySelectorAll('a')); - const friendsLink = links.find(a => a.href && a.href.includes('/friends')); - const followersLink = links.find(a => a.href && a.href.includes('/followers')); + // Facebook's current profile header no longer uses an h1. The profile-avatar + // link carries the display name on both the link and its role=img child. + // Scope this fallback to the requested profile path so page chrome cannot win. + if (!name) { + const requestedPrefix = '/' + String(username).toLowerCase() + '/'; + const profileMediaLink = Array.from(main.querySelectorAll('a[aria-label][href]')).find((link) => { + const href = link.getAttribute('href') || ''; + let path = ''; + try { path = new URL(href, window.location.href).pathname.toLowerCase(); } catch {} + const image = link.querySelector('[role="img"][aria-label]'); + return path.startsWith(requestedPrefix) + && image + && clean(link.getAttribute('aria-label')) === clean(image.getAttribute('aria-label')); + }); + name = clean(profileMediaLink && profileMediaLink.getAttribute('aria-label')); + } + + // Scope relationship links to the profile main region. Global navigation also + // contains /friends and previously produced empty or generic chrome text. + const links = Array.from(main.querySelectorAll('a[href]')); + const hasProfilePath = (link, suffix) => { + try { + const path = new URL(link.getAttribute('href') || '', window.location.href).pathname + .replace(/\\/+$/, '') + .toLowerCase(); + return path === '/' + String(username).toLowerCase() + suffix; + } catch { return false; } + }; + const friendsLink = links.find((a) => hasProfilePath(a, '/friends')); + const followersLink = links.find((a) => hasProfilePath(a, '/followers')); return [{ name: name, - username: \${{ args.username | json }}, - friends: friendsLink ? friendsLink.textContent.trim() : '-', - followers: followersLink ? followersLink.textContent.trim() : '-', + username, + friends: friendsLink ? clean(friendsLink.textContent) : '-', + followers: followersLink ? clean(followersLink.textContent) : '-', url: window.location.href, }]; })() diff --git a/plugins/facebook/search.js b/plugins/facebook/search.js index 0a01c107..42ee83f9 100644 --- a/plugins/facebook/search.js +++ b/plugins/facebook/search.js @@ -54,11 +54,34 @@ function buildSearchExtractScript(limit) { try { u = new URL(href, 'https://www.facebook.com'); } catch (e) { return false; } // drop hidden-domain .com spam — real results stay on facebook.com if (!/(^|\\.)facebook\\.com$/i.test(u.hostname)) return false; + // l.facebook.com / lm.facebook.com only wrap outbound links (/l.php?u=…); + // they are redirect shims, never search entities. + if (/^lm?\\.facebook\\.com$/i.test(u.hostname)) return false; const p = u.pathname; if (/^\\/search(\\/|$)/i.test(p)) return false; // decoy links back to search (incl. bare /search) // chrome / non-result destinations that the catch-all below would keep if (/^\\/(login|checkpoint|help|policies|privacy|settings|bookmarks|messages|notifications|marketplace|gaming|friends|requests|saved|me)\\b/i.test(p)) return false; - return /^\\/(profile\\.php|groups\\/|events\\/|watch\\/|reel\\/|pages\\/|permalink\\.php|story\\.php|[^/]+\\/posts\\/|[^/]+\\/videos\\/|[A-Za-z0-9.\\-]{2,}\\/?$)/i.test(p); + return /^\\/(profile\\.php|photo\\.php|groups\\/|events\\/|watch\\/|reel\\/|pages\\/|permalink\\.php|story\\.php|[^/]+\\/posts\\/|[^/]+\\/videos\\/|[A-Za-z0-9.\\-]{2,}\\/?$)/i.test(p); + } + + // Query-identity destinations (permalink.php?story_fbid=…, story.php, + // photo.php?fbid=…, watch/?v=…) collapse into a single row when the query is + // dropped — different posts/videos share the same pathname. Keep the identity + // params, but only those: FB appends per-render tracking nonces (__cft__, + // __tn__, ref) that would otherwise defeat dedup by making the same post look + // unique on every render. + function entityKey(u) { + const p = u.pathname.toLowerCase(); + let identityParams = []; + if (p === '/profile.php') identityParams = ['id']; + else if (p === '/permalink.php' || p === '/story.php') { + identityParams = ['story_fbid', 'story_id', 'fbid', 'id']; + } else if (p === '/photo.php') identityParams = ['fbid', 'id']; + else if (p === '/watch' || p === '/watch/') identityParams = ['v']; + const ids = identityParams + .filter((k) => u.searchParams.has(k)) + .map((k) => k + '=' + u.searchParams.get(k)); + return ids.length ? (u.origin + u.pathname + '?' + ids.join('&')) : (u.origin + u.pathname); } function isAuthPage() { @@ -81,8 +104,8 @@ function buildSearchExtractScript(limit) { const rawHref = a.href || a.getAttribute('href') || ''; if (!isEntityHref(rawHref)) continue; let key; - try { const u = new URL(rawHref, 'https://www.facebook.com'); key = u.origin + u.pathname; } - catch (e) { key = rawHref.split('?')[0].split('#')[0]; } + try { const u = new URL(rawHref, 'https://www.facebook.com'); key = entityKey(u); } + catch (e) { key = rawHref.split('#')[0]; } if (seen.has(key)) continue; const title = clean(a.textContent).substring(0, 80); diff --git a/plugins/facebook/test/feed.test.js b/plugins/facebook/test/feed.test.js index e274e44b..30a3d33f 100644 --- a/plugins/facebook/test/feed.test.js +++ b/plugins/facebook/test/feed.test.js @@ -156,6 +156,24 @@ describe('facebook feed', () => { }]); }); + it('keeps scrolling when raw article markers reach the limit but valid rows do not', async () => { + const page = { + evaluate: vi.fn() + .mockResolvedValueOnce(4) + .mockResolvedValueOnce({ status: 'no_rows', rows: [] }) + .mockResolvedValueOnce(5) + .mockResolvedValueOnce({ + status: 'ok', + rows: [{ index: 1, author: 'A', content: 'Body', likes: '-', comments: '-', shares: '-' }], + }), + }; + + await __test__.loadFeedPosts(page, 1); + + expect(page.evaluate).toHaveBeenCalledTimes(4); + expect(String(page.evaluate.mock.calls[1][0])).toContain('primaryContainers'); + }); + it('maps auth, real empty, parser drift, and malformed payloads to typed errors', async () => { await expect(__test__.command.func(createPage({ status: 'auth', rows: [] }), { limit: 1 })) .rejects.toBeInstanceOf(AuthRequiredError); diff --git a/plugins/facebook/test/profile.test.js b/plugins/facebook/test/profile.test.js new file mode 100644 index 00000000..39b28000 --- /dev/null +++ b/plugins/facebook/test/profile.test.js @@ -0,0 +1,81 @@ +import { describe, expect, it } from 'vitest'; +import { JSDOM } from 'jsdom'; +import { getRegistry } from '@agentrhq/webcmd/registry'; +import '../profile.js'; + +function runExtract(html, username = 'zuck') { + const dom = new JSDOM(html, { url: `https://www.facebook.com/${username}` }); + const command = getRegistry().get('facebook/profile'); + const script = command.pipeline.find((step) => step.evaluate).evaluate + .replace('${{ args.username | json }}', JSON.stringify(username)); + return Function('window', 'document', `return ${script};`)(dom.window, dom.window.document); +} + +describe('facebook profile', () => { + it('keeps the existing profile row contract', () => { + const command = getRegistry().get('facebook/profile'); + expect(command).toBeDefined(); + expect(command.columns).toEqual(['name', 'username', 'friends', 'followers', 'url']); + }); + + it('extracts the display name from the current profile-avatar header', () => { + const rows = runExtract(` + +
+ + + + + + +
Mark Zuckerberg
+ 1.2M followers + 1,234 friends +
+ `); + + expect(rows).toEqual([{ + name: 'Mark Zuckerberg', + username: 'zuck', + friends: '1,234 friends', + followers: '1.2M followers', + url: 'https://www.facebook.com/zuck', + }]); + }); + + it('prefers an h1 when Facebook still renders the legacy header', () => { + const rows = runExtract(` +
+

Legacy Profile Name

+ + + +
+ `, 'legacy'); + + expect(rows[0]).toMatchObject({ + name: 'Legacy Profile Name', + username: 'legacy', + friends: '-', + followers: '-', + }); + }); + + it('does not take a display name or friends label from global page chrome', () => { + const rows = runExtract(` + +
+ `); + + expect(rows[0]).toMatchObject({ + name: '', + friends: '-', + followers: '-', + }); + }); +}); diff --git a/plugins/facebook/test/search.test.js b/plugins/facebook/test/search.test.js index 53145b85..89631081 100644 --- a/plugins/facebook/test/search.test.js +++ b/plugins/facebook/test/search.test.js @@ -80,6 +80,76 @@ describe('facebook search', () => { expect(rows[0].url).toBe('https://www.facebook.com/carol.page'); }); + it('keeps distinct permalink.php posts apart by their identity query', async () => { + // permalink.php / story.php / watch encode identity in the query string; + // deduping on pathname alone collapses different posts into one row. + const page = createDomPage(` +
+
First distinct post about AI research
+
Second distinct post about AI safety
+
Video one about AI agents here
+
Video two about AI agents here
+
+ `); + + const rows = await searchCommand().func(page, { query: 'ai', limit: 10 }); + expect(rows.map((row) => row.url)).toEqual([ + 'https://www.facebook.com/permalink.php?story_fbid=1001&id=50', + 'https://www.facebook.com/permalink.php?story_fbid=2002&id=50', + 'https://www.facebook.com/watch/?v=111', + 'https://www.facebook.com/watch/?v=222', + ]); + }); + + it('keeps profile and photo identities without treating arbitrary query params as identity', async () => { + const page = createDomPage(` +
+
First profile result with details
+
Second profile result with details
+
Photo result with useful details
+
Same vanity page first render
+
Same vanity page second render
+
+ `); + + const rows = await searchCommand().func(page, { query: 'ai', limit: 10 }); + expect(rows.map((row) => row.url)).toEqual([ + 'https://www.facebook.com/profile.php?id=1001', + 'https://www.facebook.com/profile.php?id=2002', + 'https://www.facebook.com/photo.php?fbid=3003&id=1001', + 'https://www.facebook.com/realpage', + ]); + }); + + it('dedupes one post rendered with different per-render tracking nonces', async () => { + // FB appends __cft__ / __tn__ nonces that differ on every render; keeping + // them in the key would make the same post appear as multiple rows. + const page = createDomPage(` +
+
Same post rendered once here now
+
Same post rendered twice here now
+
+ `); + + const rows = await searchCommand().func(page, { query: 'ai', limit: 10 }); + expect(rows.map((row) => row.url)).toEqual(['https://www.facebook.com/permalink.php?story_fbid=1001&id=50']); + }); + + it('drops l.facebook.com / lm.facebook.com outbound-redirect shims', async () => { + // /l.php?u=… wrappers are external-link redirects, not search entities; their + // pathname would otherwise slip through the vanity catch-all. + const page = createDomPage(` +
+
Real Page result here
+ External article link in a post + Another external redirect link here +
+ `); + + const rows = await searchCommand().func(page, { query: 'ai', limit: 10 }); + expect(rows.map((row) => row.url)).toEqual(['https://www.facebook.com/realpage']); + }); + it('validates query and limit before navigation', async () => { const page = createPage({ status: 'ok', rows: [] }); await expect(searchCommand().func(page, { query: ' ', limit: 3 })).rejects.toBeInstanceOf(ArgumentError); diff --git a/plugins/tiktok/explore.js b/plugins/tiktok/explore.js index ff000cb6..262fb32b 100644 --- a/plugins/tiktok/explore.js +++ b/plugins/tiktok/explore.js @@ -60,32 +60,54 @@ function buildExploreScript(limit) { if (row && !dedup.has(row.id)) dedup.set(row.id, row); } - const msToken = getCookie('msToken'); - let apiFailure = null; - if (dedup.size < limit) { - let cursor = 0; + async function collectEndpoint(path, label, extraParams, initialCursor) { + let cursor = initialCursor; for (let page = 0; page < maxPages && dedup.size < limit; page += 1) { const params = new URLSearchParams({ aid, count: String(pageSize), - from_page: 'fyp', - cursor: String(cursor), + ...extraParams, }); + if (cursor !== undefined) params.set('cursor', String(cursor)); if (msToken) params.set('msToken', msToken); - try { - const data = await fetchJson('/api/recommend/item_list/?' + params.toString()); + const data = await fetchJson(path + '?' + params.toString()); + if (label === 'explore') { + assertTikTokApiSuccess(data, 'explore'); + } else { assertTikTokApiSuccess(data, 'recommend'); - const items = Array.isArray(data.itemList) ? data.itemList : []; - for (const item of items) { - const row = normalizeVideoItem(item, dedup.size + 1); - if (row && !dedup.has(row.id)) dedup.set(row.id, row); - } - if (data.hasMore !== true && items.length === 0) break; - cursor = asNumber(data.cursor) ?? cursor + items.length; - } catch (error) { - apiFailure = error instanceof Error ? error.message : String(error); - break; } + const items = Array.isArray(data.itemList) + ? data.itemList + : (Array.isArray(data.items) ? data.items : []); + for (const item of items) { + const row = normalizeVideoItem(item, dedup.size + 1); + if (row && !dedup.has(row.id)) dedup.set(row.id, row); + } + const nextCursor = asNumber(data.cursor); + const hasMore = data.hasMore === true || data.has_more === true; + if (!hasMore || items.length === 0 || (cursor !== undefined && nextCursor === cursor)) break; + cursor = nextCursor ?? ((cursor ?? 0) + items.length); + } + } + + const msToken = getCookie('msToken'); + const apiFailures = []; + if (dedup.size < limit) { + try { + // TikTok's current /explore page uses this endpoint. categoryType=120 + // is the "All" tab shown by default. + await collectEndpoint('/api/explore/item_list/', 'explore', { categoryType: '120' }); + } catch (error) { + apiFailures.push(error instanceof Error ? error.message : String(error)); + } + } + if (dedup.size < limit) { + try { + // Keep the previous recommend feed as a compatibility fallback for + // regions where /explore still hydrates from the For You endpoint. + await collectEndpoint('/api/recommend/item_list/', 'recommend', { from_page: 'fyp' }, 0); + } catch (error) { + apiFailures.push(error instanceof Error ? error.message : String(error)); } } @@ -94,7 +116,7 @@ function buildExploreScript(limit) { .map((row, index) => ({ ...row, index: index + 1 })); if (rows.length === 0) { - const suffix = apiFailure ? ' (recommend API failed: ' + apiFailure + ')' : ''; + const suffix = apiFailures.length ? ' (explore APIs failed: ' + apiFailures.join('; ') + ')' : ''; throw new Error('No videos found on /explore' + suffix); } return rows; diff --git a/plugins/twitter/README.md b/plugins/twitter/README.md index a6f9208a..d8d4f167 100644 --- a/plugins/twitter/README.md +++ b/plugins/twitter/README.md @@ -19,6 +19,7 @@ webcmd plugin install github:agentrhq/webcmd/twitter | `webcmd twitter bookmark-folder` | Read the tweets inside a single Twitter/X bookmark folder. Get the folder id from `webcmd twitter bookmark-folders`. | | `webcmd twitter bookmark-folders` | List your Twitter/X bookmark folders (the user-created collections under Bookmarks). Returns folder id, name, item count, and created_at. | | `webcmd twitter bookmarks` | Fetch your Twitter/X bookmarks (the logged-in user's saved tweets, newest first) | +| `webcmd twitter collection` | Fetch a user timeline with relationship facts and a bounded completion receipt. | | `webcmd twitter delete` | Delete a specific tweet by URL | | `webcmd twitter device-follow` | Read the /i/timeline device-follow notification stream (tweets aggregated under a bell-icon "new posts from @userA and N others" notification) | | `webcmd twitter download` | Download Twitter/X media (images and videos). Provide either to fetch every media item from their profile via the GraphQL UserMedia endpoint with cursor pagination, or --tweet-url to download a single tweet. | diff --git a/plugins/twitter/article.js b/plugins/twitter/article.js index 71821f0b..a5bdabc0 100644 --- a/plugins/twitter/article.js +++ b/plugins/twitter/article.js @@ -173,6 +173,32 @@ cli({ if (!Array.isArray(blocks)) { return {error: 'Twitter API response article blocks were malformed'}; } + // The current GraphQL response serializes Draft.js entityMap as an + // unordered [{key, value}] array, not an object keyed by entity id. + // Normalize both representations before resolving atomic blocks. + const rawEntityMap = contentState.entityMap || {}; + const entityByKey = {}; + if (Array.isArray(rawEntityMap)) { + for (const entry of rawEntityMap) { + if (entry && entry.key != null && entry.value) { + entityByKey[String(entry.key)] = entry.value; + } + } + } else { + for (const [key, entry] of Object.entries(rawEntityMap)) { + entityByKey[String(key)] = entry?.value || entry; + } + } + + // Build media_id -> original_img_url lookup from media_entities. + const mediaEntities = articleResults.media_entities || []; + const mediaUrlById = {}; + for (const me of Object.values(mediaEntities)) { + const url = me?.media_info?.original_img_url; + if (typeof url === 'string' && me?.media_id != null) { + mediaUrlById[String(me.media_id)] = url; + } + } // Convert draft.js blocks to Markdown const parts = []; @@ -180,7 +206,17 @@ cli({ for (const block of blocks) { if (!block || typeof block !== 'object' || Array.isArray(block)) continue; const blockType = block.type || 'unstyled'; - if (blockType === 'atomic') continue; + if (blockType === 'atomic') { + const entityKey = block.entityRanges?.[0]?.key; + const entity = entityKey == null ? null : entityByKey[String(entityKey)]; + if (entity?.type === 'MEDIA') { + const mediaId = entity.data?.mediaItems?.[0]?.mediaId; + const imgUrl = mediaId == null ? null : mediaUrlById[String(mediaId)]; + const caption = String(entity.data?.caption || 'Image').replaceAll(']', ']'); + if (imgUrl) parts.push('![' + caption + '](' + imgUrl + ')'); + } + continue; + } const text = block.text || ''; if (!text) continue; if (blockType !== 'ordered-list-item') orderedCounter = 0; diff --git a/plugins/twitter/bookmarks.js b/plugins/twitter/bookmarks.js index c1538105..e6b07ade 100644 --- a/plugins/twitter/bookmarks.js +++ b/plugins/twitter/bookmarks.js @@ -1,9 +1,13 @@ +import fs from 'node:fs'; +import path from 'node:path'; import { cli, Strategy } from '@agentrhq/webcmd/registry'; -import { AuthRequiredError, CommandExecutionError } from '@agentrhq/webcmd/errors'; -import { extractMedia, describeTwitterApiError } from './shared.js'; +import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors'; +import { extractMedia, describeTwitterApiError, resolveTwitterQueryId, unwrapBrowserResult } from './shared.js'; import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js'; const BOOKMARKS_QUERY_ID = 'Fy0QMy4q_aZCpkO0PnyLYw'; -const MAX_PAGINATION_PAGES = 100; +// Safety cap only. Full-archive runs can set a higher page budget via --max-pages. +const DEFAULT_MAX_PAGINATION_PAGES = 100; +const HARD_MAX_PAGINATION_PAGES = 100000; const FEATURES = { rweb_video_screen_enabled: false, profile_label_improvements_pcf_label_in_post_enabled: true, @@ -100,6 +104,137 @@ export function parseBookmarks(data, seen) { } return { tweets, nextCursor }; } +function resolveOptionalFilePath(raw, label) { + if (raw === undefined || raw === null || raw === '') + return ''; + const value = String(raw).trim(); + if (!value) + throw new ArgumentError(`${label} cannot be empty`); + return path.resolve(value); +} +function readResumeFile(filePath, expected = null) { + if (!filePath || !fs.existsSync(filePath)) + return null; + let parsed; + try { + parsed = JSON.parse(fs.readFileSync(filePath, 'utf8')); + } + catch (error) { + throw new CommandExecutionError(`Could not parse Twitter bookmarks resume file ${filePath}: ${error instanceof Error ? error.message : String(error)}`); + } + const count = parsed?.count; + const cursor = parsed?.cursor == null ? null : String(parsed.cursor); + const outputFile = parsed?.outputFile ? path.resolve(String(parsed.outputFile)) : null; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed) + || !Number.isInteger(count) || count < 0 + || (parsed.cursor != null && typeof parsed.cursor !== 'string') + || (cursor !== null && !cursor.trim())) { + throw new CommandExecutionError(`Twitter bookmarks resume file ${filePath} has an invalid shape`); + } + if (expected) { + if (parsed.source !== expected.source) + throw new ArgumentError(`Resume file source mismatch: expected ${expected.source}, found ${parsed.source || 'unknown'}`); + if (outputFile !== expected.outputFile) + throw new ArgumentError(`Resume file output mismatch: expected ${expected.outputFile || 'in-memory mode'}, found ${outputFile || 'in-memory mode'}`); + if (!expected.outputFile && !Array.isArray(parsed.tweets)) + throw new CommandExecutionError(`Twitter bookmarks resume file ${filePath} is missing in-memory tweets`); + if (!expected.outputFile && parsed.tweets.length !== count) + throw new CommandExecutionError(`Twitter bookmarks resume file ${filePath} count does not match its in-memory tweets`); + if (parsed.complete) + throw new CommandExecutionError(`Twitter bookmarks resume file ${filePath} is already marked complete`); + } + return { + cursor, + count, + tweets: Array.isArray(parsed.tweets) ? parsed.tweets : [], + complete: Boolean(parsed.complete), + source: parsed.source || null, + outputFile, + updatedAt: parsed.updatedAt || null, + }; +} +function ensureParentDir(filePath) { + if (!filePath) + return; + fs.mkdirSync(path.dirname(filePath), { recursive: true }); +} +function removeFile(filePath) { + if (!filePath) + return; + try { + fs.rmSync(filePath, { force: true }); + } + catch { + } +} +function loadJsonlArchiveState(filePath) { + const seen = new Set(); + let count = 0; + if (!filePath || !fs.existsSync(filePath)) + return { seen, count }; + const text = fs.readFileSync(filePath, 'utf8'); + for (const [index, line] of text.split('\n').entries()) { + const trimmed = line.trim(); + if (!trimmed) + continue; + try { + const row = JSON.parse(trimmed); + if (!row?.id) + throw new Error('missing id'); + const id = String(row.id); + if (seen.has(id)) + throw new Error(`duplicate id ${id}`); + seen.add(id); + count += 1; + } + catch (error) { + throw new CommandExecutionError(`Invalid JSONL record in ${filePath} at line ${index + 1}: ${error instanceof Error ? error.message : String(error)}`); + } + } + return { seen, count }; +} +function appendJsonlRows(filePath, rows) { + if (!filePath || !Array.isArray(rows) || rows.length === 0) + return; + ensureParentDir(filePath); + // Escape LS/PS so JSONL stays one physical line even when tweet text contains them. + const text = rows + .map((row) => JSON.stringify(row).replace(/\u2028/g, '\\u2028').replace(/\u2029/g, '\\u2029')) + .join('\n') + '\n'; + fs.appendFileSync(filePath, text, 'utf8'); +} +function writeResumeFile(filePath, payload) { + if (!filePath) + return; + ensureParentDir(filePath); + const temporaryPath = `${filePath}.tmp-${process.pid}`; + try { + fs.writeFileSync(temporaryPath, JSON.stringify(payload, null, 2) + '\n'); + fs.renameSync(temporaryPath, filePath); + } + catch (error) { + try { + fs.rmSync(temporaryPath, { force: true }); + } + catch { + } + throw new CommandExecutionError(`Could not persist Twitter bookmarks resume state: ${error instanceof Error ? error.message : String(error)}`); + } +} +function removeResumeFile(filePath) { + removeFile(filePath); +} +function resolveMaxPages(kwargs, fetchAll) { + const raw = kwargs['max-pages']; + if (raw === undefined || raw === null || raw === '') { + return fetchAll ? HARD_MAX_PAGINATION_PAGES : DEFAULT_MAX_PAGINATION_PAGES; + } + const value = Number(raw); + if (!Number.isInteger(value) || value < 1 || value > HARD_MAX_PAGINATION_PAGES) { + throw new ArgumentError(`--max-pages must be an integer between 1 and ${HARD_MAX_PAGINATION_PAGES}`); + } + return value; +} cli({ site: 'twitter', name: 'bookmarks', @@ -109,73 +244,147 @@ cli({ strategy: Strategy.COOKIE, browser: true, args: [ - { name: 'limit', type: 'int', default: 20, help: 'Maximum number of bookmarks to return (default 20).' }, - { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the bookmarks by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API\'s native (saved-time) ordering.' }, + { name: 'limit', type: 'int', default: 20, help: 'Maximum number of bookmarks to return (default 20). Ignored when --all is set.' }, + { name: 'all', type: 'bool', default: false, help: 'Fetch all bookmark pages until exhausted. Prefer --output-file for large archives.' }, + { name: 'resume-file', type: 'string', help: 'Resume file for long-running all-pages bookmark syncs.' }, + { name: 'output-file', type: 'string', help: 'Write all-page results to JSONL. Requires --all and --resume-file.' }, + { name: 'max-pages', type: 'int', help: `Optional pagination safety cap (default ${DEFAULT_MAX_PAGINATION_PAGES}; raised automatically with --all).` }, + { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the bookmarks by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API\'s native (saved-time) ordering. Incompatible with --output-file.' }, ], columns: ['id', 'author', 'text', 'likes', 'retweets', 'bookmarks', 'created_at', 'url', 'has_media', 'media_urls', 'media_posters'], func: async (page, kwargs) => { - const limit = kwargs.limit || 20; + const fetchAll = Boolean(kwargs.all); + const limit = fetchAll ? Number.POSITIVE_INFINITY : (kwargs.limit || 20); + const resumeFile = resolveOptionalFilePath(kwargs['resume-file'], '--resume-file'); + const outputFile = resolveOptionalFilePath(kwargs['output-file'], '--output-file'); + const useOutputFile = Boolean(fetchAll && outputFile); + const maxPages = resolveMaxPages(kwargs, fetchAll); + const topByEngagement = Number(kwargs['top-by-engagement'] || 0); + if (useOutputFile && topByEngagement > 0) { + throw new ArgumentError('--top-by-engagement cannot be combined with --output-file'); + } + if (outputFile && !fetchAll) { + throw new ArgumentError('--output-file requires --all'); + } + if (resumeFile && !fetchAll) { + throw new ArgumentError('--resume-file requires --all'); + } + if (outputFile && !resumeFile) { + throw new ArgumentError('--output-file requires --resume-file so partial archives remain resumable'); + } const cookies = await page.getCookies({ url: 'https://x.com' }); const ct0 = cookies.find((c) => c.name === 'ct0')?.value || null; if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)'); - const queryId = await page.evaluate(`async () => { - try { - const ghResp = await fetch('https://raw.githubusercontent.com/fa0311/twitter-openapi/refs/heads/main/src/config/placeholder.json'); - if (ghResp.ok) { - const data = await ghResp.json(); - const entry = data['Bookmarks']; - if (entry && entry.queryId) return entry.queryId; - } - } catch {} - try { - const scripts = performance.getEntriesByType('resource') - .filter(r => r.name.includes('client-web') && r.name.endsWith('.js')) - .map(r => r.name); - for (const scriptUrl of scripts.slice(0, 15)) { - try { - const text = await (await fetch(scriptUrl)).text(); - const re = /queryId:"([A-Za-z0-9_-]+)"[^}]{0,200}operationName:"Bookmarks"/; - const m = text.match(re); - if (m) return m[1]; - } catch {} - } - } catch {} - return null; - }`) || BOOKMARKS_QUERY_ID; + const queryId = await resolveTwitterQueryId(page, 'Bookmarks', BOOKMARKS_QUERY_ID); const headers = JSON.stringify({ 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, 'X-Csrf-Token': ct0, 'X-Twitter-Auth-Type': 'OAuth2Session', 'X-Twitter-Active-User': 'yes', }); - const allTweets = []; - const seen = new Set(); - let cursor = null; - // Runaway guard only; --limit and cursor exhaustion control normal pagination. - for (let i = 0; i < MAX_PAGINATION_PAGES && allTweets.length < limit; i++) { - const fetchCount = Math.min(100, limit - allTweets.length + 10); + const resumed = fetchAll ? readResumeFile(resumeFile, { + source: 'bookmarks', + outputFile: useOutputFile ? outputFile : null, + }) : null; + if (useOutputFile && resumed && resumed.count > 0 && !fs.existsSync(outputFile)) { + throw new CommandExecutionError(`Twitter bookmarks output file is missing for resume state: ${outputFile}`); + } + if (useOutputFile && !resumed && fs.existsSync(outputFile)) { + throw new ArgumentError(`Refusing to overwrite existing Twitter bookmarks output file: ${outputFile}`); + } + const allTweets = useOutputFile ? [] : (resumed?.tweets ? [...resumed.tweets] : []); + const jsonlState = useOutputFile ? loadJsonlArchiveState(outputFile) : null; + const seen = useOutputFile + ? jsonlState.seen + : new Set(allTweets.map((tweet) => tweet?.id).filter(Boolean)); + if (useOutputFile && resumed && jsonlState.count !== resumed.count) { + throw new CommandExecutionError(`Twitter bookmarks output file has ${jsonlState.count} record(s), expected resume count ${resumed.count}`); + } + let outputCount = useOutputFile ? jsonlState.count : 0; + let cursor = resumed?.cursor || null; + let pages = 0; + let exhausted = false; + // Runaway guard only; --limit/--all and cursor exhaustion control normal pagination. + while (pages < maxPages && (fetchAll || allTweets.length < limit)) { + pages += 1; + const currentCount = useOutputFile ? outputCount : allTweets.length; + const remaining = fetchAll ? 100 : (limit - currentCount + 10); + const fetchCount = Math.min(100, remaining); const apiUrl = buildBookmarksUrl(fetchCount, cursor).replace(BOOKMARKS_QUERY_ID, queryId); - const data = await page.evaluate(`async () => { - const r = await fetch("${apiUrl}", { headers: ${headers}, credentials: 'include' }); + const data = unwrapBrowserResult(await page.evaluate(`async () => { + const r = await fetch(${JSON.stringify(apiUrl)}, { headers: ${headers}, credentials: 'include' }); return r.ok ? await r.json() : { error: r.status }; - }`); + }`)); if (data?.error) { - if (allTweets.length === 0) + if ((useOutputFile ? outputCount : allTweets.length) === 0) throw new CommandExecutionError(describeTwitterApiError('Bookmarks', data.error)); break; } + const hasInstructions = Array.isArray(data?.data?.bookmark_timeline_v2?.timeline?.instructions) + || Array.isArray(data?.data?.bookmark_timeline?.timeline?.instructions); + if (!hasInstructions) { + throw new CommandExecutionError('twitter_bookmarks_protocol_error: missing Bookmarks timeline instructions'); + } const { tweets, nextCursor } = parseBookmarks(data, seen); - allTweets.push(...tweets); - if (!nextCursor || nextCursor === cursor) + if (useOutputFile) { + appendJsonlRows(outputFile, tweets); + outputCount += tweets.length; + } + else { + allTweets.push(...tweets); + } + const pageComplete = !nextCursor; + writeResumeFile(resumeFile, { + cursor: pageComplete ? null : nextCursor, + count: useOutputFile ? outputCount : allTweets.length, + tweets: useOutputFile ? undefined : allTweets, + updatedAt: new Date().toISOString(), + complete: pageComplete, + source: 'bookmarks', + outputFile: useOutputFile ? outputFile : null, + }); + if (pageComplete) { + exhausted = true; break; + } + if (nextCursor === cursor) { + throw new CommandExecutionError('twitter_bookmarks_repeated_cursor: archive completion cannot be proven; resume state was retained'); + } cursor = nextCursor; } - const trimmed = allTweets.slice(0, limit); - return applyTopByEngagement(trimmed, kwargs['top-by-engagement']); + const finalCount = useOutputFile ? outputCount : allTweets.length; + if (finalCount === 0) { + throw new EmptyResultError('twitter bookmarks', 'No bookmarks found for the logged-in account'); + } + // Resume is only removed after the timeline is truly exhausted. Hitting + // --max-pages, partial API errors after some rows, or an interrupt must + // leave the resume file so the next run can continue. + if (exhausted) + removeResumeFile(resumeFile); + if (useOutputFile) { + return { + outputFile, + count: outputCount, + source: 'bookmarks', + complete: exhausted, + pages, + ...(exhausted ? {} : { cursor, resumeFile: resumeFile || null }), + }; + } + if (fetchAll && !exhausted) { + throw new CommandExecutionError( + `twitter_bookmarks_archive_incomplete: stopped after ${pages} page(s); completion cannot be proven`, + resumeFile ? `Resume with --resume-file ${resumeFile}` : 'Rerun with --resume-file to preserve continuation state.', + ); + } + const trimmed = fetchAll ? allTweets : allTweets.slice(0, limit); + return applyTopByEngagement(trimmed, topByEngagement); }, }); export const __test__ = { parseBookmarks, extractBookmarkTweet, + appendJsonlRows, + readResumeFile, }; diff --git a/plugins/twitter/collection.js b/plugins/twitter/collection.js new file mode 100644 index 00000000..ff2f56ce --- /dev/null +++ b/plugins/twitter/collection.js @@ -0,0 +1,339 @@ +import { cli, Strategy } from '@agentrhq/webcmd/registry'; +import { ArgumentError, CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors'; +import { + extractMedia, + extractQuotedTweet, + looksLikePrivateTwitterTimeline, + normalizeTwitterScreenName, +} from './shared.js'; +import { + DEFAULT_USER_TWEETS_PAGE_DELAY_SECONDS, + MAX_USER_TWEETS_LIMIT, + MAX_USER_TWEETS_PAGES, + USER_TWEETS_PAGE_SIZE, + fetchUserTimelinePage, + resolveUserTimelineContext, +} from './user-timeline.js'; + +const RFC3339_TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(Z|([+-])(\d{2}):(\d{2}))$/; + +function isLeapYear(year) { + return year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); +} + +function normalizeUntil(raw) { + const value = String(raw ?? '').trim(); + const match = value.match(RFC3339_TIMESTAMP); + if (!match) { + throw new ArgumentError( + 'twitter collection --until must be an RFC3339 timestamp', + 'Example: webcmd twitter collection @jack --until 2026-07-23T00:00:00Z', + ); + } + const [, yearRaw, monthRaw, dayRaw, hourRaw, minuteRaw, secondRaw, , zone, , offsetHourRaw, offsetMinuteRaw] = match; + const year = Number(yearRaw); + const month = Number(monthRaw); + const day = Number(dayRaw); + const hour = Number(hourRaw); + const minute = Number(minuteRaw); + const second = Number(secondRaw); + const offsetHour = zone === 'Z' ? 0 : Number(offsetHourRaw); + const offsetMinute = zone === 'Z' ? 0 : Number(offsetMinuteRaw); + const daysInMonth = month === 2 + ? (isLeapYear(year) ? 29 : 28) + : ([4, 6, 9, 11].includes(month) ? 30 : 31); + if ( + month < 1 || month > 12 + || day < 1 || day > daysInMonth + || hour > 23 || minute > 59 || second > 59 + || offsetHour > 23 || offsetMinute > 59 + ) { + throw new ArgumentError( + 'twitter collection --until must be an RFC3339 timestamp', + 'Example: webcmd twitter collection @jack --until 2026-07-23T00:00:00Z', + ); + } + const parsed = new Date(value); + if (Number.isNaN(parsed.getTime())) { + throw new ArgumentError( + 'twitter collection --until must be an RFC3339 timestamp', + 'Example: webcmd twitter collection @jack --until 2026-07-23T00:00:00Z', + ); + } + return parsed; +} + +function normalizeCollectionLimit(rawLimit) { + const limit = rawLimit ?? MAX_USER_TWEETS_LIMIT; + if (!Number.isInteger(limit) || limit < 1 || limit > MAX_USER_TWEETS_LIMIT) { + throw new ArgumentError( + `twitter collection --limit must be an integer between 1 and ${MAX_USER_TWEETS_LIMIT}`, + 'Example: webcmd twitter collection @jack --until 2026-07-23T00:00:00Z --limit 250', + ); + } + return limit; +} + +function normalizeCollectionPageDelaySeconds(rawDelay) { + const delay = rawDelay ?? DEFAULT_USER_TWEETS_PAGE_DELAY_SECONDS; + if (!Number.isInteger(delay) || delay < 0 || delay > 60) { + throw new ArgumentError( + 'twitter collection --page-delay must be an integer between 0 and 60 seconds', + 'Example: webcmd twitter collection @jack --until 2026-07-23T00:00:00Z --page-delay 2', + ); + } + return delay; +} + +function unwrapTweetResult(result) { + if (!result) return null; + if (result.__typename === 'TweetWithVisibilityResults' && result.tweet) return result.tweet; + return result.tweet || result; +} + +function relationshipTarget(result, fallbackId = null, contextStatus = 'complete') { + const tweet = unwrapTweetResult(result); + const user = tweet?.core?.user_results?.result; + const rawHandle = user?.legacy?.screen_name || user?.core?.screen_name || null; + const authorHandle = typeof rawHandle === 'string' && normalizeTwitterScreenName(rawHandle) + ? normalizeTwitterScreenName(rawHandle) + : null; + const rawAuthorId = user?.rest_id || user?.legacy?.id_str || null; + const authorId = typeof rawAuthorId === 'string' && rawAuthorId.trim() ? rawAuthorId : null; + const rawPostId = tweet?.rest_id || fallbackId || null; + const postId = typeof rawPostId === 'string' && rawPostId.trim() ? rawPostId : null; + const hasVisibleContext = Boolean(tweet?.note_tweet?.note_tweet_results?.result?.text || tweet?.legacy?.full_text); + const resolvedContextStatus = contextStatus === 'complete' && !hasVisibleContext + ? (postId ? 'unavailable' : 'unknown') + : contextStatus; + return { + post_id: postId, + author_handle: authorHandle, + author_id: authorId, + url: postId && authorHandle ? `https://x.com/${authorHandle}/status/${postId}` : null, + context_status: resolvedContextStatus, + }; +} + +function extractRelationship(result) { + const tweet = unwrapTweetResult(result); + const legacy = tweet?.legacy || {}; + const repostResult = tweet?.retweeted_status_result?.result || legacy.retweeted_status_result?.result || null; + const repostId = legacy.retweeted_status_id_str || null; + if (repostResult || repostId) { + const target = relationshipTarget(repostResult, repostId, repostResult ? 'complete' : 'unknown'); + if (!repostResult || !target.post_id) { + throw new CommandExecutionError('twitter_collection_unresolved_relationship: repost target is unavailable'); + } + return { kind: 'repost', target }; + } + const quoteResult = tweet?.quoted_status_result?.result || legacy.quoted_status_result?.result || null; + const quoteId = legacy.quoted_status_id_str || null; + if (legacy.is_quote_status || quoteResult || quoteId) { + return { + kind: 'quote', + target: relationshipTarget(quoteResult, quoteId, quoteResult ? 'complete' : 'unavailable'), + }; + } + const replyId = legacy.in_reply_to_status_id_str || null; + const replyHandle = normalizeTwitterScreenName(legacy.in_reply_to_screen_name || '') || null; + const replyAuthorId = typeof legacy.in_reply_to_user_id_str === 'string' && legacy.in_reply_to_user_id_str.trim() + ? legacy.in_reply_to_user_id_str + : null; + if (replyId || replyHandle || replyAuthorId) { + return { + kind: 'reply', + target: { + post_id: replyId, + author_handle: replyHandle, + author_id: replyAuthorId, + url: replyId && replyHandle ? `https://x.com/${replyHandle}/status/${replyId}` : null, + context_status: replyId ? 'unavailable' : 'unknown', + }, + }; + } + return { kind: 'original', target: null }; +} + +function extractCollectionPost(result, seen) { + const tweet = unwrapTweetResult(result); + if (!tweet?.rest_id || typeof tweet.rest_id !== 'string') { + throw new CommandExecutionError('twitter_collection_protocol_error: timeline post is missing a stable ID'); + } + if (seen.has(tweet.rest_id)) return null; + seen.add(tweet.rest_id); + const legacy = tweet.legacy || {}; + const user = tweet.core?.user_results?.result; + const author = user?.legacy?.screen_name || user?.core?.screen_name || null; + if (!author || !normalizeTwitterScreenName(author)) { + throw new CommandExecutionError('twitter_collection_protocol_error: timeline post is missing an author handle'); + } + return { + id: tweet.rest_id, + author, + name: user?.legacy?.name || user?.core?.name || '', + text: tweet.note_tweet?.note_tweet_results?.result?.text || legacy.full_text || '', + likes: legacy.favorite_count || 0, + retweets: legacy.retweet_count || 0, + replies: legacy.reply_count || 0, + views: Number(tweet.views?.count) || 0, + is_retweet: Boolean(legacy.retweeted_status_result), + created_at: legacy.created_at || '', + url: `https://x.com/${author}/status/${tweet.rest_id}`, + ...extractMedia(legacy), + quoted_tweet: extractQuotedTweet(tweet), + relationship: extractRelationship(tweet), + }; +} + +function parseCollectionPage(payload, seen) { + if (looksLikePrivateTwitterTimeline(payload)) { + throw new EmptyResultError( + 'twitter collection', + 'Timeline is private or unavailable to the current X account; completion cannot be proven.', + ); + } + const result = payload?.data?.user?.result; + if (!result || typeof result !== 'object') { + throw new CommandExecutionError('twitter_collection_protocol_error: missing UserTweets result'); + } + const instructionSets = [ + result.timeline_v2?.timeline?.instructions, + result.timeline?.timeline?.instructions, + ].filter(Array.isArray); + if (instructionSets.length === 0) { + throw new CommandExecutionError( + 'twitter_collection_protocol_error: missing UserTweets timeline instructions', + ); + } + const posts = []; + let nextCursor = null; + const visit = (value) => { + if (!value || typeof value !== 'object') return; + if (value.type === 'TimelinePinEntry') return; + if (value.tweet_results?.result) { + const post = extractCollectionPost(value.tweet_results.result, seen); + if (post) posts.push(post); + } + if ( + (value.entryType === 'TimelineTimelineCursor' || value.__typename === 'TimelineTimelineCursor') + && (value.cursorType === 'Bottom' || value.cursorType === 'ShowMore') + && value.value + ) { + nextCursor = value.value; + } + if (Array.isArray(value)) { + for (const item of value) visit(item); + return; + } + for (const child of Object.values(value)) { + if (child && typeof child === 'object') visit(child); + } + }; + for (const instructions of instructionSets) visit(instructions); + return [posts, nextCursor]; +} + +function parseCreatedAt(post) { + const parsed = new Date(post.created_at); + if (typeof post.created_at !== 'string' || !post.created_at || Number.isNaN(parsed.getTime())) { + throw new CommandExecutionError(`twitter_collection_invalid_timestamp: post ${post.id}`); + } + return parsed; +} + +function completedReceipt(stopReason, until, pagesFetched, oldestSeenAt) { + return { + completed: true, + stop_reason: stopReason, + requested_until: until.toISOString(), + pages_fetched: pagesFetched, + oldest_seen_at: oldestSeenAt ? oldestSeenAt.toISOString() : null, + }; +} + +async function paginateCollection({ until, limit, maxPages = MAX_USER_TWEETS_PAGES, fetchPage, wait }) { + const seen = new Set(); + const seenCursors = new Set(); + const posts = []; + let cursor = null; + let oldestSeenAt = null; + for (let pageIndex = 0; pageIndex < maxPages; pageIndex++) { + if (pageIndex > 0 && wait) await wait(); + const payload = await fetchPage(cursor, USER_TWEETS_PAGE_SIZE); + if (payload?.error) { + throw new CommandExecutionError(`twitter_collection_request_error: UserTweets returned ${payload.error}`); + } + const [pagePosts, nextCursor] = parseCollectionPage(payload, seen); + for (const post of pagePosts) { + if (posts.length >= limit) { + throw new CommandExecutionError('twitter_collection_limit_reached: pagination cannot prove completion'); + } + const createdAt = parseCreatedAt(post); + if (!oldestSeenAt || createdAt < oldestSeenAt) oldestSeenAt = createdAt; + posts.push(post); + if (createdAt <= until) { + return { + posts, + receipt: completedReceipt('time_boundary_reached', until, pageIndex + 1, oldestSeenAt), + }; + } + } + if (!nextCursor) { + return { + posts, + receipt: completedReceipt('cursor_exhausted', until, pageIndex + 1, oldestSeenAt), + }; + } + if (nextCursor === cursor || seenCursors.has(nextCursor)) { + throw new CommandExecutionError('twitter_collection_repeated_cursor: pagination cannot prove completion'); + } + if (posts.length >= limit) { + throw new CommandExecutionError('twitter_collection_limit_reached: pagination cannot prove completion'); + } + seenCursors.add(nextCursor); + cursor = nextCursor; + } + throw new CommandExecutionError('twitter_collection_page_guard_hit: pagination cannot prove completion'); +} + +cli({ + site: 'twitter', + name: 'collection', + access: 'read', + description: 'Fetch a user timeline with relationship facts and a bounded completion receipt.', + domain: 'x.com', + strategy: Strategy.COOKIE, + browser: true, + args: [ + { name: 'username', type: 'string', positional: true, required: true, help: 'Twitter screen name (with or without @).' }, + { name: 'until', type: 'string', required: true, help: 'RFC3339 lower time boundary that must be reached or exhausted.' }, + { name: 'limit', type: 'int', default: MAX_USER_TWEETS_LIMIT, help: 'Safety ceiling; reaching it is a typed failure.' }, + { name: 'page-delay', type: 'int', default: DEFAULT_USER_TWEETS_PAGE_DELAY_SECONDS, help: 'Seconds to wait between cursor pages.' }, + ], + columns: ['posts', 'receipt'], + func: async (page, kwargs) => { + const until = normalizeUntil(kwargs.until); + const limit = normalizeCollectionLimit(kwargs.limit); + const pageDelaySeconds = normalizeCollectionPageDelaySeconds(kwargs['page-delay']); + const context = await resolveUserTimelineContext(page, kwargs.username, { + allowLoggedInDefault: false, + commandName: 'collection', + }); + return paginateCollection({ + until, + limit, + fetchPage: (cursor, count) => fetchUserTimelinePage(page, context, cursor, count), + wait: pageDelaySeconds > 0 ? () => page.wait(pageDelaySeconds) : null, + }); + }, +}); + +export const __test__ = { + normalizeUntil, + normalizeCollectionLimit, + normalizeCollectionPageDelaySeconds, + extractRelationship, + parseCollectionPage, + paginateCollection, +}; diff --git a/plugins/twitter/likes.js b/plugins/twitter/likes.js index ba23720d..fc98db97 100644 --- a/plugins/twitter/likes.js +++ b/plugins/twitter/likes.js @@ -1,10 +1,14 @@ +import fs from 'node:fs'; +import path from 'node:path'; import { cli, Strategy } from '@agentrhq/webcmd/registry'; import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors'; import { looksLikePrivateTwitterTimeline, normalizeTwitterScreenName, resolveTwitterQueryId, sanitizeQueryId, extractMedia, unwrapBrowserResult, describeTwitterApiError } from './shared.js'; import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js'; const LIKES_QUERY_ID = 'CDWHmpZeSdIJ3HGeRbNm0w'; const USER_BY_SCREEN_NAME_QUERY_ID = 'IGgvgiOx4QZndDHuD3x9TQ'; -const MAX_PAGINATION_PAGES = 100; +// Safety cap only. Full-archive runs can set a higher page budget via --max-pages. +const DEFAULT_MAX_PAGINATION_PAGES = 100; +const HARD_MAX_PAGINATION_PAGES = 100000; const FEATURES = { rweb_video_screen_enabled: false, profile_label_improvements_pcf_label_in_post_enabled: true, @@ -135,6 +139,140 @@ function parseLikes(data, seen) { } return { tweets, nextCursor }; } +function resolveOptionalFilePath(raw, label) { + if (raw === undefined || raw === null || raw === '') + return ''; + const value = String(raw).trim(); + if (!value) + throw new ArgumentError(`${label} cannot be empty`); + return path.resolve(value); +} +function readResumeFile(filePath, expected = null) { + if (!filePath || !fs.existsSync(filePath)) + return null; + let parsed; + try { + parsed = JSON.parse(fs.readFileSync(filePath, 'utf8')); + } + catch (error) { + throw new CommandExecutionError(`Could not parse Twitter likes resume file ${filePath}: ${error instanceof Error ? error.message : String(error)}`); + } + const count = parsed?.count; + const cursor = parsed?.cursor == null ? null : String(parsed.cursor); + const outputFile = parsed?.outputFile ? path.resolve(String(parsed.outputFile)) : null; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed) + || !Number.isInteger(count) || count < 0 + || (parsed.cursor != null && typeof parsed.cursor !== 'string') + || (cursor !== null && !cursor.trim())) { + throw new CommandExecutionError(`Twitter likes resume file ${filePath} has an invalid shape`); + } + if (expected) { + if (parsed.source !== expected.source) + throw new ArgumentError(`Resume file source mismatch: expected ${expected.source}, found ${parsed.source || 'unknown'}`); + if (String(parsed.username || '').toLowerCase() !== String(expected.username).toLowerCase()) + throw new ArgumentError(`Resume file username mismatch: expected @${expected.username}, found @${parsed.username || 'unknown'}`); + if (outputFile !== expected.outputFile) + throw new ArgumentError(`Resume file output mismatch: expected ${expected.outputFile || 'in-memory mode'}, found ${outputFile || 'in-memory mode'}`); + if (!expected.outputFile && !Array.isArray(parsed.tweets)) + throw new CommandExecutionError(`Twitter likes resume file ${filePath} is missing in-memory tweets`); + if (!expected.outputFile && parsed.tweets.length !== count) + throw new CommandExecutionError(`Twitter likes resume file ${filePath} count does not match its in-memory tweets`); + if (parsed.complete) + throw new CommandExecutionError(`Twitter likes resume file ${filePath} is already marked complete`); + } + return { + cursor, + count, + tweets: Array.isArray(parsed.tweets) ? parsed.tweets : [], + username: parsed.username || null, + complete: Boolean(parsed.complete), + source: parsed.source || null, + outputFile, + updatedAt: parsed.updatedAt || null, + }; +} +function ensureParentDir(filePath) { + if (!filePath) + return; + fs.mkdirSync(path.dirname(filePath), { recursive: true }); +} +function removeFile(filePath) { + if (!filePath) + return; + try { + fs.rmSync(filePath, { force: true }); + } + catch { + } +} +function loadJsonlArchiveState(filePath) { + const seen = new Set(); + let count = 0; + if (!filePath || !fs.existsSync(filePath)) + return { seen, count }; + const text = fs.readFileSync(filePath, 'utf8'); + for (const [index, line] of text.split('\n').entries()) { + const trimmed = line.trim(); + if (!trimmed) + continue; + try { + const row = JSON.parse(trimmed); + if (!row?.id) + throw new Error('missing id'); + const id = String(row.id); + if (seen.has(id)) + throw new Error(`duplicate id ${id}`); + seen.add(id); + count += 1; + } + catch (error) { + throw new CommandExecutionError(`Invalid JSONL record in ${filePath} at line ${index + 1}: ${error instanceof Error ? error.message : String(error)}`); + } + } + return { seen, count }; +} +function appendJsonlRows(filePath, rows) { + if (!filePath || !Array.isArray(rows) || rows.length === 0) + return; + ensureParentDir(filePath); + // Escape LS/PS so JSONL stays one physical line even when tweet text contains them. + const text = rows + .map((row) => JSON.stringify(row).replace(/\u2028/g, '\\u2028').replace(/\u2029/g, '\\u2029')) + .join('\n') + '\n'; + fs.appendFileSync(filePath, text, 'utf8'); +} +function writeResumeFile(filePath, payload) { + if (!filePath) + return; + ensureParentDir(filePath); + const temporaryPath = `${filePath}.tmp-${process.pid}`; + try { + fs.writeFileSync(temporaryPath, JSON.stringify(payload, null, 2) + '\n'); + fs.renameSync(temporaryPath, filePath); + } + catch (error) { + try { + fs.rmSync(temporaryPath, { force: true }); + } + catch { + } + throw new CommandExecutionError(`Could not persist Twitter likes resume state: ${error instanceof Error ? error.message : String(error)}`); + } +} +function removeResumeFile(filePath) { + removeFile(filePath); +} +function resolveMaxPages(kwargs, fetchAll) { + const raw = kwargs['max-pages']; + if (raw === undefined || raw === null || raw === '') { + return fetchAll ? HARD_MAX_PAGINATION_PAGES : DEFAULT_MAX_PAGINATION_PAGES; + } + const value = Number(raw); + if (!Number.isInteger(value) || value < 1 || value > HARD_MAX_PAGINATION_PAGES) { + throw new ArgumentError(`--max-pages must be an integer between 1 and ${HARD_MAX_PAGINATION_PAGES}`); + } + return value; +} cli({ site: 'twitter', name: 'likes', @@ -145,12 +283,34 @@ cli({ browser: true, args: [ { name: 'username', type: 'string', positional: true, help: 'Twitter screen name (with or without @). Defaults to the logged-in user when omitted.' }, - { name: 'limit', type: 'int', default: 20, help: 'Maximum number of liked tweets to return (default 20).' }, - { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the liked tweets by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API\'s native (recency) ordering.' }, + { name: 'limit', type: 'int', default: 20, help: 'Maximum number of liked tweets to return (default 20). Ignored when --all is set.' }, + { name: 'all', type: 'bool', default: false, help: 'Fetch all liked-tweet pages until exhausted. Prefer --output-file for large archives.' }, + { name: 'resume-file', type: 'string', help: 'Resume file for long-running all-pages likes syncs.' }, + { name: 'output-file', type: 'string', help: 'Write all-page results to JSONL. Requires --all and --resume-file.' }, + { name: 'max-pages', type: 'int', help: `Optional pagination safety cap (default ${DEFAULT_MAX_PAGINATION_PAGES}; raised automatically with --all).` }, + { name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the liked tweets by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API\'s native (recency) ordering. Incompatible with --output-file.' }, ], columns: ['id', 'author', 'name', 'text', 'likes', 'retweets', 'created_at', 'url', 'has_media', 'media_urls', 'media_posters'], func: async (page, kwargs) => { - const limit = kwargs.limit || 20; + const fetchAll = Boolean(kwargs.all); + const limit = fetchAll ? Number.POSITIVE_INFINITY : (kwargs.limit || 20); + const resumeFile = resolveOptionalFilePath(kwargs['resume-file'], '--resume-file'); + const outputFile = resolveOptionalFilePath(kwargs['output-file'], '--output-file'); + const useOutputFile = Boolean(fetchAll && outputFile); + const maxPages = resolveMaxPages(kwargs, fetchAll); + const topByEngagement = Number(kwargs['top-by-engagement'] || 0); + if (useOutputFile && topByEngagement > 0) { + throw new ArgumentError('--top-by-engagement cannot be combined with --output-file'); + } + if (outputFile && !fetchAll) { + throw new ArgumentError('--output-file requires --all'); + } + if (resumeFile && !fetchAll) { + throw new ArgumentError('--resume-file requires --all'); + } + if (outputFile && !resumeFile) { + throw new ArgumentError('--output-file requires --resume-file so partial archives remain resumable'); + } const rawUsername = String(kwargs.username ?? '').trim(); let username = normalizeTwitterScreenName(rawUsername); if (rawUsername && !username) { @@ -199,38 +359,114 @@ cli({ if (!userId) { throw new CommandExecutionError(`Could not find user @${username}`); } - const allTweets = []; - const seen = new Set(); - let cursor = null; + const resumed = fetchAll ? readResumeFile(resumeFile, { + source: 'likes', + username, + outputFile: useOutputFile ? outputFile : null, + }) : null; + if (useOutputFile && resumed && resumed.count > 0 && !fs.existsSync(outputFile)) { + throw new CommandExecutionError(`Twitter likes output file is missing for resume state: ${outputFile}`); + } + if (useOutputFile && !resumed && fs.existsSync(outputFile)) { + throw new ArgumentError(`Refusing to overwrite existing Twitter likes output file: ${outputFile}`); + } + const allTweets = useOutputFile ? [] : (resumed?.tweets ? [...resumed.tweets] : []); + const jsonlState = useOutputFile ? loadJsonlArchiveState(outputFile) : null; + const seen = useOutputFile + ? jsonlState.seen + : new Set(allTweets.map((tweet) => tweet?.id).filter(Boolean)); + if (useOutputFile && resumed && jsonlState.count !== resumed.count) { + throw new CommandExecutionError(`Twitter likes output file has ${jsonlState.count} record(s), expected resume count ${resumed.count}`); + } + let outputCount = useOutputFile ? jsonlState.count : 0; + let cursor = resumed?.cursor || null; let lastRawResponse = null; - // Runaway guard only; --limit and cursor exhaustion control normal pagination. - for (let i = 0; i < MAX_PAGINATION_PAGES && allTweets.length < limit; i++) { - const fetchCount = Math.min(100, limit - allTweets.length + 10); + let pages = 0; + let exhausted = false; + // Runaway guard only; --limit/--all and cursor exhaustion control normal pagination. + while (pages < maxPages && (fetchAll || allTweets.length < limit)) { + pages += 1; + const currentCount = useOutputFile ? outputCount : allTweets.length; + const remaining = fetchAll ? 100 : (limit - currentCount + 10); + const fetchCount = Math.min(100, remaining); const apiUrl = buildLikesUrl(likesQueryId, userId, fetchCount, cursor); const data = unwrapBrowserResult(await page.evaluate(`async () => { - const r = await fetch("${apiUrl}", { headers: ${headers}, credentials: 'include' }); + const r = await fetch(${JSON.stringify(apiUrl)}, { headers: ${headers}, credentials: 'include' }); return r.ok ? await r.json() : { error: r.status }; }`)); if (data?.error) { - if (allTweets.length === 0) + if ((useOutputFile ? outputCount : allTweets.length) === 0) throw new CommandExecutionError(describeTwitterApiError('Likes', data.error)); break; } lastRawResponse = data; + const hasInstructions = Array.isArray(data?.data?.user?.result?.timeline_v2?.timeline?.instructions) + || Array.isArray(data?.data?.user?.result?.timeline?.timeline?.instructions); + if (!hasInstructions) { + if (looksLikePrivateTwitterTimeline(data) && (useOutputFile ? outputCount : allTweets.length) === 0) { + throw new EmptyResultError('twitter likes', `No likes returned for @${username} (Likes are private by default on X; only the account owner can view their own likes)`); + } + throw new CommandExecutionError('twitter_likes_protocol_error: missing Likes timeline instructions'); + } const { tweets, nextCursor } = parseLikes(data, seen); - allTweets.push(...tweets); - if (!nextCursor || nextCursor === cursor) + if (useOutputFile) { + appendJsonlRows(outputFile, tweets); + outputCount += tweets.length; + } + else { + allTweets.push(...tweets); + } + const pageComplete = !nextCursor; + writeResumeFile(resumeFile, { + cursor: pageComplete ? null : nextCursor, + count: useOutputFile ? outputCount : allTweets.length, + tweets: useOutputFile ? undefined : allTweets, + updatedAt: new Date().toISOString(), + complete: pageComplete, + source: 'likes', + username, + outputFile: useOutputFile ? outputFile : null, + }); + if (pageComplete) { + exhausted = true; break; + } + if (nextCursor === cursor) { + throw new CommandExecutionError('twitter_likes_repeated_cursor: archive completion cannot be proven; resume state was retained'); + } cursor = nextCursor; } - if (allTweets.length === 0) { + const finalCount = useOutputFile ? outputCount : allTweets.length; + if (finalCount === 0) { if (looksLikePrivateTwitterTimeline(lastRawResponse)) { throw new EmptyResultError('twitter likes', `No likes returned for @${username} (Likes are private by default on X; only the account owner can view their own likes)`); } throw new EmptyResultError('twitter likes', `No likes found for @${username}`); } - const trimmed = allTweets.slice(0, limit); - return applyTopByEngagement(trimmed, kwargs['top-by-engagement']); + // Resume is only removed after the timeline is truly exhausted. Hitting + // --max-pages, partial API errors after some rows, or an interrupt must + // leave the resume file so the next run can continue. + if (exhausted) + removeResumeFile(resumeFile); + if (useOutputFile) { + return { + outputFile, + count: outputCount, + source: 'likes', + username, + complete: exhausted, + pages, + ...(exhausted ? {} : { cursor, resumeFile: resumeFile || null }), + }; + } + if (fetchAll && !exhausted) { + throw new CommandExecutionError( + `twitter_likes_archive_incomplete: stopped after ${pages} page(s); completion cannot be proven`, + resumeFile ? `Resume with --resume-file ${resumeFile}` : 'Rerun with --resume-file to preserve continuation state.', + ); + } + const trimmed = fetchAll ? allTweets : allTweets.slice(0, limit); + return applyTopByEngagement(trimmed, topByEngagement); }, }); export const __test__ = { @@ -238,4 +474,6 @@ export const __test__ = { buildLikesUrl, buildUserByScreenNameUrl, parseLikes, + appendJsonlRows, + readResumeFile, }; diff --git a/plugins/twitter/profile.js b/plugins/twitter/profile.js index e3574f26..724990a7 100644 --- a/plugins/twitter/profile.js +++ b/plugins/twitter/profile.js @@ -12,6 +12,14 @@ function stringField(value) { return typeof value === 'string' ? value : ''; } +function countField(...values) { + for (const value of values) { + if (typeof value === 'number' && Number.isFinite(value)) + return value; + } + return 0; +} + export function mapTwitterProfileResult(result, screenName) { if (!isPlainObject(result)) { throw new CommandExecutionError(`Twitter profile response for @${screenName} is malformed`); @@ -27,18 +35,18 @@ export function mapTwitterProfileResult(result, screenName) { throw new CommandExecutionError(`Twitter profile response for @${screenName} is missing profile identity fields`); } const location = isPlainObject(result.location) ? result.location : {}; - const expandedUrl = legacy.entities?.url?.urls?.[0]?.expanded_url || ''; + const expandedUrl = stringField(result.website?.url) || stringField(legacy.entities?.url?.urls?.[0]?.expanded_url); return [{ screen_name: stringField(core.screen_name) || stringField(legacy.screen_name) || screenName, name: stringField(core.name) || stringField(legacy.name), - bio: stringField(legacy.description), + bio: stringField(result.profile_bio?.description) || stringField(legacy.description), location: stringField(location.location) || stringField(legacy.location), url: stringField(expandedUrl), - followers: legacy.followers_count || 0, - following: legacy.friends_count || 0, - tweets: legacy.statuses_count || 0, - likes: legacy.favourites_count || 0, - verified: Boolean(result.is_blue_verified || legacy.verified), + followers: countField(result.relationship_counts?.followers, legacy.followers_count, legacy.normal_followers_count), + following: countField(result.relationship_counts?.following, legacy.friends_count), + tweets: countField(result.tweet_counts?.tweets, legacy.statuses_count), + likes: countField(result.action_counts?.favorites_count, legacy.favourites_count), + verified: Boolean(result.is_blue_verified || result.verification?.verified || legacy.verified), created_at: stringField(core.created_at) || stringField(legacy.created_at), }]; } diff --git a/plugins/twitter/test/article.test.js b/plugins/twitter/test/article.test.js index 75254a2d..a5ad065d 100644 --- a/plugins/twitter/test/article.test.js +++ b/plugins/twitter/test/article.test.js @@ -192,6 +192,49 @@ describe('twitter article command', () => { } }); + it('renders atomic media using entity keys rather than entityMap array positions', async () => { + const payload = validArticlePayload({ + article: { + article_results: { + result: { + title: 'Article with image', + content_state: { + blocks: [ + { type: 'unstyled', text: 'before' }, + { type: 'atomic', text: ' ', entityRanges: [{ key: 0, offset: 0, length: 1 }] }, + { type: 'unstyled', text: 'after' }, + ], + entityMap: [ + { key: '7', value: { type: 'LINK', data: { url: 'https://example.com' } } }, + { + key: '0', + value: { + type: 'MEDIA', + data: { + caption: 'architecture diagram', + mediaItems: [{ mediaId: 'media-1' }], + }, + }, + }, + ], + }, + media_entities: [{ + media_id: 'media-1', + media_info: { original_img_url: 'https://pbs.twimg.com/media/example.jpg' }, + }], + }, + }, + }, + }); + const page = createFetchPage(async () => jsonResponse(payload)); + + await expect(command.func(page, { 'tweet-id': TWEET_ID })).resolves.toEqual([ + expect.objectContaining({ + content: 'before\n\n![architecture diagram](https://pbs.twimg.com/media/example.jpg)\n\nafter', + }), + ]); + }); + it('keeps the valid note-tweet fallback', async () => { const payload = validArticlePayload({ article: undefined, diff --git a/plugins/twitter/test/bookmarks.test.js b/plugins/twitter/test/bookmarks.test.js index 29e500e3..a4d7ddc1 100644 --- a/plugins/twitter/test/bookmarks.test.js +++ b/plugins/twitter/test/bookmarks.test.js @@ -1,4 +1,8 @@ -import { describe, expect, it } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; +import { getRegistry } from '@agentrhq/webcmd/registry'; import { __test__ } from '../bookmarks.js'; const { parseBookmarks, extractBookmarkTweet } = __test__; @@ -204,3 +208,276 @@ describe('twitter bookmarks parser', () => { expect(parseBookmarks({}, new Set())).toEqual({ tweets: [], nextCursor: null }); }); }); + +function bookmarksPayload(withBottomCursor = false) { + const entries = [{ + entryId: 'tweet-1', + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '1', + legacy: { + full_text: 'bookmarked post', + favorite_count: 3, + retweet_count: 1, + bookmark_count: 4, + created_at: 'now', + }, + core: { + user_results: { + result: { + legacy: { screen_name: 'alice', name: 'Alice' }, + }, + }, + }, + }, + }, + }, + }, + }]; + if (withBottomCursor) { + entries.push({ + entryId: 'cursor-bottom-1', + content: { + entryType: 'TimelineTimelineCursor', + cursorType: 'Bottom', + value: 'NEXT_CURSOR', + }, + }); + } + return { + data: { + bookmark_timeline_v2: { + timeline: { + instructions: [{ entries }], + }, + }, + }, + }; +} + +describe('twitter bookmarks command', () => { + it('keeps resume state and reports complete=false when --max-pages stops early', async () => { + const command = getRegistry().get('twitter/bookmarks'); + const resumeFile = path.join(os.tmpdir(), `webcmd-bookmarks-resume-${process.pid}-${Date.now()}.json`); + const outputFile = path.join(os.tmpdir(), `webcmd-bookmarks-out-${process.pid}-${Date.now()}.jsonl`); + const page = { + getCookies: vi.fn(async () => [{ name: 'ct0', value: 'token' }]), + evaluate: vi.fn(async (script) => { + const text = String(script); + if (text.includes('Bookmarks') && text.includes('queryId')) return null; + if (text.includes('/Bookmarks')) return bookmarksPayload(true); + throw new Error(`Unexpected evaluate: ${text.slice(0, 80)}`); + }), + }; + + try { + const result = await command.func(page, { + all: true, + 'max-pages': 1, + 'resume-file': resumeFile, + 'output-file': outputFile, + }); + + expect(result).toMatchObject({ + outputFile, + count: 1, + source: 'bookmarks', + complete: false, + pages: 1, + cursor: 'NEXT_CURSOR', + resumeFile, + }); + expect(fs.existsSync(resumeFile)).toBe(true); + const resume = __test__.readResumeFile(resumeFile); + expect(resume).toMatchObject({ + cursor: 'NEXT_CURSOR', + count: 1, + complete: false, + source: 'bookmarks', + outputFile, + }); + expect(fs.readFileSync(outputFile, 'utf8').trim().split('\n')).toHaveLength(1); + } + finally { + fs.rmSync(resumeFile, { force: true }); + fs.rmSync(outputFile, { force: true }); + } + }); + + it('removes resume file only after the bookmarks timeline is exhausted', async () => { + const command = getRegistry().get('twitter/bookmarks'); + const resumeFile = path.join(os.tmpdir(), `webcmd-bookmarks-resume-done-${process.pid}-${Date.now()}.json`); + const outputFile = path.join(os.tmpdir(), `webcmd-bookmarks-out-done-${process.pid}-${Date.now()}.jsonl`); + const page = { + getCookies: vi.fn(async () => [{ name: 'ct0', value: 'token' }]), + evaluate: vi.fn(async (script) => { + const text = String(script); + if (text.includes('Bookmarks') && text.includes('queryId')) return null; + if (text.includes('/Bookmarks')) return bookmarksPayload(false); + throw new Error(`Unexpected evaluate: ${text.slice(0, 80)}`); + }), + }; + + try { + const result = await command.func(page, { + all: true, + 'max-pages': 1, + 'resume-file': resumeFile, + 'output-file': outputFile, + }); + + expect(result).toMatchObject({ + outputFile, + count: 1, + source: 'bookmarks', + complete: true, + pages: 1, + }); + expect(result.cursor).toBeUndefined(); + expect(fs.existsSync(resumeFile)).toBe(false); + expect(fs.existsSync(outputFile)).toBe(true); + } + finally { + fs.rmSync(resumeFile, { force: true }); + fs.rmSync(outputFile, { force: true }); + } + }); +}); + +describe('twitter bookmarks archive safety', () => { + function pageFor(payload = bookmarksPayload(false), { wrapped = false } = {}) { + return { + getCookies: vi.fn(async () => [{ name: 'ct0', value: 'token' }]), + evaluate: vi.fn(async (script) => { + const text = String(script); + if (text.includes('operationName')) { + return wrapped ? { session: 'site:twitter', data: null } : null; + } + if (text.includes('/Bookmarks')) { + return wrapped ? { session: 'site:twitter', data: payload } : payload; + } + throw new Error(`Unexpected evaluate: ${text.slice(0, 80)}`); + }), + }; + } + + it('rejects --resume-file without --all before touching the browser', async () => { + const command = getRegistry().get('twitter/bookmarks'); + const page = { getCookies: vi.fn(), evaluate: vi.fn() }; + await expect(command.func(page, { 'resume-file': '/tmp/resume.json' })) + .rejects.toThrow(/--resume-file requires --all/); + expect(page.getCookies).not.toHaveBeenCalled(); + }); + + it('unwraps Browser Bridge envelopes for query resolution and bookmark payloads', async () => { + const command = getRegistry().get('twitter/bookmarks'); + const rows = await command.func(pageFor(bookmarksPayload(false), { wrapped: true }), { limit: 1 }); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ id: '1', author: 'alice', text: 'bookmarked post' }); + }); + + it('refuses to overwrite an existing output file without matching resume state', async () => { + const command = getRegistry().get('twitter/bookmarks'); + const outputFile = path.join(os.tmpdir(), `webcmd-bookmarks-existing-${process.pid}-${Date.now()}.jsonl`); + const resumeFile = `${outputFile}.resume.json`; + fs.writeFileSync(outputFile, 'user-owned\n'); + try { + await expect(command.func(pageFor(), { + all: true, + 'output-file': outputFile, + 'resume-file': resumeFile, + })).rejects.toThrow(/Refusing to overwrite/); + expect(fs.readFileSync(outputFile, 'utf8')).toBe('user-owned\n'); + } + finally { + fs.rmSync(outputFile, { force: true }); + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('rejects cross-source and cross-output resume state', () => { + const resumeFile = path.join(os.tmpdir(), `webcmd-bookmarks-mismatch-${process.pid}-${Date.now()}.json`); + try { + fs.writeFileSync(resumeFile, JSON.stringify({ + cursor: 'NEXT', + count: 0, + tweets: [], + complete: false, + source: 'likes', + outputFile: null, + })); + expect(() => __test__.readResumeFile(resumeFile, { + source: 'bookmarks', + outputFile: null, + })).toThrow(/source mismatch/); + fs.writeFileSync(resumeFile, JSON.stringify({ + cursor: 'NEXT', + count: 0, + complete: false, + source: 'bookmarks', + outputFile: '/tmp/other.jsonl', + })); + expect(() => __test__.readResumeFile(resumeFile, { + source: 'bookmarks', + outputFile: '/tmp/wanted.jsonl', + })).toThrow(/output mismatch/); + } + finally { + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('rejects output files whose JSONL record count differs from resume state', async () => { + const command = getRegistry().get('twitter/bookmarks'); + const outputFile = path.join(os.tmpdir(), `webcmd-bookmarks-count-mismatch-${process.pid}-${Date.now()}.jsonl`); + const resumeFile = `${outputFile}.resume.json`; + fs.writeFileSync(outputFile, '{"id":"1"}\n{"id":"2"}\n'); + fs.writeFileSync(resumeFile, JSON.stringify({ + cursor: 'NEXT', + count: 1, + complete: false, + source: 'bookmarks', + outputFile, + })); + try { + await expect(command.func(pageFor(), { + all: true, + 'output-file': outputFile, + 'resume-file': resumeFile, + })).rejects.toThrow(/expected resume count 1/); + } + finally { + fs.rmSync(outputFile, { force: true }); + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('throws for an incomplete in-memory --all run while retaining resume state', async () => { + const command = getRegistry().get('twitter/bookmarks'); + const resumeFile = path.join(os.tmpdir(), `webcmd-bookmarks-memory-${process.pid}-${Date.now()}.json`); + try { + await expect(command.func(pageFor(bookmarksPayload(true)), { + all: true, + 'max-pages': 1, + 'resume-file': resumeFile, + })).rejects.toThrow(/archive_incomplete/); + expect(__test__.readResumeFile(resumeFile)).toMatchObject({ + cursor: 'NEXT_CURSOR', + count: 1, + source: 'bookmarks', + complete: false, + }); + } + finally { + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('fails closed when the Bookmarks payload has no timeline instructions', async () => { + const command = getRegistry().get('twitter/bookmarks'); + await expect(command.func(pageFor({ data: {} }), { all: true })) + .rejects.toThrow(/missing Bookmarks timeline instructions/); + }); +}); diff --git a/plugins/twitter/test/collection.test.js b/plugins/twitter/test/collection.test.js new file mode 100644 index 00000000..64745f24 --- /dev/null +++ b/plugins/twitter/test/collection.test.js @@ -0,0 +1,276 @@ +import { describe, expect, it, vi } from 'vitest'; +import { getRegistry } from '@agentrhq/webcmd/registry'; +import { ArgumentError, CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors'; +import '../tweets.js'; +import { __test__ } from '../collection.js'; + +function syntheticTweet(id, { + author = 'synth_author', + createdAt = '2026-07-23T12:00:00.000Z', + legacy = {}, + quotedStatusResult, + retweetedStatusResult, +} = {}) { + return { + rest_id: String(id), + legacy: { + full_text: `synthetic post ${id}`, + favorite_count: 0, + retweet_count: 0, + reply_count: 0, + created_at: createdAt, + ...legacy, + }, + core: { + user_results: { + result: { + rest_id: `user-${author}`, + legacy: { screen_name: author, name: author }, + }, + }, + }, + ...(quotedStatusResult ? { quoted_status_result: quotedStatusResult } : {}), + ...(retweetedStatusResult ? { retweeted_status_result: retweetedStatusResult } : {}), + }; +} + +function tweetEntry(tweet) { + return { content: { itemContent: { tweet_results: { result: tweet } } } }; +} + +function collectionPayload(tweets, nextCursor = null) { + const entries = tweets.map(tweetEntry); + if (nextCursor) { + entries.push({ + content: { + entryType: 'TimelineTimelineCursor', + cursorType: 'Bottom', + value: nextCursor, + }, + }); + } + return { + data: { + user: { + result: { + timeline_v2: { timeline: { instructions: [{ entries }] } }, + }, + }, + }, + }; +} + +describe('twitter collection', () => { + it('registers an independent read command with posts and receipt columns', () => { + const command = getRegistry().get('twitter/collection'); + expect(command).toMatchObject({ + access: 'read', + browser: true, + columns: ['posts', 'receipt'], + }); + expect(command?.args?.map((arg) => arg.name)).toEqual([ + 'username', 'until', 'limit', 'page-delay', + ]); + expect(getRegistry().get('twitter/tweets')?.args?.map((arg) => arg.name)) + .not.toContain('collection-receipt'); + }); + + it('classifies original, quote, reply and repost without inventing context', () => { + expect(__test__.extractRelationship(syntheticTweet('10'))).toEqual({ + kind: 'original', + target: null, + }); + expect(__test__.extractRelationship(syntheticTweet('11', { + legacy: { + in_reply_to_status_id_str: '30', + in_reply_to_screen_name: 'parent_author', + in_reply_to_user_id_str: 'user-parent_author', + }, + }))).toMatchObject({ + kind: 'reply', + target: { post_id: '30', context_status: 'unavailable' }, + }); + expect(__test__.extractRelationship(syntheticTweet('12', { + legacy: { is_quote_status: true, quoted_status_id_str: '50' }, + quotedStatusResult: { result: { __typename: 'TweetTombstone' } }, + }))).toMatchObject({ + kind: 'quote', + target: { post_id: '50', context_status: 'unavailable' }, + }); + expect(__test__.extractRelationship(syntheticTweet('13', { + legacy: { retweeted_status_id_str: '60' }, + retweetedStatusResult: { result: syntheticTweet('60', { author: 'repost_target' }) }, + }))).toMatchObject({ + kind: 'repost', + target: { + post_id: '60', + author_handle: 'repost_target', + context_status: 'complete', + }, + }); + }); + + it('rejects an unresolved repost instead of inferring it from text', () => { + expect(() => __test__.extractRelationship(syntheticTweet('14', { + legacy: { full_text: 'RT @someone: synthetic', retweeted_status_id_str: '70' }, + }))).toThrow(CommandExecutionError); + expect(() => __test__.extractRelationship(syntheticTweet('14', { + legacy: { full_text: 'RT @someone: synthetic', retweeted_status_id_str: '70' }, + }))).toThrow(/twitter_collection_unresolved_relationship/); + }); + + it('accepts only RFC3339 lower boundaries', () => { + expect(__test__.normalizeUntil('2026-07-23T00:00:00Z')).toBeInstanceOf(Date); + expect(__test__.normalizeUntil('2026-07-23T00:00:00.123456Z')).toBeInstanceOf(Date); + expect(__test__.normalizeUntil('2024-02-29T00:00:00+08:00')).toBeInstanceOf(Date); + expect(() => __test__.normalizeUntil('2026-07-23')).toThrow(ArgumentError); + expect(() => __test__.normalizeUntil('not-a-date')).toThrow(ArgumentError); + expect(() => __test__.normalizeUntil('2026-02-29T00:00:00Z')).toThrow(ArgumentError); + expect(() => __test__.normalizeUntil('2026-07-23T24:00:00Z')).toThrow(ArgumentError); + expect(() => __test__.normalizeUntil('2026-07-23T00:00:00+24:00')).toThrow(ArgumentError); + }); + + it('fails closed for private, unavailable, and malformed timelines', () => { + expect(() => __test__.parseCollectionPage({ + data: { user: { result: { __typename: 'User', timeline_v2: { timeline: {} } } } }, + }, new Set())).toThrow(EmptyResultError); + expect(() => __test__.parseCollectionPage({ + data: { user: { result: { __typename: 'UserUnavailable' } } }, + }, new Set())).toThrow(/missing UserTweets timeline instructions/); + expect(() => __test__.parseCollectionPage({ + data: { user: { result: { timeline_v2: { timeline: { unexpected: true } } } } }, + }, new Set())).toThrow(/missing UserTweets timeline instructions/); + }); + + it('completes only after the lower boundary is reached', async () => { + const result = await __test__.paginateCollection({ + until: __test__.normalizeUntil('2026-07-23T00:00:00Z'), + limit: 10, + maxPages: 5, + fetchPage: async () => collectionPayload([ + syntheticTweet('20', { createdAt: '2026-07-23T01:00:00.000Z' }), + syntheticTweet('21', { createdAt: '2026-07-22T23:59:59.000Z' }), + ], 'unused-cursor'), + }); + expect(result).toMatchObject({ + posts: [ + { id: '20' }, + { id: '21' }, + ], + receipt: { + completed: true, + stop_reason: 'time_boundary_reached', + requested_until: '2026-07-23T00:00:00.000Z', + pages_fetched: 1, + oldest_seen_at: '2026-07-22T23:59:59.000Z', + }, + }); + }); + + it('completes on cursor exhaustion and exposes no cursor', async () => { + const result = await __test__.paginateCollection({ + until: __test__.normalizeUntil('2026-07-23T00:00:00Z'), + limit: 10, + maxPages: 5, + fetchPage: async () => collectionPayload([ + syntheticTweet('22', { createdAt: '2026-07-23T01:00:00.000Z' }), + ]), + }); + expect(result).toMatchObject({ + receipt: { completed: true, stop_reason: 'cursor_exhausted', pages_fetched: 1 }, + }); + expect(Object.keys(result.receipt)).not.toContain('cursor'); + }); + + it('returns the posts and receipt envelope from the registered command', async () => { + const command = getRegistry().get('twitter/collection'); + const page = { + getCookies: vi.fn(async () => [{ name: 'ct0', value: 'test-only' }]), + wait: vi.fn(async () => undefined), + evaluate: vi.fn(async (script) => { + const source = String(script); + if (source.includes('operationName')) return null; + if (source.includes('/UserByScreenName')) return '42'; + if (source.includes('/UserTweets')) { + return collectionPayload([ + syntheticTweet('27', { createdAt: '2026-07-22T23:59:59.000Z' }), + ]); + } + return null; + }), + }; + const result = await command.func(page, { + username: 'synth_author', + until: '2026-07-23T00:00:00Z', + limit: 10, + 'page-delay': 0, + }); + expect(result).toMatchObject({ + posts: [{ id: '27', relationship: { kind: 'original' } }], + receipt: { completed: true, stop_reason: 'time_boundary_reached' }, + }); + }); + + it('uses collection-specific validation errors before touching the browser', async () => { + const command = getRegistry().get('twitter/collection'); + const page = { + goto: vi.fn(), + getCookies: vi.fn(), + evaluate: vi.fn(), + }; + + await expect(command.func(page, { + username: 'home/extra', + until: '2026-07-23T00:00:00Z', + limit: 10, + 'page-delay': 0, + })).rejects.toThrow(/twitter collection username/); + expect(page.goto).not.toHaveBeenCalled(); + expect(page.getCookies).not.toHaveBeenCalled(); + expect(page.evaluate).not.toHaveBeenCalled(); + }); + + it('fails on repeated cursor, limit, page guard, and malformed timestamps', async () => { + await expect(__test__.paginateCollection({ + until: __test__.normalizeUntil('2026-07-23T00:00:00Z'), + limit: 10, + maxPages: 5, + fetchPage: async () => collectionPayload([ + syntheticTweet('23', { createdAt: '2026-07-23T01:00:00.000Z' }), + ], 'same-cursor'), + })).rejects.toThrow(/twitter_collection_repeated_cursor/); + await expect(__test__.paginateCollection({ + until: __test__.normalizeUntil('2026-07-23T00:00:00Z'), + limit: 1, + maxPages: 5, + fetchPage: async () => collectionPayload([ + syntheticTweet('24', { createdAt: '2026-07-23T01:00:00.000Z' }), + ], 'next-cursor'), + })).rejects.toThrow(/twitter_collection_limit_reached/); + await expect(__test__.paginateCollection({ + until: __test__.normalizeUntil('2026-07-23T00:00:00Z'), + limit: 1, + maxPages: 5, + fetchPage: async () => collectionPayload([ + syntheticTweet('24a', { createdAt: '2026-07-23T01:00:00.000Z' }), + syntheticTweet('24b', { createdAt: '2026-07-22T23:59:59.000Z' }), + ]), + })).rejects.toThrow(/twitter_collection_limit_reached/); + await expect(__test__.paginateCollection({ + until: __test__.normalizeUntil('2026-07-23T00:00:00Z'), + limit: 10, + maxPages: 1, + fetchPage: async () => collectionPayload([ + syntheticTweet('25', { createdAt: '2026-07-23T01:00:00.000Z' }), + ], 'next-cursor'), + })).rejects.toThrow(/twitter_collection_page_guard_hit/); + await expect(__test__.paginateCollection({ + until: __test__.normalizeUntil('2026-07-23T00:00:00Z'), + limit: 10, + maxPages: 5, + fetchPage: async () => collectionPayload([ + syntheticTweet('26', { createdAt: 'not-a-timestamp' }), + ]), + })).rejects.toThrow(/twitter_collection_invalid_timestamp/); + }); +}); diff --git a/plugins/twitter/test/likes.test.js b/plugins/twitter/test/likes.test.js index d7385f7c..7911b7ea 100644 --- a/plugins/twitter/test/likes.test.js +++ b/plugins/twitter/test/likes.test.js @@ -1,3 +1,6 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { describe, expect, it, vi } from 'vitest'; import { getRegistry } from '@agentrhq/webcmd/registry'; import { ArgumentError, AuthRequiredError, EmptyResultError } from '@agentrhq/webcmd/errors'; @@ -154,6 +157,140 @@ describe('twitter likes command', () => { }); }); +describe('twitter likes archive safety', () => { + function pageFor(payload = likesPayload()) { + return { + getCookies: vi.fn(async () => [{ name: 'ct0', value: 'token' }]), + evaluate: vi.fn(async (script) => { + const text = String(script); + if (text.includes('operationName')) return null; + if (text.includes('/UserByScreenName')) return { session: 'site:twitter', data: '42' }; + if (text.includes('/Likes')) return { session: 'site:twitter', data: payload }; + throw new Error(`Unexpected evaluate: ${text.slice(0, 80)}`); + }), + }; + } + + it('rejects --resume-file without --all before touching the browser', async () => { + const command = getRegistry().get('twitter/likes'); + const page = { getCookies: vi.fn(), evaluate: vi.fn() }; + await expect(command.func(page, { username: 'viewer', 'resume-file': '/tmp/resume.json' })) + .rejects.toThrow(/--resume-file requires --all/); + expect(page.getCookies).not.toHaveBeenCalled(); + }); + + it('refuses to overwrite an existing output file without matching resume state', async () => { + const command = getRegistry().get('twitter/likes'); + const outputFile = path.join(os.tmpdir(), `webcmd-likes-existing-${process.pid}-${Date.now()}.jsonl`); + const resumeFile = `${outputFile}.resume.json`; + fs.writeFileSync(outputFile, 'user-owned\n'); + try { + await expect(command.func(pageFor(), { + username: 'viewer', + all: true, + 'output-file': outputFile, + 'resume-file': resumeFile, + })).rejects.toThrow(/Refusing to overwrite/); + expect(fs.readFileSync(outputFile, 'utf8')).toBe('user-owned\n'); + } + finally { + fs.rmSync(outputFile, { force: true }); + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('rejects cross-source and malformed resume state instead of silently restarting', () => { + const resumeFile = path.join(os.tmpdir(), `webcmd-likes-mismatch-${process.pid}-${Date.now()}.json`); + try { + fs.writeFileSync(resumeFile, JSON.stringify({ + cursor: 'NEXT', + count: 0, + tweets: [], + complete: false, + source: 'bookmarks', + username: 'viewer', + outputFile: null, + })); + expect(() => __test__.readResumeFile(resumeFile, { + source: 'likes', + username: 'viewer', + outputFile: null, + })).toThrow(/source mismatch/); + fs.writeFileSync(resumeFile, '{broken'); + expect(() => __test__.readResumeFile(resumeFile)).toThrow(/Could not parse/); + } + finally { + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('rejects output files whose JSONL record count differs from resume state', async () => { + const command = getRegistry().get('twitter/likes'); + const outputFile = path.join(os.tmpdir(), `webcmd-likes-count-mismatch-${process.pid}-${Date.now()}.jsonl`); + const resumeFile = `${outputFile}.resume.json`; + fs.writeFileSync(outputFile, '{"id":"1"}\n{"id":"2"}\n'); + fs.writeFileSync(resumeFile, JSON.stringify({ + cursor: 'NEXT', + count: 1, + complete: false, + source: 'likes', + username: 'viewer', + outputFile, + })); + try { + await expect(command.func(pageFor(), { + username: 'viewer', + all: true, + 'output-file': outputFile, + 'resume-file': resumeFile, + })).rejects.toThrow(/expected resume count 1/); + } + finally { + fs.rmSync(outputFile, { force: true }); + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('throws for an incomplete in-memory --all run while retaining resume state', async () => { + const command = getRegistry().get('twitter/likes'); + const resumeFile = path.join(os.tmpdir(), `webcmd-likes-memory-${process.pid}-${Date.now()}.json`); + const payload = likesPayload(); + payload.data.user.result.timeline_v2.timeline.instructions[0].entries.push({ + entryId: 'cursor-bottom-1', + content: { + entryType: 'TimelineTimelineCursor', + cursorType: 'Bottom', + value: 'NEXT_CURSOR', + }, + }); + try { + await expect(command.func(pageFor(payload), { + username: 'viewer', + all: true, + 'max-pages': 1, + 'resume-file': resumeFile, + })).rejects.toThrow(/archive_incomplete/); + expect(__test__.readResumeFile(resumeFile)).toMatchObject({ + cursor: 'NEXT_CURSOR', + count: 1, + source: 'likes', + username: 'viewer', + complete: false, + }); + } + finally { + fs.rmSync(resumeFile, { force: true }); + } + }); + + it('fails closed when a non-private Likes payload has no timeline instructions', async () => { + const command = getRegistry().get('twitter/likes'); + await expect(command.func(pageFor({ + data: { user: { result: { __typename: 'UserUnavailable' } } }, + }), { username: 'viewer', all: true })).rejects.toThrow(/missing Likes timeline instructions/); + }); +}); + describe('twitter likes command', () => { it('rejects invalid explicit username before cookies or navigation', async () => { const command = getRegistry().get('twitter/likes'); @@ -217,4 +354,124 @@ describe('twitter likes command', () => { expect(decodeURIComponent(String(likesCall[0]))).toContain('"userId":"42"'); expect(decodeURIComponent(String(likesCall[0]))).not.toContain('[object Object]'); }); + + it('keeps resume state and reports complete=false when --max-pages stops early', async () => { + const command = getRegistry().get('twitter/likes'); + const resumeFile = path.join(os.tmpdir(), `webcmd-likes-resume-${process.pid}-${Date.now()}.json`); + const outputFile = path.join(os.tmpdir(), `webcmd-likes-out-${process.pid}-${Date.now()}.jsonl`); + const page = { + goto: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + getCookies: vi.fn(async () => [{ name: 'ct0', value: 'token' }]), + evaluate: vi.fn(async (script) => { + const text = String(script); + if (text.includes('AppTabBar_Profile_Link')) { + return { session: 'site:twitter', data: '/viewer' }; + } + if (text.includes('operationName')) return null; + if (text.includes('/UserByScreenName')) { + return { session: 'site:twitter', data: '42' }; + } + if (text.includes('/Likes')) { + const payload = likesPayload(); + payload.data.user.result.timeline_v2.timeline.instructions[0].entries.push({ + entryId: 'cursor-bottom-1', + content: { + entryType: 'TimelineTimelineCursor', + cursorType: 'Bottom', + value: 'NEXT_CURSOR', + }, + }); + return { session: 'site:twitter', data: payload }; + } + throw new Error(`Unexpected evaluate: ${text.slice(0, 80)}`); + }), + }; + + try { + const result = await command.func(page, { + all: true, + 'max-pages': 1, + 'resume-file': resumeFile, + 'output-file': outputFile, + }); + + expect(result).toMatchObject({ + outputFile, + count: 1, + source: 'likes', + username: 'viewer', + complete: false, + pages: 1, + cursor: 'NEXT_CURSOR', + resumeFile, + }); + expect(fs.existsSync(resumeFile)).toBe(true); + expect(fs.existsSync(outputFile)).toBe(true); + const resume = __test__.readResumeFile(resumeFile); + expect(resume).toMatchObject({ + cursor: 'NEXT_CURSOR', + count: 1, + complete: false, + source: 'likes', + username: 'viewer', + outputFile, + }); + expect(fs.readFileSync(outputFile, 'utf8').trim().split('\n')).toHaveLength(1); + } + finally { + fs.rmSync(resumeFile, { force: true }); + fs.rmSync(outputFile, { force: true }); + } + }); + + it('removes resume file only after the likes timeline is exhausted', async () => { + const command = getRegistry().get('twitter/likes'); + const resumeFile = path.join(os.tmpdir(), `webcmd-likes-resume-done-${process.pid}-${Date.now()}.json`); + const outputFile = path.join(os.tmpdir(), `webcmd-likes-out-done-${process.pid}-${Date.now()}.jsonl`); + const page = { + goto: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + getCookies: vi.fn(async () => [{ name: 'ct0', value: 'token' }]), + evaluate: vi.fn(async (script) => { + const text = String(script); + if (text.includes('AppTabBar_Profile_Link')) { + return { session: 'site:twitter', data: '/viewer' }; + } + if (text.includes('operationName')) return null; + if (text.includes('/UserByScreenName')) { + return { session: 'site:twitter', data: '42' }; + } + if (text.includes('/Likes')) { + return { session: 'site:twitter', data: likesPayload() }; + } + throw new Error(`Unexpected evaluate: ${text.slice(0, 80)}`); + }), + }; + + try { + const result = await command.func(page, { + all: true, + 'max-pages': 1, + 'resume-file': resumeFile, + 'output-file': outputFile, + }); + + expect(result).toMatchObject({ + outputFile, + count: 1, + source: 'likes', + username: 'viewer', + complete: true, + pages: 1, + }); + expect(result.cursor).toBeUndefined(); + expect(fs.existsSync(resumeFile)).toBe(false); + expect(fs.existsSync(outputFile)).toBe(true); + } + finally { + fs.rmSync(resumeFile, { force: true }); + fs.rmSync(outputFile, { force: true }); + } + }); }); diff --git a/plugins/twitter/test/profile.test.js b/plugins/twitter/test/profile.test.js index 9d2ec761..bc243de7 100644 --- a/plugins/twitter/test/profile.test.js +++ b/plugins/twitter/test/profile.test.js @@ -68,6 +68,99 @@ describe('twitter profile command', () => { }); }); + it('reads the current UserByScreenName profile containers after X removed result.legacy', () => { + // Captures the field names and containers observed in a live + // UserByScreenName response. X renamed these keys as well as moving them, + // so searching the tree for the old legacy key names cannot recover them. + const rows = __test__.mapTwitterProfileResult({ + core: { + screen_name: 'relocated_user', + name: 'Relocated User', + created_at: 'Sun Mar 20 00:00:00 +0000 2011', + }, + relationship_counts: { followers: 7100000, following: 42 }, + tweet_counts: { tweets: 128 }, + action_counts: { favorites_count: 9 }, + profile_bio: { description: 'current bio text' }, + location: { location: 'Earth' }, + website: { url: 'https://example.com' }, + verification: { verified: true }, + }, 'fallback'); + + expect(rows[0]).toMatchObject({ + screen_name: 'relocated_user', + name: 'Relocated User', + bio: 'current bio text', + location: 'Earth', + url: 'https://example.com', + followers: 7100000, + following: 42, + tweets: 128, + likes: 9, + verified: true, + created_at: 'Sun Mar 20 00:00:00 +0000 2011', + }); + }); + + it('prefers current profile containers while retaining legacy fallbacks', () => { + const rows = __test__.mapTwitterProfileResult({ + core: { screen_name: 'u', name: 'U', created_at: 'now' }, + legacy: { + description: 'old bio', + followers_count: 100, + friends_count: 10, + statuses_count: 5, + favourites_count: 2, + }, + relationship_counts: { followers: 200, following: 20 }, + tweet_counts: { tweets: 15 }, + action_counts: { favorites_count: 12 }, + profile_bio: { description: 'current bio' }, + }, 'fallback'); + + expect(rows[0]).toMatchObject({ + bio: 'current bio', + followers: 200, + following: 20, + tweets: 15, + likes: 12, + }); + }); + + it('does not read same-named values from unrelated nested entities', () => { + const rows = __test__.mapTwitterProfileResult({ + core: { screen_name: 'u', name: 'U', created_at: 'now' }, + pinned_tweet: { + relationship_counts: { followers: 999999, following: 999999 }, + action_counts: { favorites_count: 999999 }, + profile_bio: { description: 'not a user bio' }, + }, + }, 'fallback'); + + expect(rows[0]).toMatchObject({ + bio: '', + followers: 0, + following: 0, + tweets: 0, + likes: 0, + }); + }); + + it('returns 0 / empty string when a count or bio is absent everywhere', () => { + const rows = __test__.mapTwitterProfileResult({ + core: { screen_name: 'sparse', name: 'Sparse', created_at: 'now' }, + legacy: {}, + }, 'fallback'); + + expect(rows[0]).toMatchObject({ + bio: '', + followers: 0, + following: 0, + tweets: 0, + likes: 0, + }); + }); + it('throws typed when the profile result is structurally malformed', () => { expect(() => __test__.mapTwitterProfileResult(null, 'jack')).toThrow(CommandExecutionError); expect(() => __test__.mapTwitterProfileResult([], 'jack')).toThrow(CommandExecutionError); diff --git a/plugins/twitter/user-timeline.js b/plugins/twitter/user-timeline.js new file mode 100644 index 00000000..88eebe92 --- /dev/null +++ b/plugins/twitter/user-timeline.js @@ -0,0 +1,217 @@ +import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@agentrhq/webcmd/errors'; +import { resolveTwitterOperationMetadata, normalizeTwitterGraphqlPayload, unwrapBrowserResult, normalizeTwitterScreenName } from './shared.js'; +import { TWITTER_BEARER_TOKEN } from './utils.js'; + +const USER_TWEETS_QUERY_ID = 'lrMzG9qPQHpqJdP3AbM-bQ'; +const USER_BY_SCREEN_NAME_QUERY_ID = 'IGgvgiOx4QZndDHuD3x9TQ'; + +export const MAX_USER_TWEETS_PAGES = 100; +export const USER_TWEETS_PAGE_SIZE = 100; +export const MAX_USER_TWEETS_LIMIT = MAX_USER_TWEETS_PAGES * USER_TWEETS_PAGE_SIZE; +export const DEFAULT_USER_TWEETS_PAGE_DELAY_SECONDS = 2; + +const USER_TWEETS_FEATURES = { + rweb_video_screen_enabled: true, + rweb_cashtags_enabled: true, + payments_enabled: false, + profile_label_improvements_pcf_label_in_post_enabled: true, + responsive_web_profile_redirect_enabled: true, + rweb_tipjar_consumption_enabled: true, + verified_phone_label_enabled: false, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + premium_content_api_read_enabled: false, + communities_web_enable_tweet_community_results_fetch: true, + c9s_tweet_anatomy_moderator_badge_enabled: true, + responsive_web_grok_analyze_button_fetch_trends_enabled: false, + responsive_web_grok_analyze_post_followups_enabled: true, + rweb_cashtags_composer_attachment_enabled: true, + responsive_web_jetfuel_frame: true, + responsive_web_grok_share_attachment_enabled: true, + responsive_web_grok_annotations_enabled: true, + articles_preview_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: true, + tweet_awards_web_tipping_enabled: false, + content_disclosure_indicator_enabled: true, + content_disclosure_ai_generated_indicator_enabled: true, + responsive_web_grok_show_grok_translated_post: false, + responsive_web_grok_analysis_button_from_backend: true, + post_ctas_fetch_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_grok_image_annotation_enabled: true, + responsive_web_grok_imagine_annotation_enabled: true, + responsive_web_grok_community_note_auto_translation_is_enabled: false, + responsive_web_enhance_cards_enabled: false, +}; + +const USER_TWEETS_FIELD_TOGGLES = { + withPayments: true, + withAuxiliaryUserLabels: true, + withArticleRichContentState: true, + withArticlePlainText: true, + withArticleSummaryText: true, + withArticleVoiceOver: true, + withGrokAnalyze: true, + withDisallowedReplyControls: true, +}; + +const USER_BY_SCREEN_NAME_FEATURES = { + hidden_profile_subscriptions_enabled: true, + profile_label_improvements_pcf_label_in_post_enabled: true, + responsive_web_profile_redirect_enabled: true, + rweb_tipjar_consumption_enabled: true, + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + subscriptions_verification_info_is_identity_verified_enabled: true, + subscriptions_verification_info_verified_since_enabled: true, + highlights_tweets_tab_ui_enabled: true, + responsive_web_twitter_article_notes_tab_enabled: true, + subscriptions_feature_can_gift_premium: true, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + responsive_web_graphql_timeline_navigation_enabled: true, +}; + +const USER_BY_SCREEN_NAME_FIELD_TOGGLES = { + withPayments: true, + withAuxiliaryUserLabels: true, +}; + +const USER_TWEETS_OPERATION = { + queryId: USER_TWEETS_QUERY_ID, + features: USER_TWEETS_FEATURES, + fieldToggles: USER_TWEETS_FIELD_TOGGLES, +}; + +const USER_BY_SCREEN_NAME_OPERATION = { + queryId: USER_BY_SCREEN_NAME_QUERY_ID, + features: USER_BY_SCREEN_NAME_FEATURES, + fieldToggles: USER_BY_SCREEN_NAME_FIELD_TOGGLES, +}; + +function normalizeUserTweetsOperation(operation) { + if (typeof operation === 'string') { + return { queryId: operation, features: USER_TWEETS_FEATURES, fieldToggles: USER_TWEETS_FIELD_TOGGLES }; + } + return { + queryId: operation?.queryId || USER_TWEETS_QUERY_ID, + features: operation?.features || USER_TWEETS_FEATURES, + fieldToggles: operation?.fieldToggles || USER_TWEETS_FIELD_TOGGLES, + }; +} + +function normalizeUserByScreenNameOperation(operation) { + if (typeof operation === 'string') { + return { queryId: operation, features: USER_BY_SCREEN_NAME_FEATURES, fieldToggles: USER_BY_SCREEN_NAME_FIELD_TOGGLES }; + } + return { + queryId: operation?.queryId || USER_BY_SCREEN_NAME_QUERY_ID, + features: operation?.features || USER_BY_SCREEN_NAME_FEATURES, + fieldToggles: operation?.fieldToggles || USER_BY_SCREEN_NAME_FIELD_TOGGLES, + }; +} + +function appendGraphqlParams(path, variables, operation) { + const fieldToggles = operation.fieldToggles || {}; + const params = [ + `variables=${encodeURIComponent(JSON.stringify(variables))}`, + `features=${encodeURIComponent(JSON.stringify(operation.features || {}))}`, + ]; + if (Object.keys(fieldToggles).length > 0) { + params.push(`fieldToggles=${encodeURIComponent(JSON.stringify(fieldToggles))}`); + } + return `${path}?${params.join('&')}`; +} + +export function buildUserTweetsUrl(operation, userId, count, cursor) { + const normalized = normalizeUserTweetsOperation(operation); + const vars = { + userId, + count, + includePromotedContent: false, + withQuickPromoteEligibilityTweetFields: true, + withVoice: true, + }; + if (cursor) vars.cursor = cursor; + return appendGraphqlParams(`/i/api/graphql/${normalized.queryId}/UserTweets`, vars, normalized); +} + +export function buildUserByScreenNameUrl(operation, screenName) { + const normalized = normalizeUserByScreenNameOperation(operation); + const vars = { screen_name: screenName, withSafetyModeUserFields: true }; + return appendGraphqlParams(`/i/api/graphql/${normalized.queryId}/UserByScreenName`, vars, normalized); +} + +export async function resolveUserTimelineContext( + page, + rawUsername, + { allowLoggedInDefault = false, commandName = 'tweets' } = {}, +) { + const raw = String(rawUsername ?? '').trim(); + let username = normalizeTwitterScreenName(raw); + if (raw && !username) { + throw new ArgumentError( + `twitter ${commandName} username must be a valid Twitter/X handle`, + commandName === 'collection' + ? 'Example: webcmd twitter collection @jack --until 2026-07-23T00:00:00Z' + : 'Example: webcmd twitter tweets @jack --limit 20', + ); + } + if (!username && !allowLoggedInDefault) { + throw new ArgumentError('twitter collection username must be a valid Twitter/X handle', 'Example: webcmd twitter collection @jack --until 2026-07-23T00:00:00Z'); + } + if (!username) { + await page.goto('https://x.com/home'); + await page.wait({ selector: '[data-testid="primaryColumn"]' }); + const href = unwrapBrowserResult(await page.evaluate(`() => { + const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]'); + return link ? link.getAttribute('href') : null; + }`)); + if (!href || typeof href !== 'string') { + throw new AuthRequiredError('x.com', 'Could not detect logged-in user. Are you logged in?'); + } + username = normalizeTwitterScreenName(href); + if (!username) { + throw new AuthRequiredError('x.com', 'Could not detect logged-in user. Are you logged in?'); + } + } + + const cookies = await page.getCookies({ url: 'https://x.com' }); + const ct0 = cookies.find((cookie) => cookie.name === 'ct0')?.value || null; + if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)'); + + const userTweetsOperation = await resolveTwitterOperationMetadata(page, 'UserTweets', USER_TWEETS_OPERATION); + const userByScreenNameOperation = await resolveTwitterOperationMetadata(page, 'UserByScreenName', USER_BY_SCREEN_NAME_OPERATION); + const headers = JSON.stringify({ + Authorization: `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, + 'X-Csrf-Token': ct0, + 'X-Twitter-Auth-Type': 'OAuth2Session', + 'X-Twitter-Active-User': 'yes', + }); + const userByScreenNameUrl = buildUserByScreenNameUrl(userByScreenNameOperation, username); + const userId = unwrapBrowserResult(await page.evaluate(`async () => { + const resp = await fetch(${JSON.stringify(userByScreenNameUrl)}, { headers: ${headers}, credentials: 'include' }); + if (!resp.ok) return null; + const data = await resp.json(); + return data?.data?.user?.result?.rest_id || null; + }`)); + if (!userId) throw new CommandExecutionError(`Could not resolve @${username}`); + return { username, userId, headers, userTweetsOperation }; +} + +export async function fetchUserTimelinePage(page, context, cursor, count) { + const url = buildUserTweetsUrl(context.userTweetsOperation, context.userId, count, cursor); + return normalizeTwitterGraphqlPayload(await page.evaluate(`async () => { + const response = await fetch(${JSON.stringify(url)}, { headers: ${context.headers}, credentials: 'include' }); + return response.ok ? await response.json() : { error: response.status }; + }`)); +} diff --git a/scripts/check-plugin-command-parity.mjs b/scripts/check-plugin-command-parity.mjs index 610f2f81..cefb15c1 100644 --- a/scripts/check-plugin-command-parity.mjs +++ b/scripts/check-plugin-command-parity.mjs @@ -10,10 +10,15 @@ const core = read('cli-manifest.json'); const frozen = read('test/fixtures/core-cli-manifest-v0.5.3.json'); const coreKeys = new Set(core.map(key)); const pluginByKey = new Map(plugins.map(entry => [key(entry), entry])); +// `args` is checked separately: a migrated command may still gain new optional +// flags, so strict deep-equality would forbid ordinary feature work rather than +// the regression this guards against. const fields = [ - 'aliases', 'access', 'domain', 'strategy', 'browser', 'args', 'columns', 'tags', 'keywords', + 'aliases', 'access', 'domain', 'strategy', 'browser', 'columns', 'tags', 'keywords', 'defaultFormat', 'pipeline', 'navigateBefore', 'siteSession', 'freshPage', ]; +// Help text documents an argument; it does not change how one is invoked. +const argFields = ['name', 'type', 'default', 'required', 'positional']; const issues = []; for (const expected of frozen) { @@ -27,6 +32,7 @@ for (const expected of frozen) { if (JSON.stringify(pick(actual)) !== JSON.stringify(pick(expected))) { issues.push(`${command} executable metadata differs from frozen core manifest`); } + issues.push(...argIssues(command, actual.args ?? [], expected.args ?? [])); } if (issues.length) { @@ -47,3 +53,32 @@ function key(entry) { function pick(entry) { return Object.fromEntries(fields.map(field => [field, entry[field]])); } + +function pickArg(arg) { + return Object.fromEntries(argFields.map(field => [field, arg[field]])); +} + +// Every frozen argument must still be invocable exactly as it was in v0.5.3: +// present, same type / default / required / positional, and in the same relative +// order so positional invocations keep working. New arguments may be added +// around them, and help text may be reworded. +function argIssues(command, actual, expected) { + const found = []; + const actualByName = new Map(actual.map(arg => [arg.name, arg])); + for (const arg of expected) { + const match = actualByName.get(arg.name); + if (!match) { + found.push(`${command} dropped the frozen argument --${arg.name}`); + continue; + } + if (JSON.stringify(pickArg(match)) !== JSON.stringify(pickArg(arg))) { + found.push(`${command} changed how --${arg.name} is invoked`); + } + } + const frozenOrder = expected.map(arg => arg.name); + const actualOrder = actual.map(arg => arg.name).filter(name => frozenOrder.includes(name)); + if (JSON.stringify(actualOrder) !== JSON.stringify(frozenOrder.filter(n => actualByName.has(n)))) { + found.push(`${command} reordered the frozen arguments`); + } + return found; +} diff --git a/scripts/typed-error-lint-baseline.json b/scripts/typed-error-lint-baseline.json index 5d8ec0f2..81287d69 100644 --- a/scripts/typed-error-lint-baseline.json +++ b/scripts/typed-error-lint-baseline.json @@ -327,14 +327,6 @@ "text": "const fetchCount = Math.min(100, limit - allTweets.length + 10);", "occurrence": 0 }, - { - "rule": "silent-clamp", - "command": "twitter/bookmarks", - "file": "plugins/twitter/bookmarks.js", - "line": 157, - "text": "const fetchCount = Math.min(100, limit - allTweets.length + 10);", - "occurrence": 0 - }, { "rule": "silent-clamp", "command": "twitter/following", @@ -343,14 +335,6 @@ "text": "const fetchCount = Math.min(50, limit - allUsers.length + 10);", "occurrence": 0 }, - { - "rule": "silent-clamp", - "command": "twitter/likes", - "file": "plugins/twitter/likes.js", - "line": 208, - "text": "const fetchCount = Math.min(100, limit - allTweets.length + 10);", - "occurrence": 0 - }, { "rule": "silent-clamp", "command": "twitter/list-tweets", @@ -491,7 +475,7 @@ "rule": "silent-sentinel", "command": "twitter/bookmarks", "file": "plugins/twitter/bookmarks.js", - "line": 55, + "line": 59, "text": "const screenName = user?.legacy?.screen_name || user?.core?.screen_name || 'unknown';", "occurrence": 0 }, @@ -499,7 +483,7 @@ "rule": "silent-sentinel", "command": "twitter/likes", "file": "plugins/twitter/likes.js", - "line": 91, + "line": 95, "text": "const screenName = user?.legacy?.screen_name || user?.core?.screen_name || 'unknown';", "occurrence": 0 }, diff --git a/src/browser/cdp.ts b/src/browser/cdp.ts index cbe89a54..5291e2cd 100644 --- a/src/browser/cdp.ts +++ b/src/browser/cdp.ts @@ -401,7 +401,7 @@ class CDPPage extends BasePage { } if (level === 'all') return [...this._consoleMessages]; // 'error' level includes both console.error() and uncaught exceptions - if (level === 'error') return this._consoleMessages.filter(m => m.type === 'error' || m.type === 'warning'); + if (level === 'error') return this._consoleMessages.filter(m => m.type === 'error'); return this._consoleMessages.filter(m => m.type === level); } diff --git a/src/download/article-download.test.ts b/src/download/article-download.test.ts index 264e22e1..0431feeb 100644 --- a/src/download/article-download.test.ts +++ b/src/download/article-download.test.ts @@ -70,6 +70,28 @@ describe('downloadArticle', () => { expect(md).toMatch(/\|\s*1\s*\|\s*2\s*\|/); }); + // A caption is the only child that survives inside a row-less table: the parser + // foster-parents any other content out of the table before turndown sees it. + it('keeps the text of a table that carries no rows instead of failing the download', async () => { + const md = await runAndRead( + '

before

' + + '
orphan cell
' + + '

after

', + ); + expect(md).toContain('orphan cell'); + expect(md).toContain('before'); + expect(md).toContain('after'); + }); + + it('drops an empty table and still converts a real one beside it', async () => { + const md = await runAndRead( + '
' + + '
a
1
', + ); + expect(md).toMatch(/\|\s*a\s*\|/); + expect(md).toMatch(/\|\s*1\s*\|/); + }); + it('converts strikethrough and task lists', async () => { const md = await runAndRead( '

gone

' + diff --git a/src/download/article-download.ts b/src/download/article-download.ts index 3cc6f6cd..1d2dd0e7 100644 --- a/src/download/article-download.ts +++ b/src/download/article-download.ts @@ -110,6 +110,15 @@ function createTurndown( }); td.use(gfm); td.remove(STRIPPED_TAGS); + // turndown-plugin-gfm@1.0.2 reads `table.rows[0].parentNode` from both its + // table rule and its keep filter, so a table carrying no `tr` throws before + // either can decide, taking the whole document with it. Claiming those tables + // here bypasses both: `addRule` unshifts onto the rule array, and `forNode` + // exhausts that array before it consults the keep list. + td.addRule('rowlessTable', { + filter: (node) => node.nodeName === 'TABLE' && !(node as HTMLTableElement).rows?.length, + replacement: (content) => (content.trim() ? `\n\n${content.trim()}\n\n` : ''), + }); // turndown-plugin-gfm@1.0.2 emits single-tilde strikethrough (`~x~`), which // is not the canonical GFM form. Override it so exported markdown is // portable across common renderers. diff --git a/src/hosted/runner.test.ts b/src/hosted/runner.test.ts index f5823b90..44745841 100644 --- a/src/hosted/runner.test.ts +++ b/src/hosted/runner.test.ts @@ -1356,10 +1356,10 @@ describe('runHostedCli', () => { expected: 'username\n"a,""b\nline 2"\n', }, { - name: 'literal Markdown cells', + name: 'escaped Markdown cells', result: [{ username: 'a|b\nline 2' }], argv: ['-f', 'md'], - expected: '| username |\n| --- |\n| a|b\nline 2 |\n', + expected: '| username |\n| --- |\n| a\\|b\nline 2 |\n', }, ])('renders hosted $name with canonical literal bytes', async ({ result, argv, expected }) => { const stdout = sink(true); diff --git a/src/output.test.ts b/src/output.test.ts index caeeb43d..ea0f5d57 100644 --- a/src/output.test.ts +++ b/src/output.test.ts @@ -45,11 +45,11 @@ describe('formatOutput', () => { ); }); - it('preserves the legacy literal Markdown table cells', () => { + it('escapes pipe characters in Markdown table cells', () => { expect(formatOutput([ { name: 'a|b', note: 'line 1\nline 2' }, ], { fmt: 'md', fmtExplicit: true, columns: ['name', 'note'], isTTY: false })).toBe( - '| name | note |\n| --- | --- |\n| a|b | line 1\nline 2 |\n', + '| name | note |\n| --- | --- |\n| a\\|b | line 1\nline 2 |\n', ); }); diff --git a/src/output.ts b/src/output.ts index f54deeda..bd2f2776 100644 --- a/src/output.ts +++ b/src/output.ts @@ -162,7 +162,7 @@ function formatMarkdown(data: unknown, opts: RenderOptions): string { const output = [ `| ${columns.join(' | ')} |`, `| ${columns.map(() => '---').join(' | ')} |`, - ...rows.map(row => `| ${columns.map(column => String(row[column] ?? '')).join(' | ')} |`), + ...rows.map(row => `| ${columns.map(column => String(row[column] ?? '').replace(/\|/g, '\\|')).join(' | ')} |`), ]; return `${output.join('\n')}\n`; }