diff --git a/actors/apify_rag-web-browser/.actor/input_schema.json b/actors/apify_rag-web-browser/.actor/input_schema.json index cb62402..8757a8c 100644 --- a/actors/apify_rag-web-browser/.actor/input_schema.json +++ b/actors/apify_rag-web-browser/.actor/input_schema.json @@ -75,7 +75,7 @@ "proxyConfiguration": { "title": "Proxy configuration", "type": "object", - "description": "Apify Proxy configuration used for scraping the target web pages.", + "description": "Apify Proxy configuration used for scraping the target web pages. In Standby mode this applies to the shared crawlers started with the Actor, so it cannot be set per request.", "default": { "useApifyProxy": true }, @@ -119,8 +119,8 @@ "desiredConcurrency": { "title": "Desired browsing concurrency", "type": "integer", - "description": "The desired number of web browsers running in parallel. The system automatically scales the number based on the CPU and memory usage. If the initial value is `0`, the Actor picks the number automatically based on the available memory.", - "minimum": 0, + "description": "The desired number of web browsers running in parallel. The system automatically scales the number based on the CPU and memory usage. In Standby mode this applies to the shared crawlers started with the Actor, so it cannot be set per request.", + "minimum": 1, "maximum": 50, "default": 5, "editor": "hidden" diff --git a/actors/apify_url-to-markdown/.actor/input_schema.json b/actors/apify_url-to-markdown/.actor/input_schema.json index e0573a0..57d3eaf 100644 --- a/actors/apify_url-to-markdown/.actor/input_schema.json +++ b/actors/apify_url-to-markdown/.actor/input_schema.json @@ -30,7 +30,7 @@ "sectionCaption": "Scraping settings", "title": "Proxy configuration", "type": "object", - "description": "Apify Proxy configuration used for scraping the target web pages.", + "description": "Apify Proxy configuration used for scraping the target web pages. In Standby mode this applies to the shared crawlers started with the Actor, so it cannot be set per request.", "default": { "useApifyProxy": true }, @@ -50,8 +50,8 @@ "desiredConcurrency": { "title": "Desired browsing concurrency", "type": "integer", - "description": "The desired number of web browsers running in parallel. The system automatically scales the number based on the CPU and memory usage. If the initial value is `0`, the Actor picks the number automatically based on the available memory.", - "minimum": 0, + "description": "The desired number of web browsers running in parallel. The system automatically scales the number based on the CPU and memory usage. In Standby mode this applies to the shared crawlers started with the Actor, so it cannot be set per request.", + "minimum": 1, "maximum": 50, "default": 1, "editor": "hidden" diff --git a/docs/standby-openapi-3.1.0.json b/docs/standby-openapi-3.1.0.json index a1cdfc4..eb8e955 100644 --- a/docs/standby-openapi-3.1.0.json +++ b/docs/standby-openapi-3.1.0.json @@ -88,17 +88,6 @@ "default": 2 } }, - { - "name": "proxyConfiguration", - "in": "query", - "description": "Apify Proxy configuration used for scraping the target web pages, as a JSON-encoded object string.", - "required": false, - "schema": { - "type": "string", - "default": "{\"useApifyProxy\": true}" - }, - "example": "{\"useApifyProxy\": true}" - }, { "name": "scrapingTool", "in": "query", diff --git a/docs/url-to-markdown-standby-openapi-3.1.0.json b/docs/url-to-markdown-standby-openapi-3.1.0.json index 3da3df7..e583b89 100644 --- a/docs/url-to-markdown-standby-openapi-3.1.0.json +++ b/docs/url-to-markdown-standby-openapi-3.1.0.json @@ -40,17 +40,6 @@ ], "default": "raw-http" } - }, - { - "name": "proxyConfiguration", - "in": "query", - "description": "Apify Proxy configuration used for scraping the target web pages, as a JSON-encoded object string.", - "required": false, - "schema": { - "type": "string", - "default": "{\"useApifyProxy\": true}" - }, - "example": "{\"useApifyProxy\": true}" } ], "responses": { diff --git a/src/const.ts b/src/const.ts index 7c98f83..0745445 100644 --- a/src/const.ts +++ b/src/const.ts @@ -22,4 +22,8 @@ export type CrawlerKind = 'search' | ContentCrawlerTypes; export const PLAYWRIGHT_REQUEST_TIMEOUT_NORMAL_MODE_SECS = 60; +// The widest values the input schema allows; each request narrows them down to its own. +export const CRAWLER_MAX_REQUEST_RETRIES = 5; +export const CRAWLER_REQUEST_HANDLER_TIMEOUT_SECS = 300; + export const GOOGLE_STANDARD_RESULTS_PER_PAGE = 10; diff --git a/src/crawlers.ts b/src/crawlers.ts index b0c8ccc..ca579fe 100644 --- a/src/crawlers.ts +++ b/src/crawlers.ts @@ -32,7 +32,11 @@ import type { } from './types.js'; import { addTimeMeasureEvent, createRequest, createSearchRequest, isActorStandby, randomId } from './utils.js'; -const crawlers = new Map(); +type ContentCrawler = CheerioCrawler | PlaywrightCrawler; + +// Pending rather than built, so concurrent requests for one key don't each build a crawler and +// orphan all but the last. +const crawlers = new Map>(); const client = new MemoryStorage({ persistStorage: false }); const contentCrawlerHttpClient = new ImpitHttpClient({ @@ -97,27 +101,53 @@ function canonicalJson(value: unknown): string { } /** - * Identifies a crawler in the `crawlers` cache. Listed are the only options the builders in - * `input.ts` derive from the input; the proxy options stand in for the constructed - * `ProxyConfiguration`, whose child logger snapshots the log level and so kept changing the key. - * Hashed because the key is logged and used as a queue name, while the options can hold credentials. + * The proxy options stand in for the constructed `ProxyConfiguration`, whose child logger snapshots + * the log level and so kept changing the key. Hashed because the key is logged and used as a queue + * name, while the options can hold credentials. */ -export function getCrawlerKey( - kind: CrawlerKind, - crawlerOptions: CheerioCrawlerOptions | PlaywrightCrawlerOptions, - proxyOptions: ProxyOptions, -): string { - const fingerprint = { - keepAlive: crawlerOptions.keepAlive, - maxRequestRetries: crawlerOptions.maxRequestRetries, - requestHandlerTimeoutSecs: crawlerOptions.requestHandlerTimeoutSecs, - desiredConcurrency: crawlerOptions.autoscaledPoolOptions?.desiredConcurrency, - proxy: resolveProxyOptions(proxyOptions), - }; - const hash = createHash('sha1').update(canonicalJson(fingerprint)).digest('hex'); +export function getCrawlerCount() { + return crawlers.size; +} + +export function getCrawlerKey(kind: CrawlerKind, proxyOptions: ProxyOptions): string { + const hash = createHash('sha1').update(canonicalJson(resolveProxyOptions(proxyOptions))).digest('hex'); return `${kind}-${hash.slice(0, 12)}`; } +/** A crawler that fails to build or to run is dropped, so the cache can never hand out a dead one. */ +async function getOrCreateCrawler( + key: string, + kind: CrawlerKind, + startCrawler: boolean, + build: () => Promise, +): Promise { + const cached = crawlers.get(key); + if (cached) { + return cached; + } + + const pending = (async () => { + log.info(`Creating new ${kind} crawler with key ${key}`); + const crawler = await build(); + if (startCrawler) { + crawler.run().then( + () => log.warning(`Crawler ${kind} has finished`), + (err) => { + log.error(`Crawler ${kind} failed to run: ${err instanceof Error ? err.message : String(err)}`); + crawlers.delete(key); + }, + ); + log.info(`Crawler ${kind} has started 💪🏼`); + } + return crawler; + })(); + + crawlers.set(key, pending); + pending.catch(() => crawlers.delete(key)); + log.info(`Number of crawlers ${crawlers.size}`); + return pending; +} + /** * Adds a content crawl request to selected content crawler. * Get existing crawler based on crawlerOptions and scraperSettings, if not present -> create new @@ -127,13 +157,14 @@ export const addContentCrawlRequest = async ( responseId: string, contentCrawlerKey: string, ) => { - const crawler = crawlers.get(contentCrawlerKey); - const name = crawler instanceof PlaywrightCrawler ? 'playwright' : 'cheerio'; - - if (!crawler) { + const pending = crawlers.get(contentCrawlerKey); + if (!pending) { log.error(`Content crawler not found: key ${contentCrawlerKey}`); return; } + + const crawler = await pending; + const name = crawler instanceof PlaywrightCrawler ? 'playwright' : 'cheerio'; try { await crawler.requestQueue!.addRequest(request); // create an empty result in search request response @@ -154,13 +185,8 @@ export async function createAndStartSearchCrawler( startCrawler = true, ) { const { crawlerOptions, proxyOptions } = searchCrawlerOptions; - const key = getCrawlerKey('search', crawlerOptions, proxyOptions); - if (crawlers.has(key)) { - return { key, crawler: crawlers.get(key) }; - } - - log.info(`Creating new cheerio crawler with key ${key}`); - const crawler = new CheerioCrawler({ + const key = getCrawlerKey('search', proxyOptions); + const crawler = await getOrCreateCrawler(key, 'search', startCrawler, async () => new CheerioCrawler({ ...crawlerOptions, requestQueue: await RequestQueue.open(key, { storageClient: client }), requestHandler: async ({ request, $: _$, addRequests }: CheerioCrawlingContext) => { @@ -232,17 +258,8 @@ export async function createAndStartSearchCrawler( const errorResponse = { errorMessage: err.message }; sendResponseError(request.uniqueKey, JSON.stringify(errorResponse)); }, - }); - if (startCrawler) { - crawler.run().then( - () => log.warning('Google-search-crawler has finished'), - // eslint-disable-next-line @typescript-eslint/no-empty-function - () => { }, - ); - log.info('Google-search-crawler has started 🫡'); - } - crawlers.set(key, crawler); - log.info(`Number of crawlers ${crawlers.size}`); + })); + return { key, crawler }; } @@ -257,25 +274,13 @@ export async function createAndStartContentCrawler( ) { const { type: crawlerType, crawlerOptions, proxyOptions } = contentCrawlerOptions; - const key = getCrawlerKey(crawlerType, crawlerOptions, proxyOptions); - if (crawlers.has(key)) { - return { key, crawler: crawlers.get(key) }; - } + const key = getCrawlerKey(crawlerType, proxyOptions); + const crawler = await getOrCreateCrawler(key, crawlerType, startCrawler, async () => ( + crawlerType === ContentCrawlerTypes.PLAYWRIGHT + ? createPlaywrightContentCrawler(crawlerOptions, key) + : createCheerioContentCrawler(crawlerOptions, key) + )); - const crawler = crawlerType === 'playwright' - ? await createPlaywrightContentCrawler(crawlerOptions, key) - : await createCheerioContentCrawler(crawlerOptions, key); - - if (startCrawler) { - crawler.run().then( - () => log.warning(`Crawler ${crawlerType} has finished`), - // eslint-disable-next-line @typescript-eslint/no-empty-function - () => { }, - ); - log.info(`Crawler ${crawlerType} has started 💪🏼`); - } - crawlers.set(key, crawler); - log.info(`Number of crawlers ${crawlers.size}`); return { key, crawler }; } @@ -288,7 +293,6 @@ async function createPlaywrightContentCrawler( crawlerOptions: PlaywrightCrawlerOptions, key: string, ): Promise { - log.info(`Creating new playwright crawler with key ${key}`); const blocker = await getGhosteryBlocker(); return new PlaywrightCrawler({ ...crawlerOptions, @@ -309,7 +313,6 @@ async function createCheerioContentCrawler( crawlerOptions: CheerioCrawlerOptions, key: string, ): Promise { - log.info(`Creating new cheerio crawler with key ${key}`); return new CheerioCrawler({ ...crawlerOptions, keepAlive: crawlerOptions.keepAlive, @@ -397,12 +400,13 @@ export const addSearchRequest = async ( request: RequestOptions, searchCrawlerKey: string, ) => { - const crawler = crawlers.get(searchCrawlerKey); - - if (!crawler) { + const pending = crawlers.get(searchCrawlerKey); + if (!pending) { log.error(`Search crawler not found: key ${searchCrawlerKey}`); return; } + + const crawler = await pending; addTimeMeasureEvent(request.userData!, 'before-cheerio-queue-add'); await crawler.requestQueue!.addRequest(request); log.info(`Added request to cheerio-google-search-crawler: ${request.url}`); diff --git a/src/input.ts b/src/input.ts index 1c1b555..c6e244d 100644 --- a/src/input.ts +++ b/src/input.ts @@ -5,12 +5,14 @@ import { BrowserName, log } from 'crawlee'; import { firefox } from 'playwright'; import ragWebBrowserInputSchema from '../actors/apify_rag-web-browser/.actor/input_schema.json' with { type: 'json' }; -import { ContentCrawlerTypes } from './const.js'; +import { ContentCrawlerTypes, CRAWLER_MAX_REQUEST_RETRIES, CRAWLER_REQUEST_HANDLER_TIMEOUT_SECS } from './const.js'; import { UserInputError } from './errors.js'; import { blockMediaRequests } from './media.js'; import { getMiniActor } from './mini-actors.js'; import type { + CommonInput, ContentCrawlerOptions, + ContentCrawlerUserData, ContentScraperSettings, Input, OutputFormats, @@ -21,6 +23,11 @@ import type { SERPProxyGroup, UrlToMarkdownInput, } from './types.js'; +import { requestTimeoutMillis } from './utils.js'; + +// Captured at boot so that requests resolve these the way the crawlers were built, rather than +// falling back to the schema defaults and missing them. +let crawlerInput: Pick | undefined; /** * Processes the input and returns an array of crawler settings. This is ideal for startup of STANDBY mode @@ -28,6 +35,7 @@ import type { */ export async function processStandbyInput(originalInput: Partial) { const { input, searchCrawlerOptions, contentScraperSettings } = await processInputInternal(originalInput, true); + crawlerInput = { desiredConcurrency: input.desiredConcurrency, proxyConfiguration: input.proxyConfiguration }; const proxy = await Actor.createProxyConfiguration(input.proxyConfiguration); const contentCrawlerOptions: ContentCrawlerOptions[] = [ @@ -63,6 +71,8 @@ async function processInputInternal( let input: Input; let searchCrawlerOptions: SearchCrawlerOptions = { crawlerOptions: {}, proxyOptions: {} }; + Object.assign(originalInput, crawlerInput); + if (miniActor.runsSearch) { const processedRagWebBrowserInput = await processRagWebBrowserInput( originalInput as Partial, standbyInit); @@ -89,6 +99,8 @@ async function processInputInternal( outputFormats, removeCookieWarnings, removeElementsCssSelector, + requestTimeoutSecs: input.requestTimeoutSecs, + maxRequestRetries: input.maxRequestRetries, }; return { input, searchCrawlerOptions, contentScraperSettings }; @@ -171,7 +183,7 @@ async function processRagWebBrowserInput(input: Partial, sta const searchCrawlerOptions: SearchCrawlerOptions = { crawlerOptions: { keepAlive: standbyInit, - maxRequestRetries: input.serpMaxRetries, + maxRequestRetries: CRAWLER_MAX_REQUEST_RETRIES, proxyConfiguration: proxySearch, autoscaledPoolOptions: { desiredConcurrency: 1 }, }, @@ -219,17 +231,15 @@ function createPlaywrightCrawlerOptions( proxy: ProxyConfiguration | undefined, keepAlive = true, ): ContentCrawlerOptions { - const { maxRequestRetries, desiredConcurrency } = input; - return { type: ContentCrawlerTypes.PLAYWRIGHT, proxyOptions: input.proxyConfiguration, crawlerOptions: { headless: true, keepAlive, - maxRequestRetries, + maxRequestRetries: CRAWLER_MAX_REQUEST_RETRIES, proxyConfiguration: proxy, - requestHandlerTimeoutSecs: input.requestTimeoutSecs, + requestHandlerTimeoutSecs: CRAWLER_REQUEST_HANDLER_TIMEOUT_SECS, launchContext: { launcher: firefox, }, @@ -237,9 +247,14 @@ function createPlaywrightCrawlerOptions( async ({ page }) => { await blockMediaRequests(page); }, - (_context, gotoOptions) => { + ({ request }, gotoOptions) => { // eslint-disable-next-line no-param-reassign gotoOptions.waitUntil = 'domcontentloaded'; + // eslint-disable-next-line no-param-reassign + gotoOptions.timeout = Math.min( + gotoOptions.timeout ?? Infinity, + requestTimeoutMillis(request.userData as ContentCrawlerUserData), + ); }, ], browserPoolOptions: { @@ -251,7 +266,7 @@ function createPlaywrightCrawlerOptions( retireInactiveBrowserAfterSecs: 60, }, autoscaledPoolOptions: { - desiredConcurrency, + desiredConcurrency: input.desiredConcurrency, }, }, }; @@ -262,18 +277,22 @@ function createCheerioCrawlerOptions( proxy: ProxyConfiguration | undefined, keepAlive = true, ): ContentCrawlerOptions { - const { maxRequestRetries, desiredConcurrency } = input; - return { type: ContentCrawlerTypes.CHEERIO, proxyOptions: input.proxyConfiguration, crawlerOptions: { keepAlive, - maxRequestRetries, + maxRequestRetries: CRAWLER_MAX_REQUEST_RETRIES, proxyConfiguration: proxy, - requestHandlerTimeoutSecs: input.requestTimeoutSecs, + requestHandlerTimeoutSecs: CRAWLER_REQUEST_HANDLER_TIMEOUT_SECS, + preNavigationHooks: [ + ({ request }, gotOptions) => { + // eslint-disable-next-line no-param-reassign + gotOptions.timeout = { request: requestTimeoutMillis(request.userData as ContentCrawlerUserData) }; + }, + ], autoscaledPoolOptions: { - desiredConcurrency, + desiredConcurrency: input.desiredConcurrency, }, }, }; @@ -325,6 +344,7 @@ function validateAndFillInput(input: Partial): Input { /* eslint-enable no-param-reassign */ } +/** Every range-checked field is an integer in the input schema, so non-integers are rounded. */ function validateRange( value: number | string | undefined, min: number, @@ -332,19 +352,25 @@ function validateRange( defaultValue: number, fieldName: string, ) { - // parse the value as a number to check if it's a valid number - if (value === undefined) { - log.info(`The \`${fieldName}\` parameter is not defined. Using the default value ${defaultValue}.`); + const parsed = typeof value === 'string' ? Number(value) : value; + + if (parsed === undefined || !Number.isFinite(parsed)) { + if (parsed === undefined) { + log.info(`The \`${fieldName}\` parameter is not defined. Using the default value ${defaultValue}.`); + } else { + log.warning(`The \`${fieldName}\` parameter must be a number, but was ${value}. Using ${defaultValue} instead.`); + } return defaultValue; - } if (typeof value === 'string') { - /* eslint-disable-next-line no-param-reassign */ - value = Number(value); - } if (value < min) { - log.warning(`The \`${fieldName}\` parameter must be at least ${min}, but was ${fieldName}. Using ${min} instead.`); + } + + const rounded = Math.round(parsed); + if (rounded < min) { + log.warning(`The \`${fieldName}\` parameter must be at least ${min}, but was ${value}. Using ${min} instead.`); return min; - } if (value > max) { - log.warning(`The \`${fieldName}\` parameter must be at most ${max}, but was ${fieldName}. Using ${max} instead.`); + } + if (rounded > max) { + log.warning(`The \`${fieldName}\` parameter must be at most ${max}, but was ${value}. Using ${max} instead.`); return max; } - return value; + return rounded; } diff --git a/src/search.ts b/src/search.ts index 594bd98..255ec08 100644 --- a/src/search.ts +++ b/src/search.ts @@ -86,6 +86,7 @@ function prepareRequest( maxResults, contentCrawlerKey, contentScraperSettings, + serpMaxRetries: (input as Input & RagWebBrowserInput).serpMaxRetries, userAuthorization, }, searchCrawlerOptions.proxyOptions, diff --git a/src/types.ts b/src/types.ts index 5b379d7..f4210f4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -97,6 +97,8 @@ export interface ContentScraperSettings { outputFormats: OutputFormats[]; removeCookieWarnings?: boolean; removeElementsCssSelector?: string; + requestTimeoutSecs: number; + maxRequestRetries: number; } export type SearchCrawlerUserData = { @@ -106,6 +108,7 @@ export type SearchCrawlerUserData = { contentCrawlerKey: string; responseId: string; contentScraperSettings: ContentScraperSettings; + serpMaxRetries: number; // Pagination tracking /** Results accumulated across SERP pages, passed forward for merging */ collectedResults: OrganicResult[]; diff --git a/src/utils.ts b/src/utils.ts index e163061..49daa7d 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -26,6 +26,10 @@ const inputSchema = { }, }; +// Crawlee rewrites concurrency on its own schedule, and a proxy is fixed when a crawler is built. +// Honouring either per request would mean a crawler per caller, and standby crawlers never stop. +const STARTUP_ONLY_PARAMS = new Set(['desiredConcurrency', 'proxyConfiguration']); + const SCHEMELESS_URL_DOMAIN_SUFFIXES = [ 'com', 'net', @@ -76,6 +80,11 @@ export function parseParameters(url: string): Partial { continue; } + if (STARTUP_ONLY_PARAMS.has(key)) { + log.warning(`The \`${key}\` parameter can only be set on the Actor input, not per request. Ignoring it.`); + continue; + } + const typedKey = key as SchemaKey; // Parse outputFormats parameter as an array of OutputFormats @@ -105,6 +114,11 @@ export function parseParameters(url: string): Partial { return parsedInput; } +/** Crawlee caps this at the crawler's own navigation timeout, so a request can only narrow it. */ +export function requestTimeoutMillis(userData: ContentCrawlerUserData) { + return userData.contentScraperSettings.requestTimeoutSecs * 1000; +} + export function randomId() { let result = ''; const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; @@ -145,12 +159,14 @@ export function createSearchRequest( return { url: urlSearch, uniqueKey: randomId(), + maxRetries: userData.serpMaxRetries, userData: { maxResults: userData.maxResults, timeMeasures: userData.timeMeasures || [], query: userData.query, contentCrawlerKey: userData.contentCrawlerKey, contentScraperSettings: userData.contentScraperSettings, + serpMaxRetries: userData.serpMaxRetries, responseId: userData.responseId, collectedResults, currentPage, @@ -174,6 +190,7 @@ export function createRequest( return { url: result.url!, uniqueKey: randomId(), + maxRetries: contentScraperSettings.maxRequestRetries, // Media files contain no text to extract, so don't spend any bandwidth on downloading them. skipNavigation: isMediaUrl(result.url!), userData: { diff --git a/tests/cheerio-crawler.content.test.ts b/tests/cheerio-crawler.content.test.ts index ffc091f..0b97f16 100644 --- a/tests/cheerio-crawler.content.test.ts +++ b/tests/cheerio-crawler.content.test.ts @@ -73,6 +73,8 @@ describe('Cheerio Crawler Content Tests', () => { outputFormats: ['text'], maxHtmlCharsToProcess: 100000, dynamicContentWaitSecs: 20, + requestTimeoutSecs: 40, + maxRequestRetries: 1, }, [], ); diff --git a/tests/crawler-key.test.ts b/tests/crawler-key.test.ts index 7a84df3..f8009e4 100644 --- a/tests/crawler-key.test.ts +++ b/tests/crawler-key.test.ts @@ -1,4 +1,3 @@ -import type { CheerioCrawlerOptions } from 'crawlee'; import { log } from 'crawlee'; import { beforeAll, describe, expect, it } from 'vitest'; @@ -8,17 +7,9 @@ import { processInput, processStandbyInput } from '../src/input.js'; import type { ProxyOptions } from '../src/types.js'; import { parseParameters } from '../src/utils.js'; -const baseOptions: CheerioCrawlerOptions = { - keepAlive: true, - maxRequestRetries: 1, - requestHandlerTimeoutSecs: 40, - autoscaledPoolOptions: { desiredConcurrency: 5 }, -}; - -const cheerioKey = ( - options: CheerioCrawlerOptions = {}, - proxyOptions: ProxyOptions = { useApifyProxy: true }, -) => getCrawlerKey(ContentCrawlerTypes.CHEERIO, { ...baseOptions, ...options }, proxyOptions); +const cheerioKey = (proxyOptions: ProxyOptions = { useApifyProxy: true }) => ( + getCrawlerKey(ContentCrawlerTypes.CHEERIO, proxyOptions) +); describe('getCrawlerKey', () => { // The key doubles as a request queue name, so it has to stay a short slug. @@ -28,38 +19,38 @@ describe('getCrawlerKey', () => { it('separates the crawler kinds', () => { const keys = new Set([ - getCrawlerKey('search', baseOptions, {}), - getCrawlerKey(ContentCrawlerTypes.CHEERIO, baseOptions, {}), - getCrawlerKey(ContentCrawlerTypes.PLAYWRIGHT, baseOptions, {}), + getCrawlerKey('search', {}), + getCrawlerKey(ContentCrawlerTypes.CHEERIO, {}), + getCrawlerKey(ContentCrawlerTypes.PLAYWRIGHT, {}), ]); expect(keys.size).toBe(3); }); it('ignores the order the proxy options were declared in', () => { - const a = cheerioKey({}, { useApifyProxy: true, countryCode: 'US' }); - const b = cheerioKey({}, { countryCode: 'US', useApifyProxy: true }); + const a = cheerioKey({ useApifyProxy: true, countryCode: 'US' }); + const b = cheerioKey({ countryCode: 'US', useApifyProxy: true }); expect(a).toBe(b); }); it('treats the apifyProxy* input-schema aliases as their canonical counterparts', () => { - expect(cheerioKey({}, { apifyProxyGroups: ['RESIDENTIAL'] })).toBe(cheerioKey({}, { groups: ['RESIDENTIAL'] })); - expect(cheerioKey({}, { apifyProxyCountry: 'US' })).toBe(cheerioKey({}, { countryCode: 'US' })); + expect(cheerioKey({ apifyProxyGroups: ['RESIDENTIAL'] })).toBe(cheerioKey({ groups: ['RESIDENTIAL'] })); + expect(cheerioKey({ apifyProxyCountry: 'US' })).toBe(cheerioKey({ countryCode: 'US' })); }); it('collapses the useApifyProxy spellings the way the SDK does', () => { - const noProxy = cheerioKey({}, { useApifyProxy: false }); - const custom = cheerioKey({}, { proxyUrls: ['http://proxy.example.com:8000'] }); + const noProxy = cheerioKey({ useApifyProxy: false }); + const custom = cheerioKey({ proxyUrls: ['http://proxy.example.com:8000'] }); - expect(cheerioKey({}, { useApifyProxy: true })).toBe(cheerioKey({}, {})); - expect(cheerioKey({}, { useApifyProxy: false, proxyUrls: ['http://proxy.example.com:8000'] })).toBe(custom); - expect(cheerioKey({}, { useApifyProxy: false, tieredProxyUrls: [['http://a:1']] })).toBe(noProxy); + expect(cheerioKey({ useApifyProxy: true })).toBe(cheerioKey({})); + expect(cheerioKey({ useApifyProxy: false, proxyUrls: ['http://proxy.example.com:8000'] })).toBe(custom); + expect(cheerioKey({ useApifyProxy: false, tieredProxyUrls: [['http://a:1']] })).toBe(noProxy); expect(noProxy).not.toBe(custom); }); it('never exposes proxy credentials, so the key is safe to log', () => { - const key = cheerioKey({}, { + const key = cheerioKey({ password: 'hunter2', proxyUrls: ['http://user:hunter2@proxy.example.com:8000'], }); @@ -68,22 +59,18 @@ describe('getCrawlerKey', () => { expect(key).not.toContain('proxy.example.com'); }); - it('still separates crawlers whose settings genuinely differ', () => { + it('still separates crawlers that need a different proxy', () => { const keys = new Set([ cheerioKey(), - cheerioKey({ keepAlive: false }), - cheerioKey({ maxRequestRetries: 3 }), - cheerioKey({ requestHandlerTimeoutSecs: 90 }), - cheerioKey({ autoscaledPoolOptions: { desiredConcurrency: 10 } }), - cheerioKey({}, { groups: ['RESIDENTIAL'] }), - cheerioKey({}, { countryCode: 'US' }), - cheerioKey({}, { useApifyProxy: false }), - cheerioKey({}, { password: 'hunter2' }), - cheerioKey({}, { proxyUrls: ['http://proxy.example.com:8000'] }), - cheerioKey({}, { proxyUrls: ['http://other.example.com:8000'] }), + cheerioKey({ groups: ['RESIDENTIAL'] }), + cheerioKey({ countryCode: 'US' }), + cheerioKey({ useApifyProxy: false }), + cheerioKey({ password: 'hunter2' }), + cheerioKey({ proxyUrls: ['http://proxy.example.com:8000'] }), + cheerioKey({ proxyUrls: ['http://other.example.com:8000'] }), ]); - expect(keys.size).toBe(11); + expect(keys.size).toBe(7); }); }); @@ -93,41 +80,82 @@ describe('standby requests reuse the crawlers started at boot', () => { // A custom proxy keeps `Actor.createProxyConfiguration` off the network and off // `APIFY_PROXY_PASSWORD`, which it needs to return a `ProxyConfiguration` at all. const proxyConfiguration = { useApifyProxy: false, proxyUrls: ['http://proxy.invalid:8000'] }; - const query = (extraParams = '') => `?query=hello&proxyConfiguration=${ - encodeURIComponent(JSON.stringify(proxyConfiguration))}${extraParams}`; + const query = (extraParams = '') => `?query=hello${extraParams}`; let bootKeys: string[]; - const keysForRequest = async (queryString: string) => { - const { searchCrawlerOptions, contentCrawlerOptions } = await processInput(parseParameters(queryString)); - // Mirrors `runSearchProcess`, which forces keepAlive to match the crawlers started at boot. - searchCrawlerOptions.crawlerOptions.keepAlive = true; - contentCrawlerOptions.crawlerOptions.keepAlive = true; - - return [ - getCrawlerKey('search', searchCrawlerOptions.crawlerOptions, searchCrawlerOptions.proxyOptions), - getCrawlerKey(contentCrawlerOptions.type, contentCrawlerOptions.crawlerOptions, contentCrawlerOptions.proxyOptions), - ]; + const requestFor = async (queryString: string) => { + const { searchCrawlerOptions, contentCrawlerOptions, contentScraperSettings } = await processInput( + parseParameters(queryString), + ); + + return { + keys: [ + getCrawlerKey('search', searchCrawlerOptions.proxyOptions), + getCrawlerKey(contentCrawlerOptions.type, contentCrawlerOptions.proxyOptions), + ], + contentScraperSettings, + }; }; beforeAll(async () => { const { searchCrawlerOptions, contentCrawlerOptions } = await processStandbyInput({ proxyConfiguration }); bootKeys = [ - getCrawlerKey('search', searchCrawlerOptions.crawlerOptions, searchCrawlerOptions.proxyOptions), - ...contentCrawlerOptions.map((o) => getCrawlerKey(o.type, o.crawlerOptions, o.proxyOptions)), + getCrawlerKey('search', searchCrawlerOptions.proxyOptions), + ...contentCrawlerOptions.map((o) => getCrawlerKey(o.type, o.proxyOptions)), ]; expect(new Set(bootKeys).size).toBe(3); }); it('reuses them for a plain request', async () => { - expect(bootKeys).toEqual(expect.arrayContaining(await keysForRequest(query()))); + expect(bootKeys).toEqual(expect.arrayContaining((await requestFor(query())).keys)); }); it('reuses them when debugMode is requested, and leaves the log level alone', async () => { const levelBefore = log.getLevel(); - expect(bootKeys).toEqual(expect.arrayContaining(await keysForRequest(query('&debugMode=true')))); + expect(bootKeys).toEqual(expect.arrayContaining((await requestFor(query('&debugMode=true'))).keys)); expect(log.getLevel()).toBe(levelBefore); }); + + it('reuses them for every combination of the per-request crawl settings', async () => { + for (const params of [ + '&requestTimeoutSecs=300', + '&maxRequestRetries=0', + '&desiredConcurrency=50', + // The crawlers were built with a custom proxy, so asking for another one must not fork them. + `&proxyConfiguration=${encodeURIComponent('{"useApifyProxy":true,"apifyProxyCountry":"US"}')}`, + '&serpMaxRetries=0&maxRequestRetries=3&requestTimeoutSecs=7&desiredConcurrency=9', + ]) { + expect(bootKeys, params).toEqual(expect.arrayContaining((await requestFor(query(params))).keys)); + } + }); + + it('carries the narrowed settings on the request instead', async () => { + const { contentScraperSettings } = await requestFor(query('&requestTimeoutSecs=7&maxRequestRetries=3')); + + expect(contentScraperSettings.requestTimeoutSecs).toBe(7); + expect(contentScraperSettings.maxRequestRetries).toBe(3); + }); + + // Crawlee hands the hooks an empty object for Cheerio and the crawler's own timeout for + // Playwright, which the request may then only lower. + it.each([ + [ContentCrawlerTypes.CHEERIO, '', {}, { request: 3000 }], + [ContentCrawlerTypes.PLAYWRIGHT, '&scrapingTool=browser-playwright', { timeout: 60_000 }, 3000], + ])('narrows the %s navigation timeout to the requested one', async (_type, tool, initial, expected) => { + const { contentCrawlerOptions, contentScraperSettings } = await processInput( + parseParameters(query(`${tool}&requestTimeoutSecs=3`)), + ); + const hooks = contentCrawlerOptions.crawlerOptions.preNavigationHooks!; + const gotoOptions: Record = { ...initial }; + + await hooks[hooks.length - 1]( + { request: { userData: { contentScraperSettings } } } as never, + gotoOptions as never, + ); + + expect(gotoOptions.timeout).toEqual(expected); + }); }); diff --git a/tests/playwright-crawler.content.test.ts b/tests/playwright-crawler.content.test.ts index 920edc9..4a30a25 100644 --- a/tests/playwright-crawler.content.test.ts +++ b/tests/playwright-crawler.content.test.ts @@ -76,6 +76,8 @@ describe('Playwright Crawler Content Tests', () => { outputFormats: ['text'], maxHtmlCharsToProcess: 100000, dynamicContentWaitSecs: 20, + requestTimeoutSecs: 40, + maxRequestRetries: 1, }, [], ); diff --git a/tests/standby.test.ts b/tests/standby.test.ts index 5abc511..e3a7c2e 100644 --- a/tests/standby.test.ts +++ b/tests/standby.test.ts @@ -9,7 +9,7 @@ import { } from 'vitest'; import { ContentCrawlerStatus } from '../src/const.js'; -import { createAndStartContentCrawler, createAndStartSearchCrawler } from '../src/crawlers.js'; +import { createAndStartContentCrawler, createAndStartSearchCrawler, getCrawlerCount } from '../src/crawlers.js'; import { processStandbyInput } from '../src/input.js'; import { createServer } from '../src/server.js'; import { getImageRequestCount, resetImageRequestCount, startTestServer, stopTestServer } from './helpers/server.js'; @@ -102,6 +102,23 @@ describe('Standby RAG tests', () => { expect(getImageRequestCount()).toBe(0); }); + it('serves every combination of per-request settings without starting another crawler', async () => { + expect(getCrawlerCount()).toBe(3); + + for (const params of [ + '&debugMode=true', + '&requestTimeoutSecs=5', + '&maxRequestRetries=3&serpMaxRetries=0', + '&desiredConcurrency=17', + '&requestTimeoutSecs=299', + ]) { + const response = await fetch(`http://localhost:${browserServerPort}/search?query=${baseUrl}/basic${params}`); + + expect(response.status, params).toBe(200); + expect(getCrawlerCount(), params).toBe(3); + } + }); + it('standby request playwright does not download media files of the page', async () => { resetImageRequestCount(); diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 27e0dd7..21e164f 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -1,6 +1,16 @@ import { describe, expect, it } from 'vitest'; -import { interpretAsUrl } from '../src/utils.js'; +import type { ContentScraperSettings } from '../src/types.js'; +import { createRequest, createSearchRequest, interpretAsUrl, parseParameters } from '../src/utils.js'; + +const contentScraperSettings: ContentScraperSettings = { + debugMode: false, + dynamicContentWaitSecs: 1, + maxHtmlCharsToProcess: 1000, + outputFormats: ['markdown'], + requestTimeoutSecs: 7, + maxRequestRetries: 3, +}; describe('interpretAsUrl', () => { it('should return null for empty input', () => { @@ -31,3 +41,32 @@ describe('interpretAsUrl', () => { expect(interpretAsUrl('https%253A%252F%252Fexample.com')).toBe('https://example.com/'); }); }); + +describe('request retries', () => { + it('takes the content retry count from the scraper settings', () => { + expect(createRequest('q', { url: 'https://example.com' }, 'rid', contentScraperSettings).maxRetries).toBe(3); + }); + + it('takes the search retry count from the user data', () => { + const request = createSearchRequest({ + query: 'q', + responseId: 'rid', + maxResults: 1, + contentCrawlerKey: 'key', + contentScraperSettings, + serpMaxRetries: 4, + }, {}); + + expect(request.maxRetries).toBe(4); + expect(request.userData?.serpMaxRetries).toBe(4); + }); +}); + +describe('parseParameters', () => { + it('drops parameters that only apply to the crawlers started with the Actor', () => { + const proxyConfiguration = encodeURIComponent('{"useApifyProxy":true}'); + + expect(parseParameters(`?query=x&desiredConcurrency=17&proxyConfiguration=${proxyConfiguration}`)) + .toEqual({ query: 'x' }); + }); +});