diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 77ff08d8832e..db9ae5705080 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -8,6 +8,7 @@ export * from './internals/debug'; export * as social from './internals/social'; export * from './internals/typedefs'; export * from './internals/open_graph_parser'; +export * from './internals/schema_org_parser'; export * from './internals/gotScraping'; export * from './internals/iterables'; export * from './internals/robots'; diff --git a/packages/utils/src/internals/schema_org_parser.ts b/packages/utils/src/internals/schema_org_parser.ts new file mode 100644 index 000000000000..829e2b3fc6a2 --- /dev/null +++ b/packages/utils/src/internals/schema_org_parser.ts @@ -0,0 +1,302 @@ +import type { Dictionary } from '@crawlee/types'; +import { load, type CheerioAPI } from 'cheerio'; + +/** + * Represents a schema.org itemprop attribute + */ +export interface SchemaOrgProperty { + name: string; + outputName: string; + children?: SchemaOrgProperty[]; +} + +/** + * Result from parsing schema.org microdata + */ +type SchemaOrgResult = string | string[] | Dictionary | SchemaOrgResult[]; + +/** + * Extracts the value from a microdata element based on its itemprop + */ +const getValue = ($: CheerioAPI, elem: any): string | string[] => { + // Check for content attribute first + const content = $(elem).attr('content'); + if (content) return content; + + // Check for src for media elements + const src = $(elem).attr('src'); + if (src) return src; + + // Check for href for link elements + const href = $(elem).attr('href'); + if (href) return href; + + // Check for datetime for time elements + const datetime = $(elem).attr('datetime'); + if (datetime) return datetime; + + // Check for value for data elements + const value = $(elem).attr('value'); + if (value) return value; + + // For meta elements without recognized attributes, return empty + if ($(elem).is('meta')) return ''; + + // For other elements, return text content + return $(elem).text().trim(); +}; + +/** + * Parse a single schema.org property + */ +const parseSchemaProperty = (property: SchemaOrgProperty, $: CheerioAPI): SchemaOrgResult => { + // Handle both itemprop attribute and meta elements + const selector = property.name.startsWith('http') + ? `[itemprop="${property.name}"]` + : `[itemprop="${property.name}"]`; + + const elements = [...$(selector)]; + + if (elements.length === 0) return ''; + + if (elements.length === 1) { + const value = getValue($, elements[0]); + + if (!property.children || property.children.length === 0) { + return value; + } + + // Handle nested properties + const nested: Dictionary = {}; + for (const child of property.children) { + const childResult = parseSchemaProperty(child, $); + if (childResult) { + nested[child.outputName] = childResult as string; + } + } + + return Object.keys(nested).length > 0 + ? { value, ...nested } + : value; + } + + // Multiple elements - return array + return elements.map((elem) => { + const value = getValue($, elem); + + if (!property.children || property.children.length === 0) { + return value; + } + + const nested: Dictionary = {}; + for (const child of property.children) { + const childResult = parseSchemaProperty(child, $); + if (childResult) { + nested[child.outputName] = childResult as string; + } + } + + return Object.keys(nested).length > 0 + ? { value, ...nested } + : value; + }); +}; + +/** + * Common schema.org types and their properties + */ +export const SCHEMA_ORG_PROPERTIES: SchemaOrgProperty[] = [ + { + name: 'name', + outputName: 'name', + }, + { + name: 'description', + outputName: 'description', + }, + { + name: 'url', + outputName: 'url', + }, + { + name: 'image', + outputName: 'image', + }, + { + name: 'logo', + outputName: 'logo', + }, + { + name: 'price', + outputName: 'price', + }, + { + name: 'priceCurrency', + outputName: 'priceCurrency', + }, + { + name: 'author', + outputName: 'author', + }, + { + name: 'publisher', + outputName: 'publisher', + }, + { + name: 'datePublished', + outputName: 'datePublished', + }, + { + name: 'dateModified', + outputName: 'dateModified', + }, + { + name: 'ratingValue', + outputName: 'ratingValue', + }, + { + name: 'bestRating', + outputName: 'bestRating', + }, + { + name: 'worstRating', + outputName: 'worstRating', + }, + { + name: 'reviewCount', + outputName: 'reviewCount', + }, + { + name: 'sku', + outputName: 'sku', + }, + { + name: 'mpn', + outputName: 'mpn', + }, + { + name: 'gtin13', + outputName: 'gtin13', + }, + { + name: 'brand', + outputName: 'brand', + }, + { + name: 'aggregateRating', + outputName: 'aggregateRating', + children: [ + { name: 'ratingValue', outputName: 'ratingValue' }, + { name: 'reviewCount', outputName: 'reviewCount' }, + { name: 'bestRating', outputName: 'bestRating' }, + { name: 'worstRating', outputName: 'worstRating' }, + ], + }, + { + name: 'offers', + outputName: 'offers', + children: [ + { name: 'price', outputName: 'price' }, + { name: 'priceCurrency', outputName: 'priceCurrency' }, + { name: 'availability', outputName: 'availability' }, + { name: 'priceValidUntil', outputName: 'priceValidUntil' }, + ], + }, + { + name: 'itemListElement', + outputName: 'itemListElement', + }, +]; + +/** + * Extract all itemscope items from the page + */ +const extractItems = ($: CheerioAPI): Dictionary[] => { + const items: Dictionary[] = []; + + $('[itemscope]').each((_, elem) => { + const item: Dictionary = {}; + + // Get itemtype + const itemtype = $(elem).attr('itemtype'); + if (itemtype) { + item.itemtype = itemtype; + } + + // Get all itemprops within this scope + $(elem).find('[itemprop]').each((_, propElem) => { + const propName = $(propElem).attr('itemprop'); + if (!propName) return; + + const value = getValue($, propElem); + + // Check for nested itemscope + if ($(propElem).find('[itemscope]').length > 0) { + // Handle nested objects + const nestedItems = $(propElem).find('[itemscope]').map((_, nested) => { + const nestedItem: Dictionary = {}; + $(nested).find('[itemprop]').each((_, np) => { + const npName = $(np).attr('itemprop'); + if (npName) { + nestedItem[npName] = getValue($, np); + } + }); + return nestedItem; + }).get(); + + if (nestedItems.length > 0) { + item[propName] = nestedItems; + } + } else { + // Simple value + if (item[propName]) { + // Already exists - convert to array + const existing = item[propName]; + if (Array.isArray(existing)) { + existing.push(value); + } else { + item[propName] = [existing, value]; + } + } else { + item[propName] = value; + } + } + }); + + items.push(item); + }); + + return items; +}; + +/** + * Easily parse all Schema.org microdata from a page with just a `CheerioAPI` object or raw HTML. + * + * @param $ A `CheerioAPI` object, or a string of raw HTML. + * @param additionalProperties Any potential additional Schema.org properties you'd like to be scraped. + * @returns Scraped Schema.org microdata as an object with extracted items. + */ +export function parseSchemaOrg(raw: string, additionalProperties?: SchemaOrgProperty[]): Dictionary; +export function parseSchemaOrg($: CheerioAPI, additionalProperties?: SchemaOrgProperty[]): Dictionary; +export function parseSchemaOrg(item: CheerioAPI | string, additionalProperties?: SchemaOrgProperty[]) { + const $ = typeof item === 'string' ? load(item) : item; + + // Extract all itemscope items + const items = extractItems($); + + // Also extract common properties globally (outside itemscope) + const globalProps: Dictionary = {}; + const propsToCheck = [...(additionalProperties || []), ...SCHEMA_ORG_PROPERTIES]; + + for (const property of propsToCheck) { + const result = parseSchemaProperty(property, $); + if (result) { + globalProps[property.outputName] = result; + } + } + + return { + items, + properties: globalProps, + }; +}