From 289a1e95a59d15d713c48f9f8f6b9c92d1f33f09 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Sat, 4 Jul 2026 23:21:12 -1000 Subject: [PATCH] Harden place extractor fixtures and preview enrichment --- src/gmaps_scraper/data/place_extractor.js | 896 ++++++++++++++++++ src/gmaps_scraper/place_scraper.py | 836 ++-------------- tests/fixtures/place_pages/about.html | 41 + tests/fixtures/place_pages/limited_view.html | 18 + tests/fixtures/place_pages/overview.html | 41 + tests/fixtures/place_pages/reviews.html | 51 + tests/fixtures/place_pages/search_result.html | 32 + .../ambiguous_rating_review_count.txt | 32 + .../preview_payloads/rating_review_count.txt | 17 + tests/test_place_scraper.py | 254 ++++- 10 files changed, 1432 insertions(+), 786 deletions(-) create mode 100644 src/gmaps_scraper/data/place_extractor.js create mode 100644 tests/fixtures/place_pages/about.html create mode 100644 tests/fixtures/place_pages/limited_view.html create mode 100644 tests/fixtures/place_pages/overview.html create mode 100644 tests/fixtures/place_pages/reviews.html create mode 100644 tests/fixtures/place_pages/search_result.html create mode 100644 tests/fixtures/preview_payloads/ambiguous_rating_review_count.txt create mode 100644 tests/fixtures/preview_payloads/rating_review_count.txt diff --git a/src/gmaps_scraper/data/place_extractor.js b/src/gmaps_scraper/data/place_extractor.js new file mode 100644 index 0000000..086c80d --- /dev/null +++ b/src/gmaps_scraper/data/place_extractor.js @@ -0,0 +1,896 @@ + +() => { + + const cleanLine = (value) => (value || "").replace(/\s+/g, " ").trim(); + const searchResultTitleLabels = new Set([ + "result", + "results", + "search result", + "search results", + "結果", + "検索結果", + ]); + const titleSelectors = ["h1.DUwDvf", "h1.lfPIob", "div[role='main'] h1"]; + const isSearchResultTitle = (value) => searchResultTitleLabels.has( + cleanLine(value).toLowerCase(), + ); + const visibleRect = (element) => { + const rect = element.getBoundingClientRect(); + const width = Math.max(0, Math.min(rect.right, window.innerWidth) - Math.max(rect.left, 0)); + const height = Math.max(0, Math.min(rect.bottom, window.innerHeight) - Math.max(rect.top, 0)); + return {rect, visibleArea: width * height}; + }; + const matchingTitleElements = (root) => { + const elements = []; + const seen = new Set(); + for (const selector of titleSelectors) { + const matches = []; + if (root.matches?.(selector)) { + matches.push(root); + } + matches.push(...root.querySelectorAll(selector)); + for (const element of matches) { + if (seen.has(element)) { + continue; + } + seen.add(element); + elements.push(element); + } + } + return elements; + }; + const placePanelCandidateRoots = () => { + const roots = new Set(); + for (const selector of [ + "[role='main']", + "[role='dialog']", + "[role='region'][aria-label]", + ]) { + for (const element of document.querySelectorAll(selector)) { + roots.add(element); + } + } + for (const title of document.querySelectorAll("h1, [role='heading']")) { + let current = title; + for (let depth = 0; depth < 9 && current && current !== document.body; depth += 1) { + roots.add(current); + current = current.parentElement; + } + } + return Array.from(roots); + }; + const placePanelRoot = () => { + let best = null; + for (const root of placePanelCandidateRoots()) { + const {rect, visibleArea} = visibleRect(root); + if (visibleArea <= 0 || rect.width < 120 || rect.height < 80) { + continue; + } + const titleElements = matchingTitleElements(root); + const titleTexts = titleElements + .map((element) => cleanLine(element.innerText || element.textContent || "")) + .filter(Boolean); + const titleElement = titleElements.find((element) => { + const title = cleanLine(element.innerText || element.textContent || ""); + return title && !isSearchResultTitle(title); + }) || titleElements.find( + (element) => cleanLine(element.innerText || element.textContent || ""), + ); + const title = cleanLine(titleElement?.innerText || titleElement?.textContent || ""); + const nonResultTitle = title && !isSearchResultTitle(title) ? title : ""; + const label = cleanLine(root.getAttribute("aria-label") || ""); + const hasResultsTitle = titleTexts.some(isSearchResultTitle) || /results for/i.test(label); + const addressRows = root.querySelectorAll(`[data-item-id="address"]`).length; + const ratingSummaries = root.querySelectorAll("div.F7nice").length; + const tabLabels = Array.from( + root.querySelectorAll("button[role='tab'], div[role='tablist'] button"), + ).map((element) => cleanLine(element.getAttribute("aria-label") || element.innerText || "")); + const hasOverviewTabs = tabLabels.some((value) => /overview/i.test(value)) + && tabLabels.some((value) => /reviews?/i.test(value)); + const articleCount = root.querySelectorAll("[role='article']").length; + const placeRows = root.querySelectorAll( + `[data-item-id="address"], [data-item-id="authority"], [data-item-id^="phone:"]`, + ).length; + let score = 0; + if (nonResultTitle) { + score += 120; + } + if (addressRows) { + score += 45; + } + if (ratingSummaries) { + score += 30; + } + if (hasOverviewTabs) { + score += 25; + } + score += Math.min(placeRows * 10, 30); + if (root.getAttribute("role") === "main" && nonResultTitle) { + score += 10; + } + if (label && nonResultTitle && label === nonResultTitle) { + score += 8; + } + if (hasResultsTitle) { + score -= 70; + } + if (articleCount >= 2 && !addressRows) { + score -= 40; + } + score += Math.min(visibleArea / 20000, 20); + const tieBreak = (root.getAttribute("role") === "main" ? 2000000 : 0) + visibleArea; + if (!best || score > best.score || (score === best.score && tieBreak > best.tieBreak)) { + best = {root, titleElement, title, score, tieBreak}; + } + } + if (best && best.score > 0) { + return { + root: best.root, + titleElement: best.titleElement, + title: best.title, + found: true, + score: best.score, + }; + } + const fallbackTitle = document.querySelector(titleSelectors.join(",")); + return { + root: document.body, + titleElement: fallbackTitle, + title: cleanLine(fallbackTitle?.innerText || fallbackTitle?.textContent || ""), + found: false, + score: 0, + }; + }; + + + const panelInfo = placePanelRoot(); + const panel = panelInfo.root; + const titleElement = panelInfo.titleElement; + + const firstText = (selectors, root = panel) => { + for (const selector of selectors) { + const element = root.querySelector(selector); + const text = element?.innerText?.trim(); + if (text) { + return text; + } + } + return null; + }; + + const firstAttr = (selectors, attr, root = panel) => { + for (const selector of selectors) { + const element = root.querySelector(selector); + const value = element?.getAttribute(attr)?.trim(); + if (value) { + return value; + } + } + return null; + }; + + const isReviewScoped = (element) => { + if (!element) { + return false; + } + if (element.closest("[data-review-id]")) { + return true; + } + const label = element.getAttribute?.("aria-label") || ""; + return /(^|\W)reviews?(\W|$)/i.test(label); + }; + + const firstImageUrl = (selectors, root = panel) => { + for (const selector of selectors) { + for (const element of root.querySelectorAll(selector)) { + if (isReviewScoped(element)) { + continue; + } + const value = element?.currentSrc + || element?.getAttribute("src")?.trim() + || element?.getAttribute("data-src")?.trim(); + if (value) { + return value; + } + } + } + return null; + }; + + const firstBackgroundImageUrl = (selectors, root = panel) => { + for (const selector of selectors) { + for (const element of root.querySelectorAll(selector)) { + if (isReviewScoped(element)) { + continue; + } + const style = getComputedStyle(element).backgroundImage || ""; + const match = style.match(/url\((['"]?)(.*?)\1\)/); + if (match?.[2]) { + return match[2].trim(); + } + } + } + return null; + }; + + const itemValue = (itemId) => firstText([ + `[data-item-id="${itemId}"] .Io6YTe`, + `[data-item-id="${itemId}"]`, + ]); + + const rowValue = (row) => { + // `.DkEaL` can be a localized row label when the value is in `.Io6YTe`. + // Prefer the value node and only use `.DkEaL` for older rows where it is + // the address text itself. + const value = ( + row?.querySelector(".Io6YTe")?.innerText?.trim() + || row?.querySelector(".DkEaL")?.innerText?.trim() + ); + return value || null; + }; + + const isAddressIcon = (icon) => { + const label = icon?.getAttribute?.("aria-label") || ""; + const glyph = icon?.innerText?.trim() || icon?.textContent?.trim() || ""; + return label === "Address" || glyph === ""; + }; + + const addressValue = () => { + // Prefer Google Maps' structured address row. The icon fallback exists for + // localized pages where the aria-label text changes but the address glyph + // and row shape remain stable. + const legacy = itemValue("address"); + if (legacy) { + return legacy; + } + for (const icon of panel.querySelectorAll(".google-symbols, [role='img']")) { + if (!isAddressIcon(icon)) { + continue; + } + const row = icon.closest(".LCF4w, .MngOvd, .RcCsl, [data-section-id]"); + const value = rowValue(row); + if (value && value !== "Address") { + return value; + } + } + return null; + }; + const addressRowElement = () => { + const legacy = panel.querySelector(`[data-item-id="address"]`); + if (legacy) { + return legacy; + } + for (const icon of panel.querySelectorAll(".google-symbols, [role='img']")) { + if (!isAddressIcon(icon)) { + continue; + } + const row = icon.closest(".LCF4w, .MngOvd, .RcCsl, [data-section-id]"); + if (row) { + return row; + } + } + return null; + }; + const elementTop = (element) => { + const rect = element?.getBoundingClientRect?.(); + return rect && rect.height > 0 ? rect.top : null; + }; + const elementBottom = (element) => { + const rect = element?.getBoundingClientRect?.(); + return rect && rect.height > 0 ? rect.bottom : null; + }; + const descriptionBoundaryTop = () => { + const rows = Array.from(panel.querySelectorAll("[data-item-id]")) + .map(elementTop) + .filter((value) => value !== null); + if (rows.length > 0) { + return Math.min(...rows); + } + const addressRow = addressRowElement(); + const addressTop = elementTop(addressRow); + return addressTop === null ? Infinity : addressTop; + }; + const descriptionValue = () => { + const direct = firstText([".WeS02d", ".PYvSYb"]); + if (direct) { + return direct; + } + const titleBottom = Math.max( + ...[ + elementBottom(titleElement), + ...Array.from(panel.querySelectorAll("div.F7nice")).map(elementBottom), + ].filter((value) => value !== null), + 0, + ); + const boundaryTop = descriptionBoundaryTop(); + const candidates = []; + for (const element of panel.querySelectorAll("div, span")) { + const text = cleanLine(element.innerText || element.textContent || ""); + if (!text || text.includes("·")) { + continue; + } + if ( + element.closest( + "button, a, [role='button'], [role='tab'], [role='tablist'], " + + "[data-item-id], [data-review-id], div.F7nice", + ) + ) { + continue; + } + if ( + Array.from(element.children).some( + (child) => cleanLine(child.innerText || child.textContent || "") === text, + ) + ) { + continue; + } + const top = elementTop(element); + if (top === null || top <= titleBottom || top >= boundaryTop) { + continue; + } + candidates.push({top, text}); + } + candidates.sort((left, right) => left.top - right.top); + return candidates[0]?.text || null; + }; + + const normalizeCount = (value) => { + if (!value) { + return 0; + } + const text = value.trim().toUpperCase(); + let multiplier = 1; + if (text.includes("K")) { + multiplier = 1000; + } else if (text.includes("M")) { + multiplier = 1000000; + } else if (text.includes("萬") || text.includes("万")) { + multiplier = 10000; + } + const numeric = parseFloat(text.replace(/[,\sKM萬万]/g, "")); + return Number.isFinite(numeric) ? numeric * multiplier : 0; + }; + + const reviewKeywords = ["review", "reviews", "評論", "クチコミ"]; + const reviewCountPattern = new RegExp( + "([0-9][0-9,.\\s]*[KM萬万]?)[ ]*" + + "(?:reviews?|評論|クチコミ|件のクチコミ|件の Google クチコミ|則評論|篇評論)", + "i", + ); + const reviewCountPatternReverse = new RegExp( + "(?:reviews?|評論|クチコミ)\\s*[(]([0-9][0-9,.\\s]*[KM萬万]?)[)]", + "i", + ); + + let reviewCount = null; + let reviewSource = null; + let bestCount = 0; + + const considerCount = (candidate, source) => { + if (!candidate) { + return; + } + const count = normalizeCount(candidate); + if (count <= 0) { + return; + } + if (count > bestCount) { + bestCount = count; + reviewCount = candidate.trim(); + reviewSource = source; + } + }; + + for (const span of panel.querySelectorAll("div.F7nice span")) { + const text = span.innerText?.trim() || ""; + const match = text.match(/^\(?([0-9][0-9,.\s]*[KM萬万]?)\)?$/i); + if (!match) { + continue; + } + if (/^[0-9]+([.,][0-9]+)?$/.test(match[1]) && normalizeCount(match[1]) < 10) { + continue; + } + considerCount(match[1], "f7nice"); + } + + const reviewSummaryBoundaryTop = () => { + const addressRow = addressRowElement(); + if (addressRow) { + const rect = addressRow.getBoundingClientRect(); + if (rect.height > 0) { + return rect.top; + } + } + const panelRect = panel.getBoundingClientRect(); + return panelRect.top + 520; + }; + const isOverviewReviewCountElement = (element) => { + if (element.closest("div.F7nice")) { + return true; + } + const rect = element.getBoundingClientRect(); + if (rect.height <= 0) { + return false; + } + return rect.top < reviewSummaryBoundaryTop(); + }; + + for (const element of panel.querySelectorAll("[aria-label]")) { + if (!isOverviewReviewCountElement(element)) { + continue; + } + const label = element.getAttribute("aria-label") || ""; + if (!reviewKeywords.some((keyword) => label.toLowerCase().includes(keyword.toLowerCase()))) { + continue; + } + const match = label.match(reviewCountPattern) || label.match(reviewCountPatternReverse); + if (match) { + considerCount(match[1], "aria-label"); + } + } + + if (!reviewCount) { + for (const tab of panel.querySelectorAll("div[role='tablist'] button")) { + const text = tab.innerText?.trim() || ""; + if (!reviewKeywords.some((keyword) => text.toLowerCase().includes(keyword.toLowerCase()))) { + continue; + } + const match = text.match(/([0-9][0-9,.\s]*[KM萬万]?)/i); + if (match) { + considerCount(match[1], "tab"); + } + } + } + + const mainPhotoUrl = firstImageUrl([ + "div.RZ66Rb button[jsaction*='heroHeaderImage'] img", + "button[jsaction*='heroHeaderImage'] img", + "div.ZKCDEc [data-photo-index='0'] img", + "[data-photo-index='0'] img", + "[data-photo-index] img", + ]) + || firstBackgroundImageUrl([ + "div.RZ66Rb button[jsaction*='heroHeaderImage']", + "button[jsaction*='heroHeaderImage']", + "div.ZKCDEc [data-photo-index='0']", + "[data-photo-index='0']", + "[data-photo-index]", + ]); + const photoUrl = mainPhotoUrl + || firstAttr(["meta[property='og:image']", "meta[itemprop='image']"], "content", document); + + const shallowPath = (element) => { + const parts = []; + let current = element; + for (let i = 0; i < 4 && current && current.nodeType === Node.ELEMENT_NODE; i += 1) { + let part = current.tagName.toLowerCase(); + const id = current.getAttribute("data-item-id"); + const role = current.getAttribute("role"); + if (id) { + part += `[data-item-id="${id}"]`; + } else if (role) { + part += `[role="${role}"]`; + } else if (current.classList?.length) { + part += "." + Array.from(current.classList).slice(0, 2).join("."); + } + parts.unshift(part); + current = current.parentElement; + } + return parts.join(" > "); + }; + const nearbyText = (element) => { + const texts = []; + const parent = element.parentElement; + if (!parent) { + return texts; + } + for (const child of parent.children) { + const text = cleanLine(child.innerText || child.textContent || ""); + if (text && !texts.includes(text)) { + texts.push(text); + } + if (texts.length >= 4) { + break; + } + } + return texts; + }; + const collectDomCandidates = () => { + const selectors = [ + "[data-item-id]", + "button[aria-label]", + "a[aria-label]", + "[role='button'][aria-label]", + ".Io6YTe", + ".DkEaL", + ".F7nice", + "div[role='tablist'] button", + ]; + const candidates = []; + const seen = new Set(); + for (const selector of selectors) { + for (const element of panel.querySelectorAll(selector)) { + const text = cleanLine(element.innerText || element.textContent || ""); + const ariaLabel = cleanLine(element.getAttribute("aria-label") || ""); + const dataItemId = cleanLine(element.getAttribute("data-item-id") || ""); + if (!text && !ariaLabel && !dataItemId) { + continue; + } + if (text.length > 240 || ariaLabel.length > 240) { + continue; + } + const key = `${selector}\n${text}\n${ariaLabel}\n${dataItemId}`; + if (seen.has(key)) { + continue; + } + seen.add(key); + candidates.push({ + text, + tag: element.tagName.toLowerCase(), + role: cleanLine(element.getAttribute("role") || ""), + aria_label: ariaLabel, + data_item_id: dataItemId, + selector_hint: shallowPath(element), + nearby_text: nearbyText(element), + }); + if (candidates.length >= 120) { + return candidates; + } + } + } + return candidates; + }; + const collectReviewTopics = () => { + const selectors = [ + "button[jsaction*='review']", + "button[aria-label*='review' i]", + "button[role='radio']", + "button[aria-pressed]", + "div[role='button'][aria-label]", + ]; + const topics = []; + const seen = new Set(); + for (const selector of selectors) { + for (const element of panel.querySelectorAll(selector)) { + const text = cleanLine(element.innerText || element.textContent || ""); + const ariaLabel = cleanLine(element.getAttribute("aria-label") || ""); + const candidate = /[0-9]/.test(text) + ? text + : (/[0-9]/.test(ariaLabel) ? ariaLabel : text || ariaLabel); + if (!candidate || candidate.length > 120 || !/[0-9]/.test(candidate)) { + continue; + } + const key = `${candidate}\n${ariaLabel}`; + if (seen.has(key)) { + continue; + } + seen.add(key); + topics.push({ + text: candidate, + aria_label: ariaLabel, + source: selector, + }); + } + } + return topics; + }; + const priceSymbols = "(?:[$€£¥₩₹₫฿₱₦₺₴₽]|SGD|USD|EUR|GBP|JPY|TWD|NT\\$|HK\\$|CA\\$|A\\$)"; + const pricePattern = new RegExp( + "(?:^|\\s|·)((?:\\${1,4})|" + priceSymbols + + "\\s*[0-9][0-9,.\u00a0\\s]*(?:\\+|[-–]\\s*" + priceSymbols + + "?\\s*[0-9][0-9,.\u00a0\\s]*)?)", + "i", + ); + const exactNumericPricePattern = new RegExp( + "^" + priceSymbols + "\\s*[0-9][0-9,.\u00a0\\s]*$", + "i", + ); + const extractPrice = (value) => { + const text = cleanLine(value); + const match = text.match(pricePattern); + if (!match?.[1]) { + return null; + } + return cleanLine(match[1].replace(/\u00a0/g, " ")); + }; + const looksLikePriceRangeText = (value) => { + const text = cleanLine(value); + if (!text) { + return false; + } + if (/^\${1,4}$/.test(text)) { + return true; + } + return text.includes("·") && pricePattern.test(text); + }; + const headingAliases = (value) => Array.isArray(value) ? value : [value]; + const normalizedHeading = (value) => cleanLine(value).toLowerCase(); + const sectionRootByHeading = (headingText) => { + const aliases = headingAliases(headingText) + .map((value) => normalizedHeading(value)) + .filter(Boolean); + if (aliases.length === 0) { + return null; + } + for (const heading of panel.querySelectorAll("h2, h3, [role='heading']")) { + const text = normalizedHeading(heading.innerText || heading.textContent || ""); + if (!aliases.includes(text)) { + continue; + } + return ( + heading.closest(".m6QErb, section, [role='region'], [data-section-id]") + || heading.parentElement + || null + ); + } + return null; + }; + const collectLeafPrices = (root) => { + if (!root) { + return []; + } + const prices = []; + const seen = new Set(); + for (const element of root.querySelectorAll("*")) { + const text = cleanLine(element.innerText || element.textContent || ""); + if (!text || text.length > 48) { + continue; + } + if ( + Array.from(element.children).some( + (child) => cleanLine(child.innerText || child.textContent || "") === text, + ) + ) { + continue; + } + const price = extractPrice(text); + if (!price || price !== text || !exactNumericPricePattern.test(price)) { + continue; + } + if (seen.has(price)) { + continue; + } + seen.add(price); + prices.push(price); + } + return prices; + }; + const providerLabelFromUrl = (href) => { + try { + const host = new URL(href).hostname.replace(/^www\./, ""); + const base = host.split(".")[0] || host; + return base + .replace(/[-_]+/g, " ") + .replace(/\b\w/g, (char) => char.toUpperCase()); + } catch { + return "Find a Table"; + } + }; + const reservationLabel = (element, href) => { + const actionPrefixPattern = new RegExp( + "^(?:find a table|reserve|make a reservation|book a table)" + + "(?:\\s+(?:with|on|at|via))?\\s*", + "i", + ); + const raw = cleanLine( + element.innerText + || element.textContent + || element.getAttribute("aria-label") + || element.getAttribute("title") + || "", + ); + const cleaned = raw + .replace(actionPrefixPattern, "") + .replace(/\s+(?:opens in new tab|website)$/i, "") + .trim(); + return cleaned || providerLabelFromUrl(href); + }; + const collectReservationLinks = () => { + const links = []; + const seen = new Set(); + const reservationPattern = new RegExp( + String.raw`\b(find a table|reserve|reservation|book a table)\b`, + "i", + ); + for (const element of panel.querySelectorAll("a[href]")) { + const href = element.href || element.getAttribute("href") || ""; + if (!/^https?:\/\//i.test(href) || seen.has(href)) { + continue; + } + const evidence = [ + element.innerText, + element.textContent, + element.getAttribute("aria-label"), + element.getAttribute("title"), + element.getAttribute("data-item-id"), + ].filter(Boolean).join(" "); + if (!reservationPattern.test(evidence)) { + continue; + } + seen.add(href); + links.push({ + label: reservationLabel(element, href), + url: href, + }); + if (links.length >= 8) { + break; + } + } + return links; + }; + const roomOverlayPrice = () => { + const selectors = [ + ".rlmNhf button[aria-label]", + "button[aria-label*='per night' i]", + "button[aria-label*='prices from' i]", + ]; + for (const selector of selectors) { + for (const element of document.querySelectorAll(selector)) { + const price = extractPrice(element.getAttribute("aria-label") || ""); + if (price && exactNumericPricePattern.test(price)) { + return price; + } + } + } + return null; + }; + const detailsBoundaryTop = () => { + const selectors = [ + `[data-item-id="address"]`, + `[data-item-id="authority"]`, + `[data-item-id="oloc"]`, + `[data-item-id="locatedin"]`, + `button[data-item-id^="phone:"]`, + ]; + let boundary = Number.POSITIVE_INFINITY; + for (const selector of selectors) { + for (const element of panel.querySelectorAll(selector)) { + const rect = element.getBoundingClientRect(); + if (rect.height <= 0) { + continue; + } + boundary = Math.min(boundary, rect.top); + } + } + const addressRow = addressRowElement(); + if (addressRow) { + const rect = addressRow.getBoundingClientRect(); + if (rect.height > 0) { + boundary = Math.min(boundary, rect.top); + } + } + return Number.isFinite(boundary) ? boundary : Number.POSITIVE_INFINITY; + }; + const structuralOfferSignals = () => { + const titleTop = ( + titleElement?.getBoundingClientRect()?.top + || panel.getBoundingClientRect().top + ); + const boundaryTop = detailsBoundaryTop(); + const prices = []; + const seenPrices = new Set(); + for (const element of panel.querySelectorAll("*")) { + const text = cleanLine(element.innerText || element.textContent || ""); + if (!text || text.length > 48) { + continue; + } + const rect = element.getBoundingClientRect(); + if (rect.height <= 0 || rect.top <= titleTop || rect.top >= boundaryTop) { + continue; + } + if ( + Array.from(element.children).some( + (child) => cleanLine(child.innerText || child.textContent || "") === text, + ) + ) { + continue; + } + const price = extractPrice(text); + if (!price || price !== text || !exactNumericPricePattern.test(price)) { + continue; + } + if (seenPrices.has(price)) { + continue; + } + seenPrices.add(price); + prices.push(price); + } + const kind = ( + prices.length > 0 + ? ( + roomOverlayPrice() + || panel.querySelector(`[data-item-id="place-info-links:"]`) + ? "room" + : "admission" + ) + : null + ); + return {kind, prices}; + }; + const priceRangeValue = () => { + const roots = [ + panel.querySelector(".dmRWX"), + panel.querySelector(".F7nice")?.parentElement, + panel.querySelector(".F7nice"), + panel, + ].filter(Boolean); + for (const root of roots) { + const text = cleanLine(root.innerText || root.textContent || ""); + if (!looksLikePriceRangeText(text)) { + continue; + } + const match = text.match(pricePattern); + if (match?.[1]) { + return cleanLine(match[1].replace(/\u00a0/g, " ")); + } + } + return null; + }; + const structuralOffers = structuralOfferSignals(); + + return { + name: firstText(titleSelectors), + secondary_name: firstText(["h2.bwoZTb span", "h2.bwoZTb"]), + rating: firstText([ + "div.F7nice > span > span[aria-hidden='true']:first-child", + "span.ceNzKf[role='img']", + "span[role='img'][aria-label*='star']", + ]), + review_count: reviewCount, + review_count_source: reviewSource, + category: firstText([ + "button[jsaction*='category']", + ".skqShb .fontBodyMedium button", + "button.DkEaL", + ]), + price_range: priceRangeValue(), + address: addressValue(), + located_in: itemValue("locatedin"), + status: firstText(["div.OqCZI .ZDu9vd", "div.OqCZI .o0Svhf"]), + website: firstAttr(["a[data-item-id='authority']"], "href", document) || itemValue("authority"), + reservation_links: collectReservationLinks(), + phone: firstText([ + "button[data-item-id^='phone:'] .Io6YTe", + "button[data-item-id^='phone:']", + ]), + plus_code: itemValue("oloc"), + description: descriptionValue(), + review_topics: collectReviewTopics(), + admission_prices: collectLeafPrices(sectionRootByHeading([ + "Admission", + "Ticket prices", + "Entry fee", + "Entrance fee", + "入場", + "入場料", + "入園料", + "票價", + "票价", + "門票", + "门票", + ])), + room_prices: collectLeafPrices(sectionRootByHeading([ + "Compare prices", + "Compare room prices", + "Room prices", + "價格比較", + "价格比较", + "比較價格", + "比較房價", + "料金を比較", + "価格を比較", + "宿泊料金を比較", + ])), + structural_offer_kind: structuralOffers.kind, + structural_offer_prices: structuralOffers.prices, + room_price_overlay: roomOverlayPrice(), + dom_candidates: collectDomCandidates(), + main_photo_url: mainPhotoUrl, + photo_url: photoUrl, + panel_text: panel?.innerText || "", + body_text: document.body?.innerText || "", + limited_view: (document.body?.innerText || "") + .toLowerCase() + .includes("limited view of google maps"), + }; +} diff --git a/src/gmaps_scraper/place_scraper.py b/src/gmaps_scraper/place_scraper.py index 6aac32a..6b34789 100644 --- a/src/gmaps_scraper/place_scraper.py +++ b/src/gmaps_scraper/place_scraper.py @@ -9,6 +9,7 @@ from collections.abc import Callable, Iterable, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor, as_completed from hashlib import sha256 +from importlib import resources from pathlib import Path from typing import Any, Literal, cast from urllib.parse import parse_qs, unquote, urlparse @@ -475,762 +476,9 @@ }; }; """ -_PLACE_JS_EXTRACTOR = r""" -() => { -""" + _PLACE_PANEL_HELPERS_JS + r""" - - const panelInfo = placePanelRoot(); - const panel = panelInfo.root; - const titleElement = panelInfo.titleElement; - - const firstText = (selectors, root = panel) => { - for (const selector of selectors) { - const element = root.querySelector(selector); - const text = element?.innerText?.trim(); - if (text) { - return text; - } - } - return null; - }; - - const firstAttr = (selectors, attr, root = panel) => { - for (const selector of selectors) { - const element = root.querySelector(selector); - const value = element?.getAttribute(attr)?.trim(); - if (value) { - return value; - } - } - return null; - }; - - const isReviewScoped = (element) => { - if (!element) { - return false; - } - if (element.closest("[data-review-id]")) { - return true; - } - const label = element.getAttribute?.("aria-label") || ""; - return /(^|\W)reviews?(\W|$)/i.test(label); - }; - - const firstImageUrl = (selectors, root = panel) => { - for (const selector of selectors) { - for (const element of root.querySelectorAll(selector)) { - if (isReviewScoped(element)) { - continue; - } - const value = element?.currentSrc - || element?.getAttribute("src")?.trim() - || element?.getAttribute("data-src")?.trim(); - if (value) { - return value; - } - } - } - return null; - }; - - const firstBackgroundImageUrl = (selectors, root = panel) => { - for (const selector of selectors) { - for (const element of root.querySelectorAll(selector)) { - if (isReviewScoped(element)) { - continue; - } - const style = getComputedStyle(element).backgroundImage || ""; - const match = style.match(/url\((['"]?)(.*?)\1\)/); - if (match?.[2]) { - return match[2].trim(); - } - } - } - return null; - }; - - const itemValue = (itemId) => firstText([ - `[data-item-id="${itemId}"] .Io6YTe`, - `[data-item-id="${itemId}"]`, - ]); - - const rowValue = (row) => { - // `.DkEaL` can be a localized row label when the value is in `.Io6YTe`. - // Prefer the value node and only use `.DkEaL` for older rows where it is - // the address text itself. - const value = ( - row?.querySelector(".Io6YTe")?.innerText?.trim() - || row?.querySelector(".DkEaL")?.innerText?.trim() - ); - return value || null; - }; - - const isAddressIcon = (icon) => { - const label = icon?.getAttribute?.("aria-label") || ""; - const glyph = icon?.innerText?.trim() || icon?.textContent?.trim() || ""; - return label === "Address" || glyph === ""; - }; - - const addressValue = () => { - // Prefer Google Maps' structured address row. The icon fallback exists for - // localized pages where the aria-label text changes but the address glyph - // and row shape remain stable. - const legacy = itemValue("address"); - if (legacy) { - return legacy; - } - for (const icon of panel.querySelectorAll(".google-symbols, [role='img']")) { - if (!isAddressIcon(icon)) { - continue; - } - const row = icon.closest(".LCF4w, .MngOvd, .RcCsl, [data-section-id]"); - const value = rowValue(row); - if (value && value !== "Address") { - return value; - } - } - return null; - }; - const addressRowElement = () => { - const legacy = panel.querySelector(`[data-item-id="address"]`); - if (legacy) { - return legacy; - } - for (const icon of panel.querySelectorAll(".google-symbols, [role='img']")) { - if (!isAddressIcon(icon)) { - continue; - } - const row = icon.closest(".LCF4w, .MngOvd, .RcCsl, [data-section-id]"); - if (row) { - return row; - } - } - return null; - }; - const elementTop = (element) => { - const rect = element?.getBoundingClientRect?.(); - return rect && rect.height > 0 ? rect.top : null; - }; - const elementBottom = (element) => { - const rect = element?.getBoundingClientRect?.(); - return rect && rect.height > 0 ? rect.bottom : null; - }; - const descriptionBoundaryTop = () => { - const rows = Array.from(panel.querySelectorAll("[data-item-id]")) - .map(elementTop) - .filter((value) => value !== null); - if (rows.length > 0) { - return Math.min(...rows); - } - const addressRow = addressRowElement(); - const addressTop = elementTop(addressRow); - return addressTop === null ? Infinity : addressTop; - }; - const descriptionValue = () => { - const direct = firstText([".WeS02d", ".PYvSYb"]); - if (direct) { - return direct; - } - const titleBottom = Math.max( - ...[ - elementBottom(titleElement), - ...Array.from(panel.querySelectorAll("div.F7nice")).map(elementBottom), - ].filter((value) => value !== null), - 0, - ); - const boundaryTop = descriptionBoundaryTop(); - const candidates = []; - for (const element of panel.querySelectorAll("div, span")) { - const text = cleanLine(element.innerText || element.textContent || ""); - if (!text || text.includes("·")) { - continue; - } - if ( - element.closest( - "button, a, [role='button'], [role='tab'], [role='tablist'], " - + "[data-item-id], [data-review-id], div.F7nice", - ) - ) { - continue; - } - if ( - Array.from(element.children).some( - (child) => cleanLine(child.innerText || child.textContent || "") === text, - ) - ) { - continue; - } - const top = elementTop(element); - if (top === null || top <= titleBottom || top >= boundaryTop) { - continue; - } - candidates.push({top, text}); - } - candidates.sort((left, right) => left.top - right.top); - return candidates[0]?.text || null; - }; - - const normalizeCount = (value) => { - if (!value) { - return 0; - } - const text = value.trim().toUpperCase(); - let multiplier = 1; - if (text.includes("K")) { - multiplier = 1000; - } else if (text.includes("M")) { - multiplier = 1000000; - } else if (text.includes("萬") || text.includes("万")) { - multiplier = 10000; - } - const numeric = parseFloat(text.replace(/[,\sKM萬万]/g, "")); - return Number.isFinite(numeric) ? numeric * multiplier : 0; - }; - - const reviewKeywords = ["review", "reviews", "評論", "クチコミ"]; - const reviewCountPattern = new RegExp( - "([0-9][0-9,.\\s]*[KM萬万]?)[ ]*" - + "(?:reviews?|評論|クチコミ|件のクチコミ|件の Google クチコミ|則評論|篇評論)", - "i", - ); - const reviewCountPatternReverse = new RegExp( - "(?:reviews?|評論|クチコミ)\\s*[(]([0-9][0-9,.\\s]*[KM萬万]?)[)]", - "i", - ); - - let reviewCount = null; - let reviewSource = null; - let bestCount = 0; - - const considerCount = (candidate, source) => { - if (!candidate) { - return; - } - const count = normalizeCount(candidate); - if (count <= 0) { - return; - } - if (count > bestCount) { - bestCount = count; - reviewCount = candidate.trim(); - reviewSource = source; - } - }; - - for (const span of panel.querySelectorAll("div.F7nice span")) { - const text = span.innerText?.trim() || ""; - const match = text.match(/^\(?([0-9][0-9,.\s]*[KM萬万]?)\)?$/i); - if (!match) { - continue; - } - if (/^[0-9]+([.,][0-9]+)?$/.test(match[1]) && normalizeCount(match[1]) < 10) { - continue; - } - considerCount(match[1], "f7nice"); - } - - const reviewSummaryBoundaryTop = () => { - const addressRow = addressRowElement(); - if (addressRow) { - const rect = addressRow.getBoundingClientRect(); - if (rect.height > 0) { - return rect.top; - } - } - const panelRect = panel.getBoundingClientRect(); - return panelRect.top + 520; - }; - const isOverviewReviewCountElement = (element) => { - if (element.closest("div.F7nice")) { - return true; - } - const rect = element.getBoundingClientRect(); - if (rect.height <= 0) { - return false; - } - return rect.top < reviewSummaryBoundaryTop(); - }; - - for (const element of panel.querySelectorAll("[aria-label]")) { - if (!isOverviewReviewCountElement(element)) { - continue; - } - const label = element.getAttribute("aria-label") || ""; - if (!reviewKeywords.some((keyword) => label.toLowerCase().includes(keyword.toLowerCase()))) { - continue; - } - const match = label.match(reviewCountPattern) || label.match(reviewCountPatternReverse); - if (match) { - considerCount(match[1], "aria-label"); - } - } - - if (!reviewCount) { - for (const tab of panel.querySelectorAll("div[role='tablist'] button")) { - const text = tab.innerText?.trim() || ""; - if (!reviewKeywords.some((keyword) => text.toLowerCase().includes(keyword.toLowerCase()))) { - continue; - } - const match = text.match(/([0-9][0-9,.\s]*[KM萬万]?)/i); - if (match) { - considerCount(match[1], "tab"); - } - } - } - - const mainPhotoUrl = firstImageUrl([ - "div.RZ66Rb button[jsaction*='heroHeaderImage'] img", - "button[jsaction*='heroHeaderImage'] img", - "div.ZKCDEc [data-photo-index='0'] img", - "[data-photo-index='0'] img", - "[data-photo-index] img", - ]) - || firstBackgroundImageUrl([ - "div.RZ66Rb button[jsaction*='heroHeaderImage']", - "button[jsaction*='heroHeaderImage']", - "div.ZKCDEc [data-photo-index='0']", - "[data-photo-index='0']", - "[data-photo-index]", - ]); - const photoUrl = mainPhotoUrl - || firstAttr(["meta[property='og:image']", "meta[itemprop='image']"], "content", document); - - const shallowPath = (element) => { - const parts = []; - let current = element; - for (let i = 0; i < 4 && current && current.nodeType === Node.ELEMENT_NODE; i += 1) { - let part = current.tagName.toLowerCase(); - const id = current.getAttribute("data-item-id"); - const role = current.getAttribute("role"); - if (id) { - part += `[data-item-id="${id}"]`; - } else if (role) { - part += `[role="${role}"]`; - } else if (current.classList?.length) { - part += "." + Array.from(current.classList).slice(0, 2).join("."); - } - parts.unshift(part); - current = current.parentElement; - } - return parts.join(" > "); - }; - const nearbyText = (element) => { - const texts = []; - const parent = element.parentElement; - if (!parent) { - return texts; - } - for (const child of parent.children) { - const text = cleanLine(child.innerText || child.textContent || ""); - if (text && !texts.includes(text)) { - texts.push(text); - } - if (texts.length >= 4) { - break; - } - } - return texts; - }; - const collectDomCandidates = () => { - const selectors = [ - "[data-item-id]", - "button[aria-label]", - "a[aria-label]", - "[role='button'][aria-label]", - ".Io6YTe", - ".DkEaL", - ".F7nice", - "div[role='tablist'] button", - ]; - const candidates = []; - const seen = new Set(); - for (const selector of selectors) { - for (const element of panel.querySelectorAll(selector)) { - const text = cleanLine(element.innerText || element.textContent || ""); - const ariaLabel = cleanLine(element.getAttribute("aria-label") || ""); - const dataItemId = cleanLine(element.getAttribute("data-item-id") || ""); - if (!text && !ariaLabel && !dataItemId) { - continue; - } - if (text.length > 240 || ariaLabel.length > 240) { - continue; - } - const key = `${selector}\n${text}\n${ariaLabel}\n${dataItemId}`; - if (seen.has(key)) { - continue; - } - seen.add(key); - candidates.push({ - text, - tag: element.tagName.toLowerCase(), - role: cleanLine(element.getAttribute("role") || ""), - aria_label: ariaLabel, - data_item_id: dataItemId, - selector_hint: shallowPath(element), - nearby_text: nearbyText(element), - }); - if (candidates.length >= 120) { - return candidates; - } - } - } - return candidates; - }; - const collectReviewTopics = () => { - const selectors = [ - "button[jsaction*='review']", - "button[aria-label*='review' i]", - "button[role='radio']", - "button[aria-pressed]", - "div[role='button'][aria-label]", - ]; - const topics = []; - const seen = new Set(); - for (const selector of selectors) { - for (const element of panel.querySelectorAll(selector)) { - const text = cleanLine(element.innerText || element.textContent || ""); - const ariaLabel = cleanLine(element.getAttribute("aria-label") || ""); - const candidate = /[0-9]/.test(text) - ? text - : (/[0-9]/.test(ariaLabel) ? ariaLabel : text || ariaLabel); - if (!candidate || candidate.length > 120 || !/[0-9]/.test(candidate)) { - continue; - } - const key = `${candidate}\n${ariaLabel}`; - if (seen.has(key)) { - continue; - } - seen.add(key); - topics.push({ - text: candidate, - aria_label: ariaLabel, - source: selector, - }); - } - } - return topics; - }; - const priceSymbols = "(?:[$€£¥₩₹₫฿₱₦₺₴₽]|SGD|USD|EUR|GBP|JPY|TWD|NT\\$|HK\\$|CA\\$|A\\$)"; - const pricePattern = new RegExp( - "(?:^|\\s|·)((?:\\${1,4})|" + priceSymbols - + "\\s*[0-9][0-9,.\u00a0\\s]*(?:\\+|[-–]\\s*" + priceSymbols - + "?\\s*[0-9][0-9,.\u00a0\\s]*)?)", - "i", - ); - const exactNumericPricePattern = new RegExp( - "^" + priceSymbols + "\\s*[0-9][0-9,.\u00a0\\s]*$", - "i", - ); - const extractPrice = (value) => { - const text = cleanLine(value); - const match = text.match(pricePattern); - if (!match?.[1]) { - return null; - } - return cleanLine(match[1].replace(/\u00a0/g, " ")); - }; - const looksLikePriceRangeText = (value) => { - const text = cleanLine(value); - if (!text) { - return false; - } - if (/^\${1,4}$/.test(text)) { - return true; - } - return text.includes("·") && pricePattern.test(text); - }; - const headingAliases = (value) => Array.isArray(value) ? value : [value]; - const normalizedHeading = (value) => cleanLine(value).toLowerCase(); - const sectionRootByHeading = (headingText) => { - const aliases = headingAliases(headingText) - .map((value) => normalizedHeading(value)) - .filter(Boolean); - if (aliases.length === 0) { - return null; - } - for (const heading of panel.querySelectorAll("h2, h3, [role='heading']")) { - const text = normalizedHeading(heading.innerText || heading.textContent || ""); - if (!aliases.includes(text)) { - continue; - } - return ( - heading.closest(".m6QErb, section, [role='region'], [data-section-id]") - || heading.parentElement - || null - ); - } - return null; - }; - const collectLeafPrices = (root) => { - if (!root) { - return []; - } - const prices = []; - const seen = new Set(); - for (const element of root.querySelectorAll("*")) { - const text = cleanLine(element.innerText || element.textContent || ""); - if (!text || text.length > 48) { - continue; - } - if ( - Array.from(element.children).some( - (child) => cleanLine(child.innerText || child.textContent || "") === text, - ) - ) { - continue; - } - const price = extractPrice(text); - if (!price || price !== text || !exactNumericPricePattern.test(price)) { - continue; - } - if (seen.has(price)) { - continue; - } - seen.add(price); - prices.push(price); - } - return prices; - }; - const providerLabelFromUrl = (href) => { - try { - const host = new URL(href).hostname.replace(/^www\./, ""); - const base = host.split(".")[0] || host; - return base - .replace(/[-_]+/g, " ") - .replace(/\b\w/g, (char) => char.toUpperCase()); - } catch { - return "Find a Table"; - } - }; - const reservationLabel = (element, href) => { - const actionPrefixPattern = new RegExp( - "^(?:find a table|reserve|make a reservation|book a table)" - + "(?:\\s+(?:with|on|at|via))?\\s*", - "i", - ); - const raw = cleanLine( - element.innerText - || element.textContent - || element.getAttribute("aria-label") - || element.getAttribute("title") - || "", - ); - const cleaned = raw - .replace(actionPrefixPattern, "") - .replace(/\s+(?:opens in new tab|website)$/i, "") - .trim(); - return cleaned || providerLabelFromUrl(href); - }; - const collectReservationLinks = () => { - const links = []; - const seen = new Set(); - const reservationPattern = new RegExp( - String.raw`\b(find a table|reserve|reservation|book a table)\b`, - "i", - ); - for (const element of panel.querySelectorAll("a[href]")) { - const href = element.href || element.getAttribute("href") || ""; - if (!/^https?:\/\//i.test(href) || seen.has(href)) { - continue; - } - const evidence = [ - element.innerText, - element.textContent, - element.getAttribute("aria-label"), - element.getAttribute("title"), - element.getAttribute("data-item-id"), - ].filter(Boolean).join(" "); - if (!reservationPattern.test(evidence)) { - continue; - } - seen.add(href); - links.push({ - label: reservationLabel(element, href), - url: href, - }); - if (links.length >= 8) { - break; - } - } - return links; - }; - const roomOverlayPrice = () => { - const selectors = [ - ".rlmNhf button[aria-label]", - "button[aria-label*='per night' i]", - "button[aria-label*='prices from' i]", - ]; - for (const selector of selectors) { - for (const element of document.querySelectorAll(selector)) { - const price = extractPrice(element.getAttribute("aria-label") || ""); - if (price && exactNumericPricePattern.test(price)) { - return price; - } - } - } - return null; - }; - const detailsBoundaryTop = () => { - const selectors = [ - `[data-item-id="address"]`, - `[data-item-id="authority"]`, - `[data-item-id="oloc"]`, - `[data-item-id="locatedin"]`, - `button[data-item-id^="phone:"]`, - ]; - let boundary = Number.POSITIVE_INFINITY; - for (const selector of selectors) { - for (const element of panel.querySelectorAll(selector)) { - const rect = element.getBoundingClientRect(); - if (rect.height <= 0) { - continue; - } - boundary = Math.min(boundary, rect.top); - } - } - const addressRow = addressRowElement(); - if (addressRow) { - const rect = addressRow.getBoundingClientRect(); - if (rect.height > 0) { - boundary = Math.min(boundary, rect.top); - } - } - return Number.isFinite(boundary) ? boundary : Number.POSITIVE_INFINITY; - }; - const structuralOfferSignals = () => { - const titleTop = ( - titleElement?.getBoundingClientRect()?.top - || panel.getBoundingClientRect().top - ); - const boundaryTop = detailsBoundaryTop(); - const prices = []; - const seenPrices = new Set(); - for (const element of panel.querySelectorAll("*")) { - const text = cleanLine(element.innerText || element.textContent || ""); - if (!text || text.length > 48) { - continue; - } - const rect = element.getBoundingClientRect(); - if (rect.height <= 0 || rect.top <= titleTop || rect.top >= boundaryTop) { - continue; - } - if ( - Array.from(element.children).some( - (child) => cleanLine(child.innerText || child.textContent || "") === text, - ) - ) { - continue; - } - const price = extractPrice(text); - if (!price || price !== text || !exactNumericPricePattern.test(price)) { - continue; - } - if (seenPrices.has(price)) { - continue; - } - seenPrices.add(price); - prices.push(price); - } - const kind = ( - prices.length > 0 - ? ( - roomOverlayPrice() - || panel.querySelector(`[data-item-id="place-info-links:"]`) - ? "room" - : "admission" - ) - : null - ); - return {kind, prices}; - }; - const priceRangeValue = () => { - const roots = [ - panel.querySelector(".dmRWX"), - panel.querySelector(".F7nice")?.parentElement, - panel.querySelector(".F7nice"), - panel, - ].filter(Boolean); - for (const root of roots) { - const text = cleanLine(root.innerText || root.textContent || ""); - if (!looksLikePriceRangeText(text)) { - continue; - } - const match = text.match(pricePattern); - if (match?.[1]) { - return cleanLine(match[1].replace(/\u00a0/g, " ")); - } - } - return null; - }; - const structuralOffers = structuralOfferSignals(); - - return { - name: firstText(titleSelectors), - secondary_name: firstText(["h2.bwoZTb span", "h2.bwoZTb"]), - rating: firstText([ - "div.F7nice > span > span[aria-hidden='true']:first-child", - "span.ceNzKf[role='img']", - "span[role='img'][aria-label*='star']", - ]), - review_count: reviewCount, - review_count_source: reviewSource, - category: firstText([ - "button[jsaction*='category']", - ".skqShb .fontBodyMedium button", - "button.DkEaL", - ]), - price_range: priceRangeValue(), - address: addressValue(), - located_in: itemValue("locatedin"), - status: firstText(["div.OqCZI .ZDu9vd", "div.OqCZI .o0Svhf"]), - website: firstAttr(["a[data-item-id='authority']"], "href", document) || itemValue("authority"), - reservation_links: collectReservationLinks(), - phone: firstText([ - "button[data-item-id^='phone:'] .Io6YTe", - "button[data-item-id^='phone:']", - ]), - plus_code: itemValue("oloc"), - description: descriptionValue(), - review_topics: collectReviewTopics(), - admission_prices: collectLeafPrices(sectionRootByHeading([ - "Admission", - "Ticket prices", - "Entry fee", - "Entrance fee", - "入場", - "入場料", - "入園料", - "票價", - "票价", - "門票", - "门票", - ])), - room_prices: collectLeafPrices(sectionRootByHeading([ - "Compare prices", - "Compare room prices", - "Room prices", - "價格比較", - "价格比较", - "比較價格", - "比較房價", - "料金を比較", - "価格を比較", - "宿泊料金を比較", - ])), - structural_offer_kind: structuralOffers.kind, - structural_offer_prices: structuralOffers.prices, - room_price_overlay: roomOverlayPrice(), - dom_candidates: collectDomCandidates(), - main_photo_url: mainPhotoUrl, - photo_url: photoUrl, - panel_text: panel?.innerText || "", - body_text: document.body?.innerText || "", - limited_view: (document.body?.innerText || "") - .toLowerCase() - .includes("limited view of google maps"), - }; -} -""" +_PLACE_JS_EXTRACTOR = resources.files("gmaps_scraper.data").joinpath( + "place_extractor.js" +).read_text(encoding="utf-8") _PLACE_REVIEW_SIGNAL_JS = r""" () => { """ + _PLACE_PANEL_HELPERS_JS + r""" @@ -3534,6 +2782,11 @@ def _extract_preview_place_enrichment(payload_text: str) -> dict[str, object]: if description is not None: enrichment["description"] = description + rating_summary = _extract_preview_rating_summary(root) + if rating_summary is not None: + enrichment["rating"] = rating_summary[0] + enrichment["review_count"] = rating_summary[1] + coordinates = _extract_preview_coordinates(root) if coordinates is not None: enrichment["lat"] = coordinates[0] @@ -4687,6 +3940,77 @@ def _snapshot_contains_ui_action_name_candidate(snapshot: Mapping[str, object]) return any(_looks_like_ui_action_label(line) for line in lines) +def _extract_preview_rating_summary(root: list[object]) -> tuple[float, int] | None: + candidates: set[tuple[float, int]] = set() + for node in _iter_lists(root): + if not _looks_like_compact_rating_summary_node(node): + continue + numeric_items = [ + (index, value) + for index, value in enumerate(node) + if isinstance(value, (int, float)) and not isinstance(value, bool) + ] + for rating_index, rating_value in numeric_items: + rating = _preview_rating_value(rating_value) + if rating is None: + continue + for count_index, count_value in numeric_items: + if rating_index == count_index or abs(rating_index - count_index) > 2: + continue + count = _preview_review_count_value(count_value) + if count is None: + continue + if _rating_count_pair_is_ambiguous(node, count): + continue + candidates.add((rating, count)) + if len(candidates) == 1: + return next(iter(candidates)) + return None + + +def _looks_like_compact_rating_summary_node(node: list[object]) -> bool: + if not 2 <= len(node) <= 8: + return False + return not any( + isinstance(value, str) and _clean_numeric_price_text(value) is not None + for value in node + ) + + +def _preview_rating_value(value: int | float) -> float | None: + if isinstance(value, int): + return None + rating = float(value) + if not 1.0 <= rating <= 5.0: + return None + if rating.is_integer(): + return None + return round(rating, 1) + + +def _preview_review_count_value(value: int | float) -> int | None: + if isinstance(value, float): + if not value.is_integer(): + return None + value = int(value) + if value < 10 or value > 50_000_000: + return None + return value + + +def _rating_count_pair_is_ambiguous(node: list[object], count: int) -> bool: + count_text = str(count) + return any( + isinstance(value, str) + and count_text in value + and ( + _clean_numeric_price_text(value) is not None + or re.search(r"\b(?:year|since|founded|established|opened)\b", value, re.IGNORECASE) + ) + for value in node + ) + + def _extract_preview_coordinates(root: list[object]) -> tuple[float, float] | None: fallback_e7_pair: tuple[float, float] | None = None for node in _iter_lists(root): diff --git a/tests/fixtures/place_pages/about.html b/tests/fixtures/place_pages/about.html new file mode 100644 index 0000000..df1f4a6 --- /dev/null +++ b/tests/fixtures/place_pages/about.html @@ -0,0 +1,41 @@ + + + + + Fixture About + + + +
+

Fixture About Bakery

+
+ + (678) +
+ + + Website + +
+ + + +
+
+
+

Service options

+
    +
  • Takeaway
    • +
  • Delivery
    • +
+
+
+
+ + diff --git a/tests/fixtures/place_pages/limited_view.html b/tests/fixtures/place_pages/limited_view.html new file mode 100644 index 0000000..f3f72bc --- /dev/null +++ b/tests/fixtures/place_pages/limited_view.html @@ -0,0 +1,18 @@ + + + + + Limited View Fixture + + + +
+

Limited Fixture Place

+

This is a limited view of Google Maps.

+
+ + diff --git a/tests/fixtures/place_pages/overview.html b/tests/fixtures/place_pages/overview.html new file mode 100644 index 0000000..6465731 --- /dev/null +++ b/tests/fixtures/place_pages/overview.html @@ -0,0 +1,41 @@ + + + + + + Fixture Place + + + +
+
+ +
+

Fixture Cafe

+
+ + (1,234) +
+ +
Compact fixture description for the place overview.
+ + Website + + +
+ + + +
+
+ + diff --git a/tests/fixtures/place_pages/reviews.html b/tests/fixtures/place_pages/reviews.html new file mode 100644 index 0000000..4cb6c5f --- /dev/null +++ b/tests/fixtures/place_pages/reviews.html @@ -0,0 +1,51 @@ + + + + + Fixture Reviews + + + +
+
+ +
+

Fixture Reviews Cafe

+
+ + (2,345) +
+ + + Website + +
+ + + +
+
+ + +
+
+ +
Reviewer Placeholder
+ + 2 months ago +
Useful visible review snippet.
+
+
+ + diff --git a/tests/fixtures/place_pages/search_result.html b/tests/fixtures/place_pages/search_result.html new file mode 100644 index 0000000..2f62071 --- /dev/null +++ b/tests/fixtures/place_pages/search_result.html @@ -0,0 +1,32 @@ + + + + + Search Result Fixture + + + +
+

Results

+
+
+ + Fixture Search Cafe + +
+ + 4.7 (321) +
+
Cafe · 10 Search Street
+
Small local search-result description.
+
+
+
+ + diff --git a/tests/fixtures/preview_payloads/ambiguous_rating_review_count.txt b/tests/fixtures/preview_payloads/ambiguous_rating_review_count.txt new file mode 100644 index 0000000..e507915 --- /dev/null +++ b/tests/fixtures/preview_payloads/ambiguous_rating_review_count.txt @@ -0,0 +1,32 @@ +)]}' +[ + [ + "price-summary", + 4.8, + 101, + "NT$101" + ], + [ + "history-summary", + 4.6, + 2024, + "Founded 2024" + ], + [ + "non-adjacent-summary", + 4.7, + null, + null, + 912 + ], + [ + "conflicting-summary-a", + 4.5, + 120 + ], + [ + "conflicting-summary-b", + 4.6, + 130 + ] +] diff --git a/tests/fixtures/preview_payloads/rating_review_count.txt b/tests/fixtures/preview_payloads/rating_review_count.txt new file mode 100644 index 0000000..6e3270d --- /dev/null +++ b/tests/fixtures/preview_payloads/rating_review_count.txt @@ -0,0 +1,17 @@ +)]}' +[ + [ + "preview-place", + [ + "rating-summary", + 4.7, + 832 + ], + [ + null, + null, + 25.2048, + 55.2708 + ] + ] +] diff --git a/tests/test_place_scraper.py b/tests/test_place_scraper.py index 3c217b4..76a7b82 100644 --- a/tests/test_place_scraper.py +++ b/tests/test_place_scraper.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import os import tempfile import unittest from pathlib import Path @@ -12,11 +13,13 @@ PlaceScrapeResult, ) from gmaps_scraper.place_scraper import ( + _PLACE_ABOUT_PANEL_JS, _PLACE_ABOUT_TAB_CLICK_JS, _PLACE_DETAIL_READY_JS, _PLACE_JS_EXTRACTOR, _PLACE_RESERVATION_BUTTON_CLICK_JS, _PLACE_RESERVATION_DIALOG_JS, + _PLACE_REVIEW_SNIPPET_JS, _PLACE_REVIEW_TAB_CLICK_JS, _PLACE_REVIEW_TOPIC_JS, _PLACE_SEARCH_RESULT_CLICK_JS, @@ -62,7 +65,150 @@ collect_place_snapshot, scrape_places, ) -from gmaps_scraper.scraper import BrowserSessionConfig, HttpSessionConfig, ScrapeError +from gmaps_scraper.scraper import ( + BrowserSessionConfig, + HttpSessionConfig, + ScrapeError, + _launch_browser_context, +) + +_FIXTURE_DIR = Path(__file__).parent / "fixtures" / "place_pages" +_PREVIEW_FIXTURE_DIR = Path(__file__).parent / "fixtures" / "preview_payloads" +_SKIP_BROWSER_FIXTURE_TESTS_ENV = "GMAPS_SCRAPER_SKIP_BROWSER_FIXTURE_TESTS" + + +class PlaceJsExtractorFixtureTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + if os.environ.get(_SKIP_BROWSER_FIXTURE_TESTS_ENV): + raise unittest.SkipTest( + f"{_SKIP_BROWSER_FIXTURE_TESTS_ENV} is set; skipping browser fixture tests." + ) + try: + cls.context = _launch_browser_context( + headless=True, + browser_session=BrowserSessionConfig(window_size=(1280, 720)), + ) + except Exception as exc: + raise unittest.SkipTest( + f"Browser fixture tests require a launchable browser: {exc}" + ) from exc + + @classmethod + def tearDownClass(cls) -> None: + context = getattr(cls, "context", None) + if context is not None: + context.close() + + def _evaluate_fixture(self, fixture_name: str, script: str = _PLACE_JS_EXTRACTOR) -> object: + page = self.context.new_page() + try: + page.goto( + (_FIXTURE_DIR / fixture_name).as_uri(), + wait_until="domcontentloaded", + timeout=5_000, + ) + return page.evaluate(script) + finally: + page.close() + + def test_overview_fixture_extracts_core_place_fields(self) -> None: + snapshot = self._evaluate_fixture("overview.html") + + self.assertIsInstance(snapshot, dict) + self.assertEqual(snapshot["name"], "Fixture Cafe") + self.assertEqual(snapshot["rating"], "4.6") + self.assertEqual(snapshot["review_count"], "1,234") + self.assertEqual(snapshot["review_count_source"], "f7nice") + self.assertEqual(snapshot["category"], "Coffee shop") + self.assertEqual( + snapshot["description"], + "Compact fixture description for the place overview.", + ) + self.assertEqual(snapshot["address"], "123 Test Street, Fixture City") + self.assertEqual(snapshot["website"], "https://fixture.example/") + self.assertEqual(snapshot["phone"], "+1 555-0100") + self.assertEqual(snapshot["plus_code"], "7FG8+22 Fixture City") + self.assertEqual( + snapshot["main_photo_url"], + "https://lh3.googleusercontent.com/place-overview.jpg", + ) + self.assertFalse(snapshot["limited_view"]) + + def test_reviews_fixture_keeps_reviewer_photo_out_of_place_photo(self) -> None: + snapshot = self._evaluate_fixture("reviews.html") + reviews = self._evaluate_fixture("reviews.html", _PLACE_REVIEW_SNIPPET_JS) + + self.assertIsInstance(snapshot, dict) + self.assertEqual(snapshot["name"], "Fixture Reviews Cafe") + self.assertEqual( + snapshot["main_photo_url"], + "https://lh3.googleusercontent.com/place-review-hero.jpg", + ) + self.assertNotEqual( + snapshot["main_photo_url"], + "https://lh3.googleusercontent.com/reviewer-placeholder.jpg", + ) + self.assertEqual( + [(topic["text"], topic["aria_label"]) for topic in snapshot["review_topics"]], + [ + ("coffee 19", "coffee, mentioned in 19 reviews"), + ("service 12", "service, mentioned in 12 reviews"), + ], + ) + self.assertIsInstance(reviews, list) + self.assertEqual(reviews[0]["author"], "Reviewer Placeholder") + self.assertEqual(reviews[0]["text"], "Useful visible review snippet.") + + def test_about_fixture_preserves_core_fields_and_reads_about_sections(self) -> None: + snapshot = self._evaluate_fixture("about.html") + about_sections = self._evaluate_fixture("about.html", _PLACE_ABOUT_PANEL_JS) + + self.assertIsInstance(snapshot, dict) + self.assertEqual(snapshot["name"], "Fixture About Bakery") + self.assertEqual(snapshot["category"], "Bakery") + self.assertEqual(snapshot["address"], "9 About Avenue, Fixture City") + self.assertEqual(snapshot["phone"], "+1 555-0102") + self.assertEqual( + about_sections, + [ + { + "title": "Service options", + "items": [ + { + "label": "Takeaway", + "aria_label": "Offers takeaway", + "source": "about_panel", + }, + { + "label": "Delivery", + "aria_label": "Offers delivery", + "source": "about_panel", + }, + ], + } + ], + ) + + def test_limited_view_fixture_sets_limited_view_flag(self) -> None: + snapshot = self._evaluate_fixture("limited_view.html") + + self.assertIsInstance(snapshot, dict) + self.assertEqual(snapshot["name"], "Limited Fixture Place") + self.assertTrue(snapshot["limited_view"]) + + def test_search_result_fixture_is_not_detail_ready(self) -> None: + snapshot = self._evaluate_fixture("search_result.html") + detail_ready = self._evaluate_fixture("search_result.html", _PLACE_DETAIL_READY_JS) + search_candidate = self._evaluate_fixture( + "search_result.html", + _PLACE_SEARCH_RESULT_CLICK_JS, + ) + + self.assertIsInstance(snapshot, dict) + self.assertEqual(snapshot["name"], "Results") + self.assertFalse(detail_ready) + self.assertIs(search_candidate, False) class PlaceScraperTests(unittest.TestCase): @@ -81,19 +227,7 @@ def test_build_place_details_from_snapshot_rejects_saved_list_resolution(self) - llm_policy="on_quality_failure", ) - def test_place_js_extractor_skips_review_scoped_photo_nodes(self) -> None: - self.assertIn('element.closest("[data-review-id]")', _PLACE_JS_EXTRACTOR) - self.assertIn("root.querySelectorAll(selector)", _PLACE_JS_EXTRACTOR) - self.assertIn(r"return /(^|\W)reviews?(\W|$)/i.test(label);", _PLACE_JS_EXTRACTOR) - self.assertIn("const descriptionValue = () => {", _PLACE_JS_EXTRACTOR) - self.assertIn('firstText([".WeS02d", ".PYvSYb"])', _PLACE_JS_EXTRACTOR) - self.assertIn("const descriptionBoundaryTop = () => {", _PLACE_JS_EXTRACTOR) - self.assertIn("[role='button'], [role='tab'], [role='tablist']", _PLACE_JS_EXTRACTOR) - - def test_place_js_extractor_uses_structural_panel_photo_selectors(self) -> None: - self.assertIn("div.RZ66Rb button[jsaction*='heroHeaderImage'] img", _PLACE_JS_EXTRACTOR) - self.assertIn("div.ZKCDEc [data-photo-index='0'] img", _PLACE_JS_EXTRACTOR) - self.assertIn("[data-photo-index='0'] img", _PLACE_JS_EXTRACTOR) + def test_place_js_extractor_avoids_broad_photo_selectors(self) -> None: self.assertNotIn("button[jsaction*='image'] img", _PLACE_JS_EXTRACTOR) self.assertNotIn("button[jsaction*='photo'] img", _PLACE_JS_EXTRACTOR) self.assertNotIn("button[aria-label^='Photo of'] img", _PLACE_JS_EXTRACTOR) @@ -712,11 +846,6 @@ def test_parse_price_amount_handles_localized_grouping(self) -> None: self.assertEqual(_parse_price_amount("1.234,56"), 1234.56) self.assertEqual(_parse_price_amount("1,234.56"), 1234.56) - def test_place_js_extractor_prefers_data_item_address_rows(self) -> None: - self.assertIn('const legacy = itemValue("address");', _PLACE_JS_EXTRACTOR) - self.assertIn("if (legacy) {", _PLACE_JS_EXTRACTOR) - self.assertIn('`[data-item-id="${itemId}"] .Io6YTe`', _PLACE_JS_EXTRACTOR) - def test_place_js_extractor_falls_back_to_address_icon_rows(self) -> None: self.assertIn('const isAddressIcon = (icon) => {', _PLACE_JS_EXTRACTOR) self.assertIn('glyph === ""', _PLACE_JS_EXTRACTOR) @@ -724,13 +853,6 @@ def test_place_js_extractor_falls_back_to_address_icon_rows(self) -> None: self.assertIn('icon.closest(".LCF4w', _PLACE_JS_EXTRACTOR) self.assertIn('const rowValue = (row) => {', _PLACE_JS_EXTRACTOR) - def test_place_js_extractor_reads_structured_info_rows(self) -> None: - self.assertIn("button[jsaction*='category']", _PLACE_JS_EXTRACTOR) - self.assertIn("button[data-item-id^='phone:'] .Io6YTe", _PLACE_JS_EXTRACTOR) - self.assertIn('plus_code: itemValue("oloc")', _PLACE_JS_EXTRACTOR) - self.assertIn("a[data-item-id='authority']", _PLACE_JS_EXTRACTOR) - self.assertIn("panel,\n ].filter(Boolean);", _PLACE_JS_EXTRACTOR) - def test_place_js_extractor_collects_quote_sections_separately(self) -> None: self.assertLess( _PLACE_JS_EXTRACTOR.index("const collectLeafPrices ="), @@ -1062,6 +1184,57 @@ def test_build_place_details_prefers_preview_over_search_card_fallback(self) -> assert details.diagnostics is not None self.assertEqual(details.diagnostics.field_sources.get("category"), "preview") + def test_build_place_details_backfills_preview_rating_summary_sources(self) -> None: + preview = _extract_preview_place_enrichment( + (_PREVIEW_FIXTURE_DIR / "rating_review_count.txt").read_text(encoding="utf-8") + ) + details = _build_place_details_from_snapshot( + "https://www.google.com/maps/place/Limited+Fixture", + snapshot={ + "resolved_url": "https://www.google.com/maps/place/Limited+Fixture", + "dom": { + "name": "Limited Fixture", + "limited_view": True, + }, + "search_result": {}, + "preview": preview, + }, + llm_fallback=None, + llm_policy="never", + ) + + self.assertEqual(details.rating, 4.7) + self.assertEqual(details.review_count, 832) + assert details.diagnostics is not None + self.assertEqual(details.diagnostics.field_sources.get("rating"), "preview") + self.assertEqual(details.diagnostics.field_sources.get("review_count"), "preview") + + def test_build_place_details_keeps_dom_rating_summary_over_preview(self) -> None: + details = _build_place_details_from_snapshot( + "https://www.google.com/maps/place/Fixture", + snapshot={ + "resolved_url": "https://www.google.com/maps/place/Fixture", + "dom": { + "name": "Fixture", + "rating": "4.9", + "review_count": "1,234", + }, + "search_result": {}, + "preview": { + "rating": 4.7, + "review_count": 832, + }, + }, + llm_fallback=None, + llm_policy="never", + ) + + self.assertEqual(details.rating, 4.9) + self.assertEqual(details.review_count, 1234) + assert details.diagnostics is not None + self.assertEqual(details.diagnostics.field_sources.get("rating"), "dom") + self.assertEqual(details.diagnostics.field_sources.get("review_count"), "dom") + def test_build_place_details_prefers_selected_card_when_search_open_fails(self) -> None: details = _build_place_details_from_snapshot( "https://www.google.com/maps/search/?api=1&query=Lola+Underground", @@ -1545,6 +1718,31 @@ def test_extract_preview_place_enrichment_backfills_core_fields(self) -> None: self.assertEqual(enrichment["lng"], 139.7127216) self.assertEqual(enrichment["google_place_id"], "ChIJ8T36HxCLGGARvpARPDyaKLA") + def test_extract_preview_place_enrichment_reads_rating_summary_fixture(self) -> None: + enrichment = _extract_preview_place_enrichment( + (_PREVIEW_FIXTURE_DIR / "rating_review_count.txt").read_text(encoding="utf-8") + ) + + self.assertEqual(enrichment["rating"], 4.7) + self.assertEqual(enrichment["review_count"], 832) + + def test_extract_preview_place_enrichment_accepts_year_range_review_count(self) -> None: + payload = ")]}'\n" + json.dumps([["rating-summary", 4.7, 2000]]) + enrichment = _extract_preview_place_enrichment(payload) + + self.assertEqual(enrichment["rating"], 4.7) + self.assertEqual(enrichment["review_count"], 2000) + + def test_extract_preview_place_enrichment_rejects_ambiguous_rating_counts(self) -> None: + enrichment = _extract_preview_place_enrichment( + (_PREVIEW_FIXTURE_DIR / "ambiguous_rating_review_count.txt").read_text( + encoding="utf-8" + ) + ) + + self.assertNotIn("rating", enrichment) + self.assertNotIn("review_count", enrichment) + def test_extract_preview_description_preserves_text_starting_with_open(self) -> None: description = _extract_preview_description( [ @@ -2637,10 +2835,6 @@ def test_search_result_candidate_js_decodes_place_id_safely(self) -> None: _PLACE_SEARCH_RESULT_CLICK_JS, ) - def test_place_js_extractor_keeps_place_page_description_selectors(self) -> None: - self.assertIn('const direct = firstText([".WeS02d", ".PYvSYb"])', _PLACE_JS_EXTRACTOR) - self.assertIn("description: descriptionValue()", _PLACE_JS_EXTRACTOR) - def test_looks_like_google_maps_place_url_accepts_google_tlds_only(self) -> None: self.assertTrue( _looks_like_google_maps_place_url(