diff --git a/README.md b/README.md index 01ee4d5..952385b 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ # WebsiteScorecard -CLI to scan websites from a CSV and enrich rows with check results (SSL certificate status, and more over time). +CLI to scan websites from a CSV and enrich rows with check results (SSL certificate status, trilingual and more over time). ## Prerequisites - Python 3.10 or newer - A CSV file with a column containing website URLs (domains or full URLs) +- **Playwright Chromium binaries** — required by the `trilingual` check (see step 2 below) ## Quick start @@ -31,7 +32,17 @@ Activate the virtual environment in any new terminal session before running comm source .venv/bin/activate # Windows: .venv\Scripts\activate ``` -### 2. Test data +### 2. Install Playwright browser binaries + +The `trilingual` check uses a headless Chromium browser to verify language switchers and localStorage-based locale detection. After installing the package, run: + +```bash +playwright install chromium +``` + +> **Note:** This is a one-time step per environment. Without it, any scan that includes `--checks trilingual` will crash at runtime with a "Executable doesn't exist" error. + +### 3. Test data The repo includes a sample CSV at `data/mins_depts_test.csv` with Sri Lankan ministries and departments: @@ -45,10 +56,10 @@ Ministry,Ministry of Digital Economy,midec.gov.lk The `URL` column contains bare domains (e.g. `defence.lk`). Full URLs such as `https://example.com/path` also work. -### 3. Run a scan +### 4. Run a scan ```bash -websitescorecard scan data/mins_depts_test.csv --column URL --checks ssl +websitescorecard scan data/mins_depts_test.csv --column URL --checks ssl,trilingual ``` This writes `data/mins_depts_test_scored.csv` by default (same name as input with `_scored` inserted). @@ -56,15 +67,15 @@ This writes `data/mins_depts_test_scored.csv` by default (same name as input wit Specify an output file: ```bash -websitescorecard scan data/mins_depts_test.csv -c URL -o data/mins_depts_scored.csv --checks ssl +websitescorecard scan data/mins_depts_test.csv -c URL -o data/mins_depts_scored.csv --checks ssl,trilingual ``` -### 4. Check the output +### 5. Check the output ```csv -Type,Institution Name,URL,ssl_status,ssl_error -Ministry,Ministry of Defence,defence.lk,valid, -Ministry,Ministry of Digital Economy,midec.gov.lk,valid, +Type,Institution Name,URL,ssl_status,ssl_error,trilingual_status,trilingual_error +Ministry,Ministry of Defence,defence.lk,valid,,trilingual, +Ministry,Ministry of Digital Economy,midec.gov.lk,valid,,trilingual, ... ``` @@ -79,6 +90,15 @@ Ministry,Ministry of Digital Economy,midec.gov.lk,valid, The `ssl_error` column contains the underlying error message when something went wrong. For scorecard reporting, `expired` and `invalid` can be grouped as cert problems; `unreachable` rows can be flagged separately for CSV cleanup. +| `trilingual_status` | Meaning | +|-------------------|---------| +| `TRILINGUAL` | Page supports all 3 official languages (Sinhala, Tamil, English) | +| `NON_TRILINGUAL` | Page is missing one or more official languages | +| `TIMEOUT` | Some checks timed out; result may be incomplete | +| `UNREACHABLE` | Could not connect to evaluate trilingual status (bad domain, DNS failure, timeout, connection error) | + +The `trilingual_details` column records which detection method confirmed trilingual support (e.g. `HTML_ATTRIBUTE`, `URL_LOCALIZATION`, `UNICODE_CONTENT`, `GOOGLE_TRANSLATE_API`, `BROWSER_STORAGE`, `VERIFIED_SWITCHER_CLICK`, `DEEPLINK_CRAWL`). The `trilingual_deeplink` column lists any internal links that were crawled during detection. + ## CLI reference ```bash @@ -90,7 +110,7 @@ websitescorecard scan INPUT_CSV -c COLUMN [OPTIONS] | `INPUT_CSV` | Input CSV file path | | `-c, --column` | Column containing website URLs (**required**) | | `-o, --output` | Output path (default: `{input}_scored.csv`) | -| `--checks` | Comma-separated checks to run (e.g. `ssl`) (**required**) | +| `--checks` | Comma-separated checks to run (e.g. `ssl,trilingual`) (**required**) | | `--concurrency` | Parallel workers (default: 5) | | `--timeout` | Socket timeout per check in seconds (default: 10) | | `--no-error-columns` | Omit `*_error` detail columns | @@ -98,8 +118,8 @@ websitescorecard scan INPUT_CSV -c COLUMN [OPTIONS] Examples: ```bash -# SSL check on the sample data with 10 parallel workers and 15s timeout -websitescorecard scan data/mins_depts_test.csv -c URL --checks ssl --concurrency 10 --timeout 15 +# SSL and trilingual checks on the sample data with 10 parallel workers and 15s timeout +websitescorecard scan data/mins_depts_test.csv -c URL --checks ssl,trilingual --concurrency 10 --timeout 15 # View all options websitescorecard scan --help diff --git a/data/mins_depts_test_scored.csv b/data/mins_depts_test_scored.csv index e959f7f..c5a4c03 100644 --- a/data/mins_depts_test_scored.csv +++ b/data/mins_depts_test_scored.csv @@ -1,50 +1,50 @@ -Type,Institution Name,URL,ssl_status,ssl_error -Ministry,Ministry of Defence,defence.lk,valid, -Ministry,"Ministry of Finance, Planning and Economic Development",treasury.gov.lk,valid, -Ministry,Ministry of Digital Economy,midec.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Ministry,"Ministry of Foreign Affairs, Foreign Employment and Tourism",mfa.gov.lk,valid, -Ministry,"Ministry of Education, Higher Education and Vocational Education",moe.gov.lk,valid, -Ministry,"Ministry of Agriculture, Livestock, Land and Irrigation",agrimin.gov.lk,valid, -Ministry,"Ministry of Transport, Highways and Urban Development",transport.gov.lk,valid, -Ministry,"Ministry of Public Administration, Provincial Councils and Local Government",pubad.gov.lk,valid, -Ministry,Ministry of Health and Mass Media,health.gov.lk,valid, -Ministry,Ministry of Justice and National Integration,moj.gov.lk,valid, -Ministry,Ministry of Public Security and Parliamentary Affairs,pubsec.gov.lk,valid, -Ministry,Ministry of Industry and Entrepreneurship Development,industry.gov.lk,valid, -Ministry,"Ministry of Trade, Commerce, Food Security and Cooperative Development",ment.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Ministry,Ministry of Environment,env.gov.lk,valid, -Ministry,Ministry of Energy,energy.gov.lk,valid, -Ministry,"Ministry of Housing, Construction and Water Supply",housingmin.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Ministry,Ministry of Labor,labourmin.gov.lk,valid, -Ministry,Ministry of Plantation and Community Infrastructure,plantationindustries.gov.lk,invalid,"Hostname mismatch, certificate is not valid for 'plantationindustries.gov.lk'." -Ministry,Ministry of Ports and Civil Aviation,portaviation.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Ministry,"Ministry of Fisheries, Aquatic and Ocean Resources",fisheries.gov.lk,valid, -Ministry,Ministry of Women and Child Affairs,childwomen.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Ministry,"Ministry of Rural Development, Social Security and Community Empowerment",socialservices.gov.lk,valid, -Ministry,"Ministry of Buddhasasana, Religious and Cultural Affairs",mbra.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Ministry,Ministry of Youth Affairs and Sports,sportsmin.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Ministry,Ministry of Science and Technology,most.gov.lk,valid, -Department,Department of Immigration and Emigration,immigration.gov.lk,valid, -Department,Department of Motor Traffic,dmt.gov.lk,valid, -Department,Department of Registration of Persons,drp.gov.lk,valid, -Department,Department of Registrar General,rgd.gov.lk,valid, -Department,Department of Inland Revenue,ird.gov.lk,valid, -Department,Sri Lanka Customs,customs.gov.lk,valid, -Department,Department of Pensions,pensions.gov.lk,valid, -Department,Department of Census and Statistics,statistics.gov.lk,unreachable,[Errno 54] Connection reset by peer -Department,Department of Government Information,dgi.gov.lk,valid, -Department,Department of Government Printing,documents.gov.lk,valid, -Department,Department of Excise,excise.gov.lk,valid, -Department,Department of National Budget,nationalbudget.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Department,Department of Meteorology,meteo.gov.lk,valid, -Department,Department of Wildlife Conservation,dwc.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Department,Department of Forest Conservation,forestdept.gov.lk,expired,unable to get local issuer certificate -Department,Department of National Zoological Gardens,colombozoo.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Department,Department of Examinations,doenets.lk,valid, -Department,Department of Archaeology,archaeology.gov.lk,valid, -Department,Department of Samurdhi,samurdhi.gov.lk,valid, -Department,LSF,opensource.lk,invalid,"Hostname mismatch, certificate is not valid for 'opensource.lk'." -Expired,Expired SSL,expired.badssl.com,unreachable,[Errno 54] Connection reset by peer -Self-Signed,Self-Signed SSL,self-signed.badssl.com,invalid,self-signed certificate -Wrong Host,Wrong Host SSL,wrong.host.badssl.com,invalid,"Hostname mismatch, certificate is not valid for 'wrong.host.badssl.com'." -Never SSL,Never SSL,neverssl.com,valid, +Type,Institution Name,URL,ssl_status,ssl_error,trilingual_status,trilingual_error,trilingual_details,trilingual_deeplink +Ministry,Ministry of Defence,defence.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Ministry,"Ministry of Finance, Planning and Economic Development",treasury.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Ministry,Ministry of Digital Economy,midec.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Ministry,"Ministry of Foreign Affairs, Foreign Employment and Tourism",mfa.gov.lk,valid,,TRILINGUAL,,DEEPLINK_CRAWL: UNICODE_CONTENT,https://www.mfa.gov.lk/en +Ministry,"Ministry of Education, Higher Education and Vocational Education",moe.gov.lk,valid,,TRILINGUAL,,HTML_ATTRIBUTE, +Ministry,"Ministry of Agriculture, Livestock, Land and Irrigation",agrimin.gov.lk,valid,,NON_TRILINGUAL,"missing languages: si, ta",, +Ministry,"Ministry of Transport, Highways and Urban Development",transport.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Ministry,"Ministry of Public Administration, Provincial Councils and Local Government",pubad.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Ministry,Ministry of Health and Mass Media,health.gov.lk,valid,,TRILINGUAL,,"DEEPLINK_CRAWL: HTML_ATTRIBUTE, URL_LOCALIZATION",https://www.health.gov.lk/home/ +Ministry,Ministry of Justice and National Integration,moj.gov.lk,valid,,TRILINGUAL,,"HTML_ATTRIBUTE, URL_LOCALIZATION", +Ministry,Ministry of Public Security and Parliamentary Affairs,pubsec.gov.lk,valid,,TRILINGUAL,,"HTML_ATTRIBUTE, URL_LOCALIZATION", +Ministry,Ministry of Industry and Entrepreneurship Development,industry.gov.lk,valid,,TRILINGUAL,,"URL_LOCALIZATION, UNICODE_CONTENT", +Ministry,"Ministry of Trade, Commerce, Food Security and Cooperative Development",ment.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Ministry,Ministry of Environment,env.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Ministry,Ministry of Energy,energy.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Ministry,"Ministry of Housing, Construction and Water Supply",housingmin.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Ministry,Ministry of Labor,labourmin.gov.lk,valid,,NON_TRILINGUAL,"missing languages: si, ta",, +Ministry,Ministry of Plantation and Community Infrastructure,plantationindustries.gov.lk,invalid,"Hostname mismatch, certificate is not valid for 'plantationindustries.gov.lk'.",TRILINGUAL,,GOOGLE_TRANSLATE_API, +Ministry,Ministry of Ports and Civil Aviation,portaviation.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Ministry,"Ministry of Fisheries, Aquatic and Ocean Resources",fisheries.gov.lk,valid,,TRILINGUAL,,HTML_ATTRIBUTE, +Ministry,Ministry of Women and Child Affairs,childwomen.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Ministry,"Ministry of Rural Development, Social Security and Community Empowerment",socialservices.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Ministry,"Ministry of Buddhasasana, Religious and Cultural Affairs",mbra.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Ministry,Ministry of Youth Affairs and Sports,sportsmin.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Ministry,Ministry of Science and Technology,most.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Department,Department of Immigration and Emigration,immigration.gov.lk,valid,,TRILINGUAL,,UNICODE_CONTENT, +Department,Department of Motor Traffic,dmt.gov.lk,valid,,TRILINGUAL,,"HTML_ATTRIBUTE, URL_LOCALIZATION", +Department,Department of Registration of Persons,drp.gov.lk,valid,,TRILINGUAL,,VERIFIED_SWITCHER_CLICK, +Department,Department of Registrar General,rgd.gov.lk,valid,,TRILINGUAL,,"URL_LOCALIZATION, UNICODE_CONTENT", +Department,Department of Inland Revenue,ird.gov.lk,valid,,UNREACHABLE,Can't Access,, +Department,Sri Lanka Customs,customs.gov.lk,valid,,TRILINGUAL,,GOOGLE_TRANSLATE_API, +Department,Department of Pensions,pensions.gov.lk,valid,,TRILINGUAL,,"HTML_ATTRIBUTE, URL_LOCALIZATION", +Department,Department of Census and Statistics,statistics.gov.lk,unreachable,[Errno 54] Connection reset by peer,TRILINGUAL,,VERIFIED_SWITCHER_CLICK, +Department,Department of Government Information,dgi.gov.lk,valid,,TRILINGUAL,,VERIFIED_SWITCHER_CLICK, +Department,Department of Government Printing,documents.gov.lk,valid,,NON_TRILINGUAL,"missing languages: si, ta",, +Department,Department of Excise,excise.gov.lk,valid,,TRILINGUAL,,"HTML_ATTRIBUTE, URL_LOCALIZATION", +Department,Department of National Budget,nationalbudget.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Department,Department of Meteorology,meteo.gov.lk,valid,,TRILINGUAL,,UNICODE_CONTENT, +Department,Department of Wildlife Conservation,dwc.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",TRILINGUAL,,DEEPLINK_CRAWL: UNICODE_CONTENT,"https://www.dwc.gov.lk/?page_id=203, https://www.dwc.gov.lk/?page_id=257, https://www.dwc.gov.lk/?p=914, https://www.dwc.gov.lk/?paged=2" +Department,Department of Forest Conservation,forestdept.gov.lk,expired,unable to get local issuer certificate,TRILINGUAL,,"HTML_ATTRIBUTE, URL_LOCALIZATION", +Department,Department of National Zoological Gardens,colombozoo.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Department,Department of Examinations,doenets.lk,valid,,TRILINGUAL,,BROWSER_STORAGE, +Department,Department of Archaeology,archaeology.gov.lk,valid,,TRILINGUAL,,GOOGLE_TRANSLATE_API, +Department,Department of Samurdhi,samurdhi.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Department,LSF,opensource.lk,valid,,NON_TRILINGUAL,"missing languages: si, ta",, +Expired,Expired SSL,expired.badssl.com,expired,certificate has expired,TIMEOUT,"Some checks timed out. Potentially missing: si, ta",, +Self-Signed,Self-Signed SSL,self-signed.badssl.com,invalid,self-signed certificate,UNREACHABLE,Can't Access,, +Wrong Host,Wrong Host SSL,wrong.host.badssl.com,unreachable,[Errno 54] Connection reset by peer,TIMEOUT,"Some checks timed out. Potentially missing: si, ta",, +Never SSL,Never SSL,neverssl.com,valid,,NON_TRILINGUAL,"missing languages: si, ta",, diff --git a/pyproject.toml b/pyproject.toml index d1686c5..f73ec41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,11 +12,16 @@ requires-python = ">=3.10" dependencies = [ "typer[all]>=0.12.0", "rich>=13.0.0", + "requests>=2.31.0", + "beautifulsoup4>=4.15.0", + "playwright>=1.40.0", ] [project.optional-dependencies] dev = [ "pytest>=8.0.0", + "pytest-playwright>=0.8.0", + "pytest-mock>=3.0.0", "ruff>=0.4.0", ] diff --git a/src/websitescorecard/checks/__init__.py b/src/websitescorecard/checks/__init__.py index fd94d37..1faee5c 100644 --- a/src/websitescorecard/checks/__init__.py +++ b/src/websitescorecard/checks/__init__.py @@ -6,11 +6,13 @@ from websitescorecard.checks.base import Check from websitescorecard.checks.ssl import SSLCheck +from websitescorecard.checks.trilingual import TrilingualCheck CheckFactory = Callable[[], Check] CHECK_REGISTRY: dict[str, CheckFactory] = { "ssl": SSLCheck, + "trilingual": TrilingualCheck } diff --git a/src/websitescorecard/checks/base.py b/src/websitescorecard/checks/base.py index 4e772f4..3bd0393 100644 --- a/src/websitescorecard/checks/base.py +++ b/src/websitescorecard/checks/base.py @@ -1,6 +1,7 @@ """Base types for pluggable website checks.""" from __future__ import annotations +from typing import Optional from dataclasses import dataclass from typing import Protocol @@ -11,10 +12,12 @@ class CheckResult: status: str error: str | None = None - class Check(Protocol): name: str column: str error_column: str | None + + # Optional mapping of Result attribute name -> CSV column name + extra_columns: Optional[dict[str, str]] = None def run(self, url: str) -> CheckResult: ... diff --git a/src/websitescorecard/checks/trilingual.py b/src/websitescorecard/checks/trilingual.py new file mode 100644 index 0000000..d522da2 --- /dev/null +++ b/src/websitescorecard/checks/trilingual.py @@ -0,0 +1,643 @@ +import logging +import threading +import re +import requests +import urllib3 +from urllib.parse import urljoin, urlparse +from websitescorecard.url_utils import parse_url +from websitescorecard.checks.base import CheckResult +from bs4 import BeautifulSoup, NavigableString, Comment +from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError +from dataclasses import dataclass + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +logger = logging.getLogger(__name__) + +@dataclass(frozen=True) +class TrilingualCheckResult(CheckResult): + details: str | None = None + deeplink: str | None = None + +LANGUAGE_KEY = ['en','si','ta'] + +# each entry is values to detect and inject, order should be preserved +LANG_STORAGE_FORMATS: list[tuple[list[str], list[str]]] = [ + (['en', 'si', 'ta'], ['en', 'si', 'ta']), + (['en-US', 'en-GB', 'si-LK', 'ta-LK', 'ta-IN'], ['en-US', 'si-LK', 'ta-LK']), + (['en-us', 'en-gb', 'si-lk', 'ta-lk', 'ta-in'], ['en-us', 'si-lk', 'ta-lk']), + (['English', 'Sinhala', 'Tamil'], ['English', 'Sinhala', 'Tamil']), + (['english', 'sinhala', 'tamil'], ['english', 'sinhala', 'tamil']), +] + +# html key and tags +HTML_KEY_HREFLANG = 'hreflang' +HTML_KEY_LINK = 'link' +HTML_KEY_ALTERNATE = 'alternate' +HTML_GOOGLE_TRANSLATE_ELEMENT = 'google_translate_element' +HTML_GOOG_TE_COMBO = 'goog-te-combo' +HTML_GOOGLE_TRANSLATE_ELEMENT_JS = 'translate.google.com/translate_a/element.js' + +_playwright_lock = threading.Lock() + +class TrilingualCheck: + name = "trilingual" + column = "trilingual_status" + error_column = "trilingual_error" + extra_columns = { + "details": "trilingual_details", + "deeplink": "trilingual_deeplink" + } + + def __init__(self, timeout: float = 10.0) -> None: + """Initialise the checker with a per-request socket timeout. + Thread-local storage is created to hold per-call state safely.""" + self.timeout = timeout + self._local = threading.local() + + def _check_html_attribute(self, url: str, soup: BeautifulSoup) -> tuple[bool, list[str]]: + """Check hreflang link tags and the root attribute for all three languages. + Returns (all_found, missing_language_codes).""" + base_domain = urlparse(url).netloc + if base_domain.startswith('www.'): + base_domain = base_domain[4:] + + hreflangs = [] + for link in soup.find_all(HTML_KEY_LINK, rel=HTML_KEY_ALTERNATE): + hreflang = link.get(HTML_KEY_HREFLANG) + href = link.get('href') + if not hreflang: + continue + if href: + absolute_url = urljoin(url, href) + href_domain = urlparse(absolute_url).netloc + if href_domain.startswith('www.'): + href_domain = href_domain[4:] + + # Check if href domain is the same or a subdomain + if base_domain and not href_domain.endswith(base_domain): + continue + hreflangs.append(hreflang) + + found_langs = {lang.split('-')[0].lower() for lang in hreflangs if lang} + + # Also check the base tag's lang attribute (e.g. ) + if soup.html and soup.html.get('lang'): + found_langs.add(soup.html.get('lang').split('-')[0].lower()) + + missing = [lang for lang in LANGUAGE_KEY if lang not in found_langs] + return (len(missing) == 0, missing) + + def _check_google_translate(self, soup: BeautifulSoup) -> str | None: + """Detect whether a Google Translate or GTranslate widget is present in the page. + Returns the widget type string ('google_translate' or 'gtranslate'), or None.""" + # Native Google Translate elements + if soup.find(id=HTML_GOOGLE_TRANSLATE_ELEMENT): + return 'google_translate' + if soup.find(class_=HTML_GOOG_TE_COMBO): + return 'google_translate' + + # GTranslate plugin + if soup.find(class_="gtranslate_wrapper"): + return 'gtranslate' + + for script in soup.find_all('script', src=True): + src = script.get('src', '').lower() + if HTML_GOOGLE_TRANSLATE_ELEMENT_JS in src: + return 'google_translate' + if 'gtranslate.net' in src: + return 'gtranslate' + + return None + + def _verify_google_translate_languages( + self, url: str + ) -> tuple[bool, list[str]]: + """Use Playwright to confirm the Google Translate/GTranslate widget on *url* offers all three languages. + Returns (all_found, missing_language_codes).""" + # Language codes used inside Google Translate / GTranslate elements + # that are part of the translate widget. + option_values: list[str] = page.evaluate("""() => { + const results = []; + // Google Translate native combo + const goog = document.querySelector('.goog-te-combo, select.gt_selector, select.notranslate'); + if (goog) { + for (const opt of goog.options) { + results.push(opt.value.toLowerCase()); + results.push(opt.textContent.trim().toLowerCase()); + } + } + // GTranslate wrappers often use a