From c31c4dc49cb574391c0210f55d5bed68b67478b1 Mon Sep 17 00:00:00 2001 From: ChanukaUOJ Date: Mon, 22 Jun 2026 16:50:26 +0530 Subject: [PATCH 01/20] trilingual check has been added to the checks with basic helpers --- README.md | 25 ++-- pyproject.toml | 3 + src/websitescorecard/checks/__init__.py | 2 + src/websitescorecard/checks/trilingual.py | 149 ++++++++++++++++++++++ 4 files changed, 170 insertions(+), 9 deletions(-) create mode 100644 src/websitescorecard/checks/trilingual.py diff --git a/README.md b/README.md index 01ee4d5..e27ed7f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # WebsiteScorecard -CLI to scan websites from a CSV and enrich rows with check results (SSL certificate status, and more over time). +CLI to scan websites from a CSV and enrich rows with check results (SSL certificate status, trilingual and more over time). ## Prerequisites @@ -48,7 +48,7 @@ The `URL` column contains bare domains (e.g. `defence.lk`). Full URLs such as `h ### 3. Run a scan ```bash -websitescorecard scan data/mins_depts_test.csv --column URL --checks ssl +websitescorecard scan data/mins_depts_test.csv --column URL --checks ssl,trilingual ``` This writes `data/mins_depts_test_scored.csv` by default (same name as input with `_scored` inserted). @@ -56,15 +56,15 @@ This writes `data/mins_depts_test_scored.csv` by default (same name as input wit Specify an output file: ```bash -websitescorecard scan data/mins_depts_test.csv -c URL -o data/mins_depts_scored.csv --checks ssl +websitescorecard scan data/mins_depts_test.csv -c URL -o data/mins_depts_scored.csv --checks ssl,trilingual ``` ### 4. Check the output ```csv -Type,Institution Name,URL,ssl_status,ssl_error -Ministry,Ministry of Defence,defence.lk,valid, -Ministry,Ministry of Digital Economy,midec.gov.lk,valid, +Type,Institution Name,URL,ssl_status,ssl_error,trilingual_status,trilingual_error +Ministry,Ministry of Defence,defence.lk,valid,,trilingual, +Ministry,Ministry of Digital Economy,midec.gov.lk,valid,,trilingual, ... ``` @@ -79,6 +79,13 @@ Ministry,Ministry of Digital Economy,midec.gov.lk,valid, The `ssl_error` column contains the underlying error message when something went wrong. For scorecard reporting, `expired` and `invalid` can be grouped as cert problems; `unreachable` rows can be flagged separately for CSV cleanup. +| `trilingual_status` | Meaning | +|-------------------|---------| +| `trilingual` | Page contains all 3 official languages (Sinhala, Tamil, English) | +| `Non-trilingual` | Page is missing one or more official languages | +| `unreachable` | Could not connect to evaluate trilingual status (bad domain, DNS failure, timeout, connection error) | +| `error` | Check raised an unexpected internal error (recorded in `trilingual_error`) | + ## CLI reference ```bash @@ -90,7 +97,7 @@ websitescorecard scan INPUT_CSV -c COLUMN [OPTIONS] | `INPUT_CSV` | Input CSV file path | | `-c, --column` | Column containing website URLs (**required**) | | `-o, --output` | Output path (default: `{input}_scored.csv`) | -| `--checks` | Comma-separated checks to run (e.g. `ssl`) (**required**) | +| `--checks` | Comma-separated checks to run (e.g. `ssl,trilingual`) (**required**) | | `--concurrency` | Parallel workers (default: 5) | | `--timeout` | Socket timeout per check in seconds (default: 10) | | `--no-error-columns` | Omit `*_error` detail columns | @@ -98,8 +105,8 @@ websitescorecard scan INPUT_CSV -c COLUMN [OPTIONS] Examples: ```bash -# SSL check on the sample data with 10 parallel workers and 15s timeout -websitescorecard scan data/mins_depts_test.csv -c URL --checks ssl --concurrency 10 --timeout 15 +# SSL and trilingual checks on the sample data with 10 parallel workers and 15s timeout +websitescorecard scan data/mins_depts_test.csv -c URL --checks ssl,trilingual --concurrency 10 --timeout 15 # View all options websitescorecard scan --help diff --git a/pyproject.toml b/pyproject.toml index d1686c5..002b5c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,9 @@ requires-python = ">=3.10" dependencies = [ "typer[all]>=0.12.0", "rich>=13.0.0", + "aiohttp>=3.14.1", + "requests>=2.31.0", + "beautifulsoup4>=4.15.0", ] [project.optional-dependencies] diff --git a/src/websitescorecard/checks/__init__.py b/src/websitescorecard/checks/__init__.py index fd94d37..1faee5c 100644 --- a/src/websitescorecard/checks/__init__.py +++ b/src/websitescorecard/checks/__init__.py @@ -6,11 +6,13 @@ from websitescorecard.checks.base import Check from websitescorecard.checks.ssl import SSLCheck +from websitescorecard.checks.trilingual import TrilingualCheck CheckFactory = Callable[[], Check] CHECK_REGISTRY: dict[str, CheckFactory] = { "ssl": SSLCheck, + "trilingual": TrilingualCheck } diff --git a/src/websitescorecard/checks/trilingual.py b/src/websitescorecard/checks/trilingual.py new file mode 100644 index 0000000..18705ce --- /dev/null +++ b/src/websitescorecard/checks/trilingual.py @@ -0,0 +1,149 @@ +import re +import requests +import urllib3 +from websitescorecard.url_utils import parse_url +from websitescorecard.checks.base import CheckResult +from bs4 import BeautifulSoup + +LANGUAGE_KEY = ['en','si','ta'] + +# html key and tags +HTML_KEY_HREFLANG = 'hreflang' +HTML_KEY_LINK = 'link' +HTML_KEY_ALTERNATE = 'alternate' +HTML_GOOGLE_TRANSLATE_ELEMENT = 'google_translate_element' +HTML_GOOG_TE_COMBO = 'goog-te-combo' +HTML_GOOGLE_TRANSLATE_ELEMENT_JS = 'translate.google.com/translate_a/element.js' + + +class TrilingualCheck: + name = "trilingual" + column = "trilingual_status" + error_column = "trilingual_error" + + def __init__(self, timeout: float = 10.0) -> None: + self.timeout = timeout + + def check_html_attribute(self, soup: BeautifulSoup) -> tuple[bool, list[str]]: + hreflangs = [link.get(HTML_KEY_HREFLANG) for link in soup.find_all(HTML_KEY_LINK, rel=HTML_KEY_ALTERNATE)] + found_langs = {lang.split('-')[0].lower() for lang in hreflangs if lang} + + # Also check the base tag's lang attribute (e.g. ) + if soup.html and soup.html.get('lang'): + found_langs.add(soup.html.get('lang').split('-')[0].lower()) + + missing = [lang for lang in LANGUAGE_KEY if lang not in found_langs] + return (len(missing) == 0, missing) + + def check_google_translate(self, soup: BeautifulSoup) -> bool: + # Native Google Translate elements + if soup.find(id=HTML_GOOGLE_TRANSLATE_ELEMENT): + return True + if soup.find(class_=HTML_GOOG_TE_COMBO): + return True + + # GTranslate plugin (used by archaeology.gov.lk and others) + if soup.find(class_="gtranslate_wrapper"): + return True + + for script in soup.find_all('script', src=True): + src = script.get('src', '').lower() + if HTML_GOOGLE_TRANSLATE_ELEMENT_JS in src or 'gtranslate.net' in src: + return True + + return False + + def check_url_localization_patterns(self, soup: BeautifulSoup) -> tuple[bool, list[str]]: + hrefs = [a.get('href') or '' for a in soup.find_all('a', href=True)] + + # Exact patterns for short and long forms, plus optional locale codes (e.g. -US, -lk) + locale_suffix = r'(?:-[a-zA-Z]+)?' + lang_regex_map = { + 'en': rf'en(?:glish|g)?{locale_suffix}', + 'si': rf'si(?:nhala|n)?{locale_suffix}', + 'ta': rf'ta(?:mil|m)?{locale_suffix}' + } + + found_langs: set[str] = set() + for lang in LANGUAGE_KEY: + # Use the specific regex for this language to avoid false positives like "sigiriya" matching "si" + pattern = lang_regex_map.get(lang, lang) + + # matches path segments: /en/ /english/ /sinhala/ + path_pattern = re.compile(rf'(?:^|/)({pattern})(?:/|$|\?|#)', re.IGNORECASE) + # matches query params: ?lang=en &lang=english + query_pattern = re.compile(rf'[?&]lang=({pattern})(?:&|$)', re.IGNORECASE) + + if any(path_pattern.search(href) or query_pattern.search(href) for href in hrefs): + found_langs.add(lang) + + missing = [lang for lang in LANGUAGE_KEY if lang not in found_langs] + return (len(missing) == 0, missing) + + def check_unicode_content(self, soup: BeautifulSoup) -> tuple[bool, list[str]]: + text = soup.get_text() + found_langs: set[str] = set() + + # Check English (basic Latin alphabet) + if re.search(r'[a-zA-Z]', text): + found_langs.add('en') + + # Check Sinhala unicode block + if re.search(r'[\u0D80-\u0DFF]', text): + found_langs.add('si') + + # Check Tamil unicode block + if re.search(r'[\u0B80-\u0BFF]', text): + found_langs.add('ta') + + missing = [lang for lang in LANGUAGE_KEY if lang not in found_langs] + return (len(missing) == 0, missing) + + def run(self, url: str) -> CheckResult: + try: + parsed = parse_url(url) + except ValueError as exc: + return CheckResult(status="unreachable", error=str(exc)) + + try: + headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'} + + # Try http without www first + try: + response = requests.get(f'http://{parsed.hostname}', timeout=self.timeout, headers=headers) + needs_fallback = (response.status_code == 404) + except Exception: + needs_fallback = True + + # Fallback to https://www if needed + if needs_fallback: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + response = requests.get(f'https://www.{parsed.hostname}', timeout=self.timeout, headers=headers, verify=False) + + html = response.text + soup = BeautifulSoup(html, 'html.parser') + + # HTML lang attribute and header check + passed_attribute, missing_attribute = self.check_html_attribute(soup) + + # check google translator use + passed_google_translator_use = self.check_google_translate(soup) + + # URL Localization patterns + passed_url_localization, missing_url_localization = self.check_url_localization_patterns(soup) + + # Direct body text contents (Unicode) + passed_unicode, missing_unicode = self.check_unicode_content(soup) + + # Languages missing from ALL checks are truly missing + missing_set = set(missing_attribute) & set(missing_url_localization) & set(missing_unicode) + + # If no languages are missing (combined), or it uses Google Translate, it passes! + if len(missing_set) == 0 or passed_google_translator_use: + return CheckResult(status="trilingual", error=None) + else: + return CheckResult(status="Non-trilingual", error=f"missing languages: {', '.join(sorted(missing_set))}") + + except Exception as exc: + return CheckResult(status="unreachable", error="Can't Access") + \ No newline at end of file From bea3eafae955b02671c5a3a4f9dc83916c3253c2 Mon Sep 17 00:00:00 2001 From: ChanukaUOJ Date: Mon, 22 Jun 2026 16:53:56 +0530 Subject: [PATCH 02/20] aiohttp package removed --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 002b5c3..c1625bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,6 @@ requires-python = ">=3.10" dependencies = [ "typer[all]>=0.12.0", "rich>=13.0.0", - "aiohttp>=3.14.1", "requests>=2.31.0", "beautifulsoup4>=4.15.0", ] From 0f9bb79c60ef914a4d2531a7d72c388f4131e9ea Mon Sep 17 00:00:00 2001 From: ChanukaUOJ Date: Mon, 22 Jun 2026 18:58:54 +0530 Subject: [PATCH 03/20] playwrite case has been added to check the local storage --- pyproject.toml | 1 + src/websitescorecard/checks/trilingual.py | 97 +++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c1625bb..c2d2b7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "rich>=13.0.0", "requests>=2.31.0", "beautifulsoup4>=4.15.0", + "pytest-playwright>=0.8.0", ] [project.optional-dependencies] diff --git a/src/websitescorecard/checks/trilingual.py b/src/websitescorecard/checks/trilingual.py index 18705ce..8d44248 100644 --- a/src/websitescorecard/checks/trilingual.py +++ b/src/websitescorecard/checks/trilingual.py @@ -1,3 +1,5 @@ +from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError +import threading import re import requests import urllib3 @@ -7,6 +9,15 @@ LANGUAGE_KEY = ['en','si','ta'] +# each entry is values to detect and inject, order should be preserved +LANG_STORAGE_FORMATS: list[tuple[list[str], list[str]]] = [ + (['en', 'si', 'ta'], ['en', 'si', 'ta']), + (['en-US', 'en-GB', 'si-LK', 'ta-LK', 'ta-IN'], ['en-US', 'si-LK', 'ta-LK']), + (['en-us', 'en-gb', 'si-lk', 'ta-lk', 'ta-in'], ['en-us', 'si-lk', 'ta-lk']), + (['English', 'Sinhala', 'Tamil'], ['English', 'Sinhala', 'Tamil']), + (['english', 'sinhala', 'tamil'], ['english', 'sinhala', 'tamil']), +] + # html key and tags HTML_KEY_HREFLANG = 'hreflang' HTML_KEY_LINK = 'link' @@ -15,6 +26,7 @@ HTML_GOOG_TE_COMBO = 'goog-te-combo' HTML_GOOGLE_TRANSLATE_ELEMENT_JS = 'translate.google.com/translate_a/element.js' +_playwright_lock = threading.Lock() class TrilingualCheck: name = "trilingual" @@ -99,6 +111,85 @@ def check_unicode_content(self, soup: BeautifulSoup) -> tuple[bool, list[str]]: missing = [lang for lang in LANGUAGE_KEY if lang not in found_langs] return (len(missing) == 0, missing) + def check_browser_storage_keys(self, url) -> tuple[bool, list[str]]: + try: + with _playwright_lock: + with sync_playwright() as p: + browser = p.chromium.launch() + page = browser.new_page() + try: + # Wait for the load event, then give JavaScript 3 seconds to build the DOM. + page.goto(url, wait_until="load", timeout=self.timeout * 1000) + page.wait_for_timeout(3000) + except PlaywrightTimeoutError: + pass + + # Extract localStorage directly + storage = page.evaluate("""()=>{ + const s = []; + for (let i = 0; i < localStorage.length; i++) { + s.push({ + key: localStorage.key(i), + value: localStorage.getItem(localStorage.key(i)) + }); + } + return s; + }""") + + # Search for any key that holds a language value and detect its format + target_key = None + injection_set = [] + + for item in storage: + val = str(item.get('value', '')) + for detect_vals, inject_vals in LANG_STORAGE_FORMATS: + if val in detect_vals: + target_key = item.get('key') + injection_set = inject_vals + break + if target_key: + break + + if target_key: + # Inject all 3 values and verify unicode + missing_injection = [] + langs = LANGUAGE_KEY + + for i in range(3): + base_lang = langs[i] + inject_val = injection_set[i] + + page.evaluate(f"window.localStorage.setItem('{target_key}', '{inject_val}')") + try: + page.reload(wait_until="load", timeout=self.timeout * 1000) + page.wait_for_timeout(3000) + except PlaywrightTimeoutError: + pass + + current_html = page.content() + current_soup = BeautifulSoup(current_html, 'html.parser') + _, missing_unicode = self.check_unicode_content(current_soup) + + if base_lang in missing_unicode: + missing_injection.append(base_lang) + + if len(missing_injection) == 0: + browser.close() + return (True, []) + + html = page.content() + browser.close() + + soup = BeautifulSoup(html, 'html.parser') + _, missing_url_localization = self.check_url_localization_patterns(soup) + _, missing_unicode = self.check_unicode_content(soup) + + missing_set = set(missing_url_localization) & set(missing_unicode) + missing = list(missing_set) + return (len(missing) == 0, missing) + except Exception: + return (False, list(LANGUAGE_KEY)) + def run(self, url: str) -> CheckResult: try: parsed = parse_url(url) @@ -141,6 +232,12 @@ def run(self, url: str) -> CheckResult: # If no languages are missing (combined), or it uses Google Translate, it passes! if len(missing_set) == 0 or passed_google_translator_use: return CheckResult(status="trilingual", error=None) + + website_lang, missing_browser = self.check_browser_storage_keys(response.url) + missing_set = missing_set & set(missing_browser) + + if len(missing_set) == 0: + return CheckResult(status="trilingual", error=None) else: return CheckResult(status="Non-trilingual", error=f"missing languages: {', '.join(sorted(missing_set))}") From a2ac5003638a3ec551ead34cdb578476e068b234 Mon Sep 17 00:00:00 2001 From: ChanukaUOJ Date: Mon, 22 Jun 2026 19:43:02 +0530 Subject: [PATCH 04/20] security issue in playwright localstorage injection is fixed(BOT COMMENT) --- src/websitescorecard/checks/trilingual.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/websitescorecard/checks/trilingual.py b/src/websitescorecard/checks/trilingual.py index 8d44248..a4a73c2 100644 --- a/src/websitescorecard/checks/trilingual.py +++ b/src/websitescorecard/checks/trilingual.py @@ -27,7 +27,6 @@ HTML_GOOGLE_TRANSLATE_ELEMENT_JS = 'translate.google.com/translate_a/element.js' _playwright_lock = threading.Lock() - class TrilingualCheck: name = "trilingual" column = "trilingual_status" @@ -159,7 +158,7 @@ def check_browser_storage_keys(self, url) -> tuple[bool, list[str]]: base_lang = langs[i] inject_val = injection_set[i] - page.evaluate(f"window.localStorage.setItem('{target_key}', '{inject_val}')") + page.evaluate("([key, val]) => window.localStorage.setItem(key, val)", [target_key, inject_val]) try: page.reload(wait_until="load", timeout=self.timeout * 1000) page.wait_for_timeout(3000) @@ -175,7 +174,7 @@ def check_browser_storage_keys(self, url) -> tuple[bool, list[str]]: if len(missing_injection) == 0: browser.close() - return (True, []) + return (len(missing_injection) == 0, missing_injection) html = page.content() browser.close() From 24ec24b57aad08bbf1e8784189a953fbb079b48d Mon Sep 17 00:00:00 2001 From: ChanukaUOJ Date: Tue, 23 Jun 2026 16:48:29 +0530 Subject: [PATCH 05/20] deeplink checking has been added --- src/websitescorecard/checks/base.py | 4 +- src/websitescorecard/checks/trilingual.py | 234 ++++++++++++++++++---- src/websitescorecard/runner.py | 6 + 3 files changed, 209 insertions(+), 35 deletions(-) diff --git a/src/websitescorecard/checks/base.py b/src/websitescorecard/checks/base.py index 4e772f4..0c8f242 100644 --- a/src/websitescorecard/checks/base.py +++ b/src/websitescorecard/checks/base.py @@ -1,6 +1,7 @@ """Base types for pluggable website checks.""" from __future__ import annotations +from typing import Optional from dataclasses import dataclass from typing import Protocol @@ -10,11 +11,12 @@ class CheckResult: status: str error: str | None = None - + details: Optional[str] = None class Check(Protocol): name: str column: str error_column: str | None + details_column: Optional[str] = None def run(self, url: str) -> CheckResult: ... diff --git a/src/websitescorecard/checks/trilingual.py b/src/websitescorecard/checks/trilingual.py index a4a73c2..1879e8b 100644 --- a/src/websitescorecard/checks/trilingual.py +++ b/src/websitescorecard/checks/trilingual.py @@ -3,19 +3,22 @@ import re import requests import urllib3 +from urllib.parse import urljoin, urlparse from websitescorecard.url_utils import parse_url from websitescorecard.checks.base import CheckResult from bs4 import BeautifulSoup +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + LANGUAGE_KEY = ['en','si','ta'] # each entry is values to detect and inject, order should be preserved LANG_STORAGE_FORMATS: list[tuple[list[str], list[str]]] = [ - (['en', 'si', 'ta'], ['en', 'si', 'ta']), - (['en-US', 'en-GB', 'si-LK', 'ta-LK', 'ta-IN'], ['en-US', 'si-LK', 'ta-LK']), - (['en-us', 'en-gb', 'si-lk', 'ta-lk', 'ta-in'], ['en-us', 'si-lk', 'ta-lk']), - (['English', 'Sinhala', 'Tamil'], ['English', 'Sinhala', 'Tamil']), - (['english', 'sinhala', 'tamil'], ['english', 'sinhala', 'tamil']), + (['en', 'si', 'ta'], ['en', 'si', 'ta']), + (['en-US', 'en-GB', 'si-LK', 'ta-LK', 'ta-IN'], ['en-US', 'si-LK', 'ta-LK']), + (['en-us', 'en-gb', 'si-lk', 'ta-lk', 'ta-in'], ['en-us', 'si-lk', 'ta-lk']), + (['English', 'Sinhala', 'Tamil'], ['English', 'Sinhala', 'Tamil']), + (['english', 'sinhala', 'tamil'], ['english', 'sinhala', 'tamil']), ] # html key and tags @@ -26,11 +29,15 @@ HTML_GOOG_TE_COMBO = 'goog-te-combo' HTML_GOOGLE_TRANSLATE_ELEMENT_JS = 'translate.google.com/translate_a/element.js' +DEEPLINK_COUNT = 5 + _playwright_lock = threading.Lock() + class TrilingualCheck: name = "trilingual" column = "trilingual_status" error_column = "trilingual_error" + details_column = "trilingual_details" def __init__(self, timeout: float = 10.0) -> None: self.timeout = timeout @@ -95,22 +102,101 @@ def check_unicode_content(self, soup: BeautifulSoup) -> tuple[bool, list[str]]: text = soup.get_text() found_langs: set[str] = set() - # Check English (basic Latin alphabet) - if re.search(r'[a-zA-Z]', text): + # Check English (basic Latin alphabet) - look for at least 10 alpha characters + if re.search(r'(?:[a-zA-Z].*?){50}', text): found_langs.add('en') - # Check Sinhala unicode block - if re.search(r'[\u0D80-\u0DFF]', text): + # Check Sinhala unicode block - look for at least 10 Sinhala characters + if re.search(r'(?:[\u0D80-\u0DFF].*?){50}', text): found_langs.add('si') - # Check Tamil unicode block - if re.search(r'[\u0B80-\u0BFF]', text): + # Check Tamil unicode block - look for at least 10 Tamil characters + if re.search(r'(?:[\u0B80-\u0BFF].*?){50}', text): found_langs.add('ta') missing = [lang for lang in LANGUAGE_KEY if lang not in found_langs] return (len(missing) == 0, missing) - def check_browser_storage_keys(self, url) -> tuple[bool, list[str]]: + def verify_language_buttons_via_browser(self, url: str, langs_to_check: list[str]) -> list[str]: + # Returns the list of languages STILL missing after verification + missing = list(langs_to_check) + found_langs: set[str] = set() + try: + with _playwright_lock: + with sync_playwright() as p: + browser = p.chromium.launch() + + targets = {} + if 'si' in langs_to_check: + targets['si'] = r'\bsinhala\b|සිංහල|^si$' + if 'ta' in langs_to_check: + targets['ta'] = r'\btamil\b|தமிழ்|^ta$' + + for lang, pattern in targets.items(): + # Use a fresh context (and page) per language to avoid navigation bleed-over + context = browser.new_context() + page = context.new_page() + try: + page.goto(url, wait_until="load", timeout=30000) + page.wait_for_timeout(2000) + except PlaywrightTimeoutError: + context.close() + continue + + # Try to find a clickable element matching the pattern + try: + element_handle = page.evaluate_handle(f"""() => {{ + const regex = /{pattern}/i; + const elements = Array.from(document.querySelectorAll('a, button, input[type="submit"], input[type="button"]')); + for (let el of elements) {{ + if (el.innerText && regex.test(el.innerText)) return el; + if (el.value && regex.test(el.value)) return el; + }} + return null; + }}""") + except Exception: + context.close() + continue + + if element_handle.as_element(): + # Click the element + try: + element_handle.as_element().click(timeout=5000) + except Exception: + pass + + # Always wait for load state to be stable before reading content + try: + page.wait_for_load_state("load", timeout=12000) + except Exception: + pass + page.wait_for_timeout(1000) + + # Safely extract the HTML + try: + html = page.content() + except Exception: + page.wait_for_timeout(3000) + try: + html = page.content() + except Exception: + context.close() + continue + + soup = BeautifulSoup(html, 'html.parser') + _, missing_uni = self.check_unicode_content(soup) + if lang not in missing_uni: + found_langs.add(lang) + + context.close() + + browser.close() + return [lang for lang in langs_to_check if lang not in found_langs] + except Exception: + return missing + + + def check_browser_storage_keys(self, url) -> tuple[bool, list[str], str]: try: with _playwright_lock: with sync_playwright() as p: @@ -174,21 +260,79 @@ def check_browser_storage_keys(self, url) -> tuple[bool, list[str]]: if len(missing_injection) == 0: browser.close() - return (len(missing_injection) == 0, missing_injection) + return (True, missing_injection, "browser_storage") html = page.content() browser.close() soup = BeautifulSoup(html, 'html.parser') - _, missing_url_localization = self.check_url_localization_patterns(soup) - _, missing_unicode = self.check_unicode_content(soup) + passed_url_loc, missing_url_localization = self.check_url_localization_patterns(soup) + passed_uni, missing_unicode = self.check_unicode_content(soup) missing_set = set(missing_url_localization) & set(missing_unicode) missing = list(missing_set) - return (len(missing) == 0, missing) + passed_criteria: list[str] = [] + if passed_url_loc: + passed_criteria.append("url_localization") + if passed_uni: + passed_criteria.append("unicode_content") + criteria_str = ", ".join(passed_criteria) if passed_criteria else "js_rendered" + return (len(missing) == 0, missing, f"{criteria_str}") except Exception: return (False, list(LANGUAGE_KEY)) + def _analyze_soup(self, soup: BeautifulSoup) -> tuple[set[str], bool, list[str]]: + passed_attribute, missing_attribute = self.check_html_attribute(soup) + passed_google = self.check_google_translate(soup) + passed_url_loc, missing_url_loc = self.check_url_localization_patterns(soup) + # passed_uni, missing_uni = self.check_unicode_content(soup) + + missing_set = set(missing_attribute) & set(missing_url_loc) + # & set(missing_uni) + found_langs = set(LANGUAGE_KEY) - missing_set + + passed_criteria: list[str] = [] + if passed_attribute: + passed_criteria.append("html_attribute") + if passed_url_loc: + passed_criteria.append("url_localization") + # if passed_uni: + # passed_criteria.append("unicode_content") + + return found_langs, passed_google, passed_criteria + + def _get_internal_links(self, base_url: str, soup: BeautifulSoup) -> list[str]: + lang_links = set() + other_links = set() + try: + base_domain = urlparse(base_url).netloc + for a in soup.find_all('a', href=True): + href = a.get('href', '').strip() + text = a.get_text().strip().lower() + if not href or href.startswith(('mailto:', 'tel:', 'javascript:', '#')): + continue + + absolute_url = urljoin(base_url, href) + parsed_url = urlparse(absolute_url) + + if parsed_url.netloc != base_domain: + continue + + if parsed_url.path.lower().endswith(('.pdf', '.jpg', '.jpeg', '.png', '.zip', '.doc', '.docx', '.xls', '.xlsx')): + continue + + clean_url = absolute_url.split('#')[0] + + href_lower = href.lower() + if any(x in href_lower for x in ['/si', '/ta', '=si', '=ta', 'sinhala', 'tamil']) or \ + any(x in text for x in ['sinhala', 'tamil', 'සිංහල', 'தமிழ்']): + lang_links.add(clean_url) + else: + other_links.add(clean_url) + except Exception: + pass + return list(lang_links) + list(other_links) + def run(self, url: str) -> CheckResult: try: parsed = parse_url(url) @@ -207,38 +351,60 @@ def run(self, url: str) -> CheckResult: # Fallback to https://www if needed if needs_fallback: - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) response = requests.get(f'https://www.{parsed.hostname}', timeout=self.timeout, headers=headers, verify=False) html = response.text soup = BeautifulSoup(html, 'html.parser') - # HTML lang attribute and header check - passed_attribute, missing_attribute = self.check_html_attribute(soup) + found_langs, passed_google, passed_criteria = self._analyze_soup(soup) - # check google translator use - passed_google_translator_use = self.check_google_translate(soup) + if passed_google: + return CheckResult(status="trilingual", error=None, details="google_translate") - # URL Localization patterns - passed_url_localization, missing_url_localization = self.check_url_localization_patterns(soup) + if len(found_langs) == 3: + return CheckResult(status="trilingual", error=None, details=", ".join(passed_criteria) or "static_checks") - # Direct body text contents (Unicode) - passed_unicode, missing_unicode = self.check_unicode_content(soup) - - # Languages missing from ALL checks are truly missing - missing_set = set(missing_attribute) & set(missing_url_localization) & set(missing_unicode) + # Try deeplink crawling + internal_links = self._get_internal_links(response.url, soup) + # Filter out the homepage itself + internal_links = [l for l in internal_links if l != response.url and l != response.url.rstrip('/')] + + sample_links = internal_links[:DEEPLINK_COUNT] # The first links are prioritized language links - # If no languages are missing (combined), or it uses Google Translate, it passes! - if len(missing_set) == 0 or passed_google_translator_use: - return CheckResult(status="trilingual", error=None) + for link in sample_links: + try: + dl_response = requests.get(link, timeout=self.timeout, headers=headers, verify=False) + dl_soup = BeautifulSoup(dl_response.text, 'html.parser') + dl_found, dl_google, dl_criteria = self._analyze_soup(dl_soup) + + if dl_google: + return CheckResult(status="trilingual", error=None, details="deeplink_crawl: google_translate") + + found_langs.update(dl_found) + passed_criteria.extend(dl_criteria) + + if len(found_langs) == 3: + unique_criteria = sorted(list(set(passed_criteria))) + return CheckResult(status="trilingual", error=None, details=f"deeplink_crawl: {', '.join(unique_criteria)}") + except Exception: + continue - website_lang, missing_browser = self.check_browser_storage_keys(response.url) + # Fallback to Playwright on homepage + website_lang, missing_browser, browser_method = self.check_browser_storage_keys(response.url) + missing_set = set(LANGUAGE_KEY) - found_langs missing_set = missing_set & set(missing_browser) if len(missing_set) == 0: - return CheckResult(status="trilingual", error=None) + return CheckResult(status="trilingual", error=None, details=browser_method) + + # Final Fallback: Functional Verification via click + missing_click = self.verify_language_buttons_via_browser(response.url, list(missing_set)) + missing_set = missing_set & set(missing_click) + + if len(missing_set) == 0: + return CheckResult(status="trilingual", error=None, details="verified_switcher_click") else: - return CheckResult(status="Non-trilingual", error=f"missing languages: {', '.join(sorted(missing_set))}") + return CheckResult(status="Non-trilingual", error=f"missing languages: {', '.join(sorted(missing_set))}", details="non_trilingual") except Exception as exc: return CheckResult(status="unreachable", error="Can't Access") diff --git a/src/websitescorecard/runner.py b/src/websitescorecard/runner.py index 9b8d55b..78e01ad 100644 --- a/src/websitescorecard/runner.py +++ b/src/websitescorecard/runner.py @@ -28,6 +28,8 @@ def _output_columns(original: list[str], checks: list[Check], include_errors: bo columns.append(check.column) if include_errors and check.error_column: columns.append(check.error_column) + if getattr(check, 'details_column', None): + columns.append(check.details_column) return columns @@ -39,10 +41,14 @@ def _run_checks_for_row(url: str, checks: list[Check]) -> dict[str, str]: results[check.column] = result.status if check.error_column: results[check.error_column] = result.error or "" + if getattr(check, 'details_column', None): + results[check.details_column] = result.details or "" except Exception as exc: results[check.column] = "error" if check.error_column: results[check.error_column] = f"Unexpected error: {exc}" + if getattr(check, 'details_column', None): + results[check.details_column] = "" return results From d1e796f42fb26dc902f9c8b161ec1d28a972b710 Mon Sep 17 00:00:00 2001 From: ChanukaUOJ Date: Tue, 23 Jun 2026 17:32:24 +0530 Subject: [PATCH 06/20] content checking function has been changed --- src/websitescorecard/checks/trilingual.py | 34 ++++++++++++----------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/websitescorecard/checks/trilingual.py b/src/websitescorecard/checks/trilingual.py index 1879e8b..542ba0c 100644 --- a/src/websitescorecard/checks/trilingual.py +++ b/src/websitescorecard/checks/trilingual.py @@ -29,10 +29,7 @@ HTML_GOOG_TE_COMBO = 'goog-te-combo' HTML_GOOGLE_TRANSLATE_ELEMENT_JS = 'translate.google.com/translate_a/element.js' -DEEPLINK_COUNT = 5 - _playwright_lock = threading.Lock() - class TrilingualCheck: name = "trilingual" column = "trilingual_status" @@ -99,21 +96,26 @@ def check_url_localization_patterns(self, soup: BeautifulSoup) -> tuple[bool, li return (len(missing) == 0, missing) def check_unicode_content(self, soup: BeautifulSoup) -> tuple[bool, list[str]]: - text = soup.get_text() + # Remove noise tags entirely + for tag in soup.find_all(['script', 'style', 'head', 'meta', 'noscript', 'nav', 'footer']): + tag.decompose() + + # Extract text only from meaningful content tags + content_tags = soup.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'td', 'th', 'span', 'div', 'article', 'section', 'main']) + text = ' '.join(tag.get_text(separator=' ', strip=True) for tag in content_tags) + found_langs: set[str] = set() - - # Check English (basic Latin alphabet) - look for at least 10 alpha characters - if re.search(r'(?:[a-zA-Z].*?){50}', text): + MIN_CHARS = 50 + + if len(re.findall(r'[a-zA-Z]', text)) >= MIN_CHARS: found_langs.add('en') - - # Check Sinhala unicode block - look for at least 10 Sinhala characters - if re.search(r'(?:[\u0D80-\u0DFF].*?){50}', text): + + if len(re.findall(r'[\u0D80-\u0DFF]', text)) >= MIN_CHARS: found_langs.add('si') - - # Check Tamil unicode block - look for at least 10 Tamil characters - if re.search(r'(?:[\u0B80-\u0BFF].*?){50}', text): + + if len(re.findall(r'[\u0B80-\u0BFF]', text)) >= MIN_CHARS: found_langs.add('ta') - + missing = [lang for lang in LANGUAGE_KEY if lang not in found_langs] return (len(missing) == 0, missing) @@ -208,7 +210,7 @@ def check_browser_storage_keys(self, url) -> tuple[bool, list[str], str]: page.wait_for_timeout(3000) except PlaywrightTimeoutError: pass - + # Extract localStorage directly storage = page.evaluate("""()=>{ const s = []; @@ -369,7 +371,7 @@ def run(self, url: str) -> CheckResult: # Filter out the homepage itself internal_links = [l for l in internal_links if l != response.url and l != response.url.rstrip('/')] - sample_links = internal_links[:DEEPLINK_COUNT] # The first links are prioritized language links + sample_links = internal_links[:5] # The first links are prioritized language links for link in sample_links: try: From e0e694d13fb7b39cdf1e0decce2e841b08294e1c Mon Sep 17 00:00:00 2001 From: ChanukaUOJ Date: Thu, 25 Jun 2026 13:46:52 +0530 Subject: [PATCH 07/20] google translate language checking has been added --- src/websitescorecard/checks/trilingual.py | 182 ++++++++++++++++------ 1 file changed, 135 insertions(+), 47 deletions(-) diff --git a/src/websitescorecard/checks/trilingual.py b/src/websitescorecard/checks/trilingual.py index 542ba0c..b06db3c 100644 --- a/src/websitescorecard/checks/trilingual.py +++ b/src/websitescorecard/checks/trilingual.py @@ -1,4 +1,5 @@ from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError +import logging import threading import re import requests @@ -10,6 +11,8 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +logger = logging.getLogger(__name__) + LANGUAGE_KEY = ['en','si','ta'] # each entry is values to detect and inject, order should be preserved @@ -39,7 +42,7 @@ class TrilingualCheck: def __init__(self, timeout: float = 10.0) -> None: self.timeout = timeout - def check_html_attribute(self, soup: BeautifulSoup) -> tuple[bool, list[str]]: + def _check_html_attribute(self, soup: BeautifulSoup) -> tuple[bool, list[str]]: hreflangs = [link.get(HTML_KEY_HREFLANG) for link in soup.find_all(HTML_KEY_LINK, rel=HTML_KEY_ALTERNATE)] found_langs = {lang.split('-')[0].lower() for lang in hreflangs if lang} @@ -50,25 +53,105 @@ def check_html_attribute(self, soup: BeautifulSoup) -> tuple[bool, list[str]]: missing = [lang for lang in LANGUAGE_KEY if lang not in found_langs] return (len(missing) == 0, missing) - def check_google_translate(self, soup: BeautifulSoup) -> bool: + def _check_google_translate(self, soup: BeautifulSoup) -> str | None: + """Detect if a Google Translate / GTranslate widget is present. + Returns the widget type string if found, or None.""" # Native Google Translate elements if soup.find(id=HTML_GOOGLE_TRANSLATE_ELEMENT): - return True + return 'google_translate' if soup.find(class_=HTML_GOOG_TE_COMBO): - return True + return 'google_translate' - # GTranslate plugin (used by archaeology.gov.lk and others) + # GTranslate plugin if soup.find(class_="gtranslate_wrapper"): - return True + return 'gtranslate' for script in soup.find_all('script', src=True): src = script.get('src', '').lower() - if HTML_GOOGLE_TRANSLATE_ELEMENT_JS in src or 'gtranslate.net' in src: - return True + if HTML_GOOGLE_TRANSLATE_ELEMENT_JS in src: + return 'google_translate' + if 'gtranslate.net' in src: + return 'gtranslate' - return False + return None + + def _verify_google_translate_languages( + self, url: str + ) -> tuple[bool, list[str]]: + """Use Playwright to check whether the Google Translate / GTranslate + widget on *url* actually offers English, Sinhala and Tamil. + + Returns (all_found, missing_languages). + """ + # Language codes used inside Google Translate / GTranslate elements + # that are part of the translate widget. + option_values: list[str] = page.evaluate("""() => { + const results = []; + // Google Translate native combo + const goog = document.querySelector('.goog-te-combo, select.gt_selector, select.notranslate'); + if (goog) { + for (const opt of goog.options) { + results.push(opt.value.toLowerCase()); + results.push(opt.textContent.trim().toLowerCase()); + } + } + // GTranslate wrappers often use a elements - # that are part of the translate widget. - option_values: list[str] = page.evaluate("""() => { - const results = []; - // Google Translate native combo - const goog = document.querySelector('.goog-te-combo, select.gt_selector, select.notranslate'); - if (goog) { - for (const opt of goog.options) { - results.push(opt.value.toLowerCase()); - results.push(opt.textContent.trim().toLowerCase()); - } - } - // GTranslate wrappers often use a elements + # that are part of the translate widget. + option_values: list[str] = page.evaluate("""() => { const results = []; - document.querySelectorAll('.gtranslate_wrapper a[data-gt-lang], a.gt-current-lang, a.glink').forEach(a => { - const lang = a.getAttribute('data-gt-lang') || a.getAttribute('href') || ''; - results.push(lang.toLowerCase()); - results.push(a.textContent.trim().toLowerCase()); + // Google Translate native combo + const goog = document.querySelector('.goog-te-combo, select.gt_selector, select.notranslate'); + if (goog) { + for (const opt of goog.options) { + results.push(opt.value.toLowerCase()); + results.push(opt.textContent.trim().toLowerCase()); + } + } + // GTranslate wrappers often use a options LANG_OPTION_MAP: dict[str, list[str]] = { 'en': ['en', 'english'], @@ -183,6 +184,8 @@ def _verify_google_translate_languages( return (False, list(LANGUAGE_KEY)) def _check_url_localization_patterns(self, soup: BeautifulSoup) -> tuple[bool, list[str]]: + """Scan anchor hrefs for path segments or query params matching known language identifiers. + Returns (all_found, missing_language_codes).""" hrefs = [a.get('href') or '' for a in soup.find_all('a', href=True)] # Exact patterns for short and long forms, plus optional locale codes (e.g. -US, -lk) @@ -210,6 +213,8 @@ def _check_url_localization_patterns(self, soup: BeautifulSoup) -> tuple[bool, l return (len(missing) == 0, missing) def _check_unicode_content(self, soup: BeautifulSoup) -> tuple[bool, list[str]]: + """Count Unicode characters for English, Sinhala, and Tamil scripts in the page text. + Returns (all_found, missing_language_codes).""" NOISE_TAGS = frozenset(['script', 'style', 'head', 'meta', 'noscript', 'nav', 'footer', 'header']) # Single O(N) pass: collect only leaf text nodes whose immediate parent @@ -244,6 +249,8 @@ def _check_unicode_content(self, soup: BeautifulSoup) -> tuple[bool, list[str]]: return (len(missing) == 0, missing) def _verify_language_buttons_via_browser(self, url: str, langs_to_check: list[str]) -> list[str]: + """Click visible language-switcher buttons on the page via Playwright and verify the resulting content. + Returns the list of language codes that remain unverified after clicking.""" # Returns the list of languages STILL missing after verification missing = list(langs_to_check) found_langs: set[str] = set() @@ -324,6 +331,8 @@ def _verify_language_buttons_via_browser(self, url: str, langs_to_check: list[st return missing def _check_browser_storage_keys(self, url) -> tuple[bool, list[str], str]: + """Use Playwright to inspect localStorage for a language key and inject all three locales to verify rendering. + Returns (all_found, missing_language_codes, detection_method_label).""" try: with _playwright_lock: with sync_playwright() as p: @@ -428,6 +437,8 @@ def _check_browser_storage_keys(self, url) -> tuple[bool, list[str], str]: return (False, list(LANGUAGE_KEY), "ERROR") def _analyze_soup(self, url: str, soup: BeautifulSoup) -> tuple[set[str], str | None, list[str]]: + """Run all static HTML heuristics (hreflang, URL patterns, Unicode) on a parsed page. + Returns (found_language_set, google_widget_type_or_None, passed_criteria_list).""" passed_attribute, missing_attribute = self._check_html_attribute(url, soup) google_widget = self._check_google_translate(soup) passed_url_loc, missing_url_loc = self._check_url_localization_patterns(soup) @@ -470,6 +481,8 @@ def _analyze_soup(self, url: str, soup: BeautifulSoup) -> tuple[set[str], str | return found_langs, google_widget, passed_criteria def _get_internal_links(self, base_url: str, soup: BeautifulSoup) -> list[str]: + """Extract same-domain internal links from the page, prioritising language-specific paths. + Returns a list with language links first, followed by other internal links.""" lang_links = set() other_links = set() try: @@ -509,6 +522,8 @@ def _get_internal_links(self, base_url: str, soup: BeautifulSoup) -> list[str]: return list(lang_links) + list(other_links) def run(self, url: str) -> CheckResult: + """Execute the full trilingual detection pipeline for *url*, from static HTML checks through Playwright fallbacks. + Returns a TrilingualCheckResult with status, optional error message, and detection details.""" try: parsed = parse_url(url) except ValueError as exc: From 42d5285d24a8d56d1faa5fd375ce8934cf5d1d72 Mon Sep 17 00:00:00 2001 From: ChanukaUOJ Date: Fri, 26 Jun 2026 09:08:45 +0530 Subject: [PATCH 20/20] final output csv updated --- data/mins_depts_test_scored.csv | 100 ++++++++++++++++---------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/data/mins_depts_test_scored.csv b/data/mins_depts_test_scored.csv index e959f7f..c5a4c03 100644 --- a/data/mins_depts_test_scored.csv +++ b/data/mins_depts_test_scored.csv @@ -1,50 +1,50 @@ -Type,Institution Name,URL,ssl_status,ssl_error -Ministry,Ministry of Defence,defence.lk,valid, -Ministry,"Ministry of Finance, Planning and Economic Development",treasury.gov.lk,valid, -Ministry,Ministry of Digital Economy,midec.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Ministry,"Ministry of Foreign Affairs, Foreign Employment and Tourism",mfa.gov.lk,valid, -Ministry,"Ministry of Education, Higher Education and Vocational Education",moe.gov.lk,valid, -Ministry,"Ministry of Agriculture, Livestock, Land and Irrigation",agrimin.gov.lk,valid, -Ministry,"Ministry of Transport, Highways and Urban Development",transport.gov.lk,valid, -Ministry,"Ministry of Public Administration, Provincial Councils and Local Government",pubad.gov.lk,valid, -Ministry,Ministry of Health and Mass Media,health.gov.lk,valid, -Ministry,Ministry of Justice and National Integration,moj.gov.lk,valid, -Ministry,Ministry of Public Security and Parliamentary Affairs,pubsec.gov.lk,valid, -Ministry,Ministry of Industry and Entrepreneurship Development,industry.gov.lk,valid, -Ministry,"Ministry of Trade, Commerce, Food Security and Cooperative Development",ment.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Ministry,Ministry of Environment,env.gov.lk,valid, -Ministry,Ministry of Energy,energy.gov.lk,valid, -Ministry,"Ministry of Housing, Construction and Water Supply",housingmin.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Ministry,Ministry of Labor,labourmin.gov.lk,valid, -Ministry,Ministry of Plantation and Community Infrastructure,plantationindustries.gov.lk,invalid,"Hostname mismatch, certificate is not valid for 'plantationindustries.gov.lk'." -Ministry,Ministry of Ports and Civil Aviation,portaviation.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Ministry,"Ministry of Fisheries, Aquatic and Ocean Resources",fisheries.gov.lk,valid, -Ministry,Ministry of Women and Child Affairs,childwomen.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Ministry,"Ministry of Rural Development, Social Security and Community Empowerment",socialservices.gov.lk,valid, -Ministry,"Ministry of Buddhasasana, Religious and Cultural Affairs",mbra.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Ministry,Ministry of Youth Affairs and Sports,sportsmin.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Ministry,Ministry of Science and Technology,most.gov.lk,valid, -Department,Department of Immigration and Emigration,immigration.gov.lk,valid, -Department,Department of Motor Traffic,dmt.gov.lk,valid, -Department,Department of Registration of Persons,drp.gov.lk,valid, -Department,Department of Registrar General,rgd.gov.lk,valid, -Department,Department of Inland Revenue,ird.gov.lk,valid, -Department,Sri Lanka Customs,customs.gov.lk,valid, -Department,Department of Pensions,pensions.gov.lk,valid, -Department,Department of Census and Statistics,statistics.gov.lk,unreachable,[Errno 54] Connection reset by peer -Department,Department of Government Information,dgi.gov.lk,valid, -Department,Department of Government Printing,documents.gov.lk,valid, -Department,Department of Excise,excise.gov.lk,valid, -Department,Department of National Budget,nationalbudget.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Department,Department of Meteorology,meteo.gov.lk,valid, -Department,Department of Wildlife Conservation,dwc.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Department,Department of Forest Conservation,forestdept.gov.lk,expired,unable to get local issuer certificate -Department,Department of National Zoological Gardens,colombozoo.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known" -Department,Department of Examinations,doenets.lk,valid, -Department,Department of Archaeology,archaeology.gov.lk,valid, -Department,Department of Samurdhi,samurdhi.gov.lk,valid, -Department,LSF,opensource.lk,invalid,"Hostname mismatch, certificate is not valid for 'opensource.lk'." -Expired,Expired SSL,expired.badssl.com,unreachable,[Errno 54] Connection reset by peer -Self-Signed,Self-Signed SSL,self-signed.badssl.com,invalid,self-signed certificate -Wrong Host,Wrong Host SSL,wrong.host.badssl.com,invalid,"Hostname mismatch, certificate is not valid for 'wrong.host.badssl.com'." -Never SSL,Never SSL,neverssl.com,valid, +Type,Institution Name,URL,ssl_status,ssl_error,trilingual_status,trilingual_error,trilingual_details,trilingual_deeplink +Ministry,Ministry of Defence,defence.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Ministry,"Ministry of Finance, Planning and Economic Development",treasury.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Ministry,Ministry of Digital Economy,midec.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Ministry,"Ministry of Foreign Affairs, Foreign Employment and Tourism",mfa.gov.lk,valid,,TRILINGUAL,,DEEPLINK_CRAWL: UNICODE_CONTENT,https://www.mfa.gov.lk/en +Ministry,"Ministry of Education, Higher Education and Vocational Education",moe.gov.lk,valid,,TRILINGUAL,,HTML_ATTRIBUTE, +Ministry,"Ministry of Agriculture, Livestock, Land and Irrigation",agrimin.gov.lk,valid,,NON_TRILINGUAL,"missing languages: si, ta",, +Ministry,"Ministry of Transport, Highways and Urban Development",transport.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Ministry,"Ministry of Public Administration, Provincial Councils and Local Government",pubad.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Ministry,Ministry of Health and Mass Media,health.gov.lk,valid,,TRILINGUAL,,"DEEPLINK_CRAWL: HTML_ATTRIBUTE, URL_LOCALIZATION",https://www.health.gov.lk/home/ +Ministry,Ministry of Justice and National Integration,moj.gov.lk,valid,,TRILINGUAL,,"HTML_ATTRIBUTE, URL_LOCALIZATION", +Ministry,Ministry of Public Security and Parliamentary Affairs,pubsec.gov.lk,valid,,TRILINGUAL,,"HTML_ATTRIBUTE, URL_LOCALIZATION", +Ministry,Ministry of Industry and Entrepreneurship Development,industry.gov.lk,valid,,TRILINGUAL,,"URL_LOCALIZATION, UNICODE_CONTENT", +Ministry,"Ministry of Trade, Commerce, Food Security and Cooperative Development",ment.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Ministry,Ministry of Environment,env.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Ministry,Ministry of Energy,energy.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Ministry,"Ministry of Housing, Construction and Water Supply",housingmin.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Ministry,Ministry of Labor,labourmin.gov.lk,valid,,NON_TRILINGUAL,"missing languages: si, ta",, +Ministry,Ministry of Plantation and Community Infrastructure,plantationindustries.gov.lk,invalid,"Hostname mismatch, certificate is not valid for 'plantationindustries.gov.lk'.",TRILINGUAL,,GOOGLE_TRANSLATE_API, +Ministry,Ministry of Ports and Civil Aviation,portaviation.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Ministry,"Ministry of Fisheries, Aquatic and Ocean Resources",fisheries.gov.lk,valid,,TRILINGUAL,,HTML_ATTRIBUTE, +Ministry,Ministry of Women and Child Affairs,childwomen.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Ministry,"Ministry of Rural Development, Social Security and Community Empowerment",socialservices.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Ministry,"Ministry of Buddhasasana, Religious and Cultural Affairs",mbra.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Ministry,Ministry of Youth Affairs and Sports,sportsmin.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Ministry,Ministry of Science and Technology,most.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Department,Department of Immigration and Emigration,immigration.gov.lk,valid,,TRILINGUAL,,UNICODE_CONTENT, +Department,Department of Motor Traffic,dmt.gov.lk,valid,,TRILINGUAL,,"HTML_ATTRIBUTE, URL_LOCALIZATION", +Department,Department of Registration of Persons,drp.gov.lk,valid,,TRILINGUAL,,VERIFIED_SWITCHER_CLICK, +Department,Department of Registrar General,rgd.gov.lk,valid,,TRILINGUAL,,"URL_LOCALIZATION, UNICODE_CONTENT", +Department,Department of Inland Revenue,ird.gov.lk,valid,,UNREACHABLE,Can't Access,, +Department,Sri Lanka Customs,customs.gov.lk,valid,,TRILINGUAL,,GOOGLE_TRANSLATE_API, +Department,Department of Pensions,pensions.gov.lk,valid,,TRILINGUAL,,"HTML_ATTRIBUTE, URL_LOCALIZATION", +Department,Department of Census and Statistics,statistics.gov.lk,unreachable,[Errno 54] Connection reset by peer,TRILINGUAL,,VERIFIED_SWITCHER_CLICK, +Department,Department of Government Information,dgi.gov.lk,valid,,TRILINGUAL,,VERIFIED_SWITCHER_CLICK, +Department,Department of Government Printing,documents.gov.lk,valid,,NON_TRILINGUAL,"missing languages: si, ta",, +Department,Department of Excise,excise.gov.lk,valid,,TRILINGUAL,,"HTML_ATTRIBUTE, URL_LOCALIZATION", +Department,Department of National Budget,nationalbudget.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Department,Department of Meteorology,meteo.gov.lk,valid,,TRILINGUAL,,UNICODE_CONTENT, +Department,Department of Wildlife Conservation,dwc.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",TRILINGUAL,,DEEPLINK_CRAWL: UNICODE_CONTENT,"https://www.dwc.gov.lk/?page_id=203, https://www.dwc.gov.lk/?page_id=257, https://www.dwc.gov.lk/?p=914, https://www.dwc.gov.lk/?paged=2" +Department,Department of Forest Conservation,forestdept.gov.lk,expired,unable to get local issuer certificate,TRILINGUAL,,"HTML_ATTRIBUTE, URL_LOCALIZATION", +Department,Department of National Zoological Gardens,colombozoo.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known",UNREACHABLE,Can't Access,, +Department,Department of Examinations,doenets.lk,valid,,TRILINGUAL,,BROWSER_STORAGE, +Department,Department of Archaeology,archaeology.gov.lk,valid,,TRILINGUAL,,GOOGLE_TRANSLATE_API, +Department,Department of Samurdhi,samurdhi.gov.lk,valid,,TRILINGUAL,,URL_LOCALIZATION, +Department,LSF,opensource.lk,valid,,NON_TRILINGUAL,"missing languages: si, ta",, +Expired,Expired SSL,expired.badssl.com,expired,certificate has expired,TIMEOUT,"Some checks timed out. Potentially missing: si, ta",, +Self-Signed,Self-Signed SSL,self-signed.badssl.com,invalid,self-signed certificate,UNREACHABLE,Can't Access,, +Wrong Host,Wrong Host SSL,wrong.host.badssl.com,unreachable,[Errno 54] Connection reset by peer,TIMEOUT,"Some checks timed out. Potentially missing: si, ta",, +Never SSL,Never SSL,neverssl.com,valid,,NON_TRILINGUAL,"missing languages: si, ta",,