diff --git a/README.md b/README.md index afed1b9..7c43a19 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,59 @@ docker-compose logs scraper-worker --- +## Scraper & Crawler Architecture + +The scraper worker runs two independent loops in parallel background threads: + +### CrawlerService — discovery loop + +Runs every `CRAWLER_INTERVAL_SECONDS` (default: 1s). + +1. BFS-crawls the source listing page to collect all live offer URLs. +2. Creates a `CrawlUrl` record (status `PENDING`) for any new URL found. +3. **Deletion detection** — any `CrawlUrl` no longer present in the live listing is bumped to `next_check_at = now`, so the scraper re-fetches it immediately and detects the 404. +4. **Manual offer cleanup** — offers created manually in the admin (no `CrawlUrl`) whose link matches the source domain but is absent from the live listing are HEAD-checked directly; those returning 404/410 are archived. + +### UrlScraperService — scrape loop + +Runs every `SCRAPER_INTERVAL_SECONDS` (default: 1s). + +1. Claims all `CrawlUrl` records where `next_check_at <= now`. +2. Fetches each URL (timeout: `SCRAPER_TIMEOUT_SECONDS`, default: 1s). +3. **4xx response** → offer is immediately archived (`PUBLISHED`/`DRAFT` → `ARCHIVED`). +4. **5xx response** → transient error; exponential backoff applied, retried later. +5. **200 with empty/generic page** → if a linked offer exists, it is archived; otherwise the URL is skipped. +6. **200 with valid content** → offer is created or updated. + +### Archiving behaviour + +Offers are **never hard-deleted** from the database. All removal triggers result in `status = ARCHIVED`. Archived offers are hidden from the public Vue frontend but remain visible in the admin dashboard. + +| Trigger | Result | +|---|---| +| URL returns 4xx (404, 410, etc.) | ARCHIVED immediately | +| URL returns 5xx | Backoff, retried later | +| URL returns 200 but page is empty/generic | ARCHIVED (if offer linked) | +| URL disappears from source listing | CrawlUrl bumped → re-fetched → 404 → ARCHIVED | +| Manually created offer URL gone from listing | HEAD-checked → 404/410 → ARCHIVED | + +### Scraper worker configuration + +| Variable | Default | Description | +|---|---|---| +| `CRAWLER_INTERVAL_SECONDS` | `1` | How often the crawler discovery loop runs | +| `SCRAPER_INTERVAL_SECONDS` | `1` | How often the scraper drain loop runs | +| `SCRAPER_TIMEOUT_SECONDS` | `1` | HTTP request timeout per URL | + +### LLM extraction + +Each scraped page is processed by one of two extraction methods: + +- **Ollama LLM (primary)** — structured JSON extraction with offer type classification. Configured per source in the admin panel. +- **Deterministic fallback** — `` as summary; TF-IDF cosine similarity for offer type classification. Used automatically when Ollama is not running or returns a connection error (no wait, no cooldown). + +--- + ## Useful Docker commands ```bash @@ -146,8 +199,11 @@ Each scraped offer is automatically classified into one of the offer types store | Variable | Default | Description | |---|---|---| -| `LLM_ENABLED` | `true` | Enable or disable the Ollama LLM | +| `LLM_ENABLED` | `true` | Enable or disable the Ollama LLM globally | | `SCRAPER_CLASSIFIER_THRESHOLD` | `0.15` | Minimum cosine similarity score for the TF-IDF fallback to accept a classification | +| `CRAWLER_INTERVAL_SECONDS` | `1` | Crawler discovery loop interval | +| `SCRAPER_INTERVAL_SECONDS` | `1` | Scraper drain loop interval | +| `SCRAPER_TIMEOUT_SECONDS` | `1` | HTTP request timeout per scraped URL | ### Adding or updating offer types diff --git a/backend/.env b/backend/.env index b4bff02..e107eeb 100644 --- a/backend/.env +++ b/backend/.env @@ -13,8 +13,9 @@ POSTGRES_HOST=localhost POSTGRES_PORT=5432 POSTGRES_SCHEMA=content -SCRAPER_TIMEOUT_SECONDS=30 -SCRAPER_INTERVAL_MINUTES=360 +SCRAPER_TIMEOUT_SECONDS=1 +CRAWLER_INTERVAL_SECONDS=1 +SCRAPER_INTERVAL_SECONDS=1 SCRAPER_RUN_ON_START=true SCRAPER_LLM_FALLBACK_THRESHOLD=0.60 SCRAPER_USER_AGENT=SUNRISE-OSS-Scraper/1.0 diff --git a/backend/.env.example b/backend/.env.example index b4bff02..e107eeb 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -13,8 +13,9 @@ POSTGRES_HOST=localhost POSTGRES_PORT=5432 POSTGRES_SCHEMA=content -SCRAPER_TIMEOUT_SECONDS=30 -SCRAPER_INTERVAL_MINUTES=360 +SCRAPER_TIMEOUT_SECONDS=1 +CRAWLER_INTERVAL_SECONDS=1 +SCRAPER_INTERVAL_SECONDS=1 SCRAPER_RUN_ON_START=true SCRAPER_LLM_FALLBACK_THRESHOLD=0.60 SCRAPER_USER_AGENT=SUNRISE-OSS-Scraper/1.0 diff --git a/backend/content/management/commands/run_scraper_worker.py b/backend/content/management/commands/run_scraper_worker.py index 5195f76..efa6cf0 100644 --- a/backend/content/management/commands/run_scraper_worker.py +++ b/backend/content/management/commands/run_scraper_worker.py @@ -1,6 +1,9 @@ +import logging import os +import threading +import time -from apscheduler.schedulers.blocking import BlockingScheduler +from apscheduler.schedulers.background import BackgroundScheduler from django.core.management.base import BaseCommand from django.utils import timezone @@ -8,6 +11,18 @@ from content.scrapers.link_checker import check_offer_links from content.scrapers.translation_service import run_offer_translation_batch +LOGGER = logging.getLogger(__name__) + + +def _loop(fn: callable, interval: float, stop: threading.Event) -> None: + """Run fn, sleep interval seconds, repeat — guarantees no overlap.""" + while not stop.is_set(): + try: + fn() + except Exception: + LOGGER.exception("Worker loop error in %s", fn.__name__) + stop.wait(interval) + def _archive_expired_offers(): from django.utils import timezone as tz @@ -36,8 +51,8 @@ def add_arguments(self, parser): ) def handle(self, *args, **options): - crawler_interval = int(os.getenv("CRAWLER_INTERVAL_MINUTES", "360")) - scraper_interval = int(os.getenv("SCRAPER_INTERVAL_MINUTES", "5")) + crawler_interval = int(os.getenv("CRAWLER_INTERVAL_SECONDS", "1")) + scraper_interval = int(os.getenv("SCRAPER_INTERVAL_SECONDS", "1")) translation_interval = int(os.getenv("TRANSLATION_INTERVAL_MINUTES", "10")) translation_batch = int(os.getenv("TRANSLATION_BATCH_SIZE", "20")) run_on_start = os.getenv("SCRAPER_RUN_ON_START", "true").lower() == "true" @@ -51,26 +66,27 @@ def handle(self, *args, **options): self.stdout.write(f"Scraper: {scraper_summary}") return - scheduler = BlockingScheduler(timezone=timezone.get_current_timezone_name()) + stop = threading.Event() - scheduler.add_job( - run_crawler, - "interval", - id="crawl-discover-urls", - minutes=crawler_interval, - max_instances=1, - coalesce=True, - misfire_grace_time=300, - ) - scheduler.add_job( - run_url_scraper_batch, - "interval", - id="scrape-url-queue", - minutes=scraper_interval, - max_instances=1, - coalesce=True, - misfire_grace_time=60, - ) + if run_on_start: + self.stdout.write("Startup crawler run...") + self.stdout.write(f"Crawler: {run_crawler()}") + self.stdout.write("Startup scraper batch...") + self.stdout.write(f"Scraper: {run_url_scraper_batch()}") + self.stdout.write("Startup offer translation...") + self.stdout.write(f"Translation: {run_offer_translation_batch(limit=translation_batch)}") + + # Crawler and scraper run in threads: each waits N seconds AFTER the + # previous run completes, so no overlap and no APScheduler skip warnings. + threading.Thread( + target=_loop, args=(run_crawler, crawler_interval, stop), daemon=True, name="crawler" + ).start() + threading.Thread( + target=_loop, args=(run_url_scraper_batch, scraper_interval, stop), daemon=True, name="scraper" + ).start() + + # Cron jobs (infrequent) stay on APScheduler. + scheduler = BackgroundScheduler(timezone=timezone.get_current_timezone_name()) scheduler.add_job( run_offer_translation_batch, "interval", @@ -99,25 +115,17 @@ def handle(self, *args, **options): max_instances=1, coalesce=True, ) - - if run_on_start: - self.stdout.write("Startup crawler run...") - crawler_summary = run_crawler() - self.stdout.write(f"Crawler: {crawler_summary}") - self.stdout.write("Startup scraper batch...") - scraper_summary = run_url_scraper_batch() - self.stdout.write(f"Scraper: {scraper_summary}") - self.stdout.write("Startup offer translation...") - translation_summary = run_offer_translation_batch(limit=translation_batch) - self.stdout.write(f"Translation: {translation_summary}") + scheduler.start() self.stdout.write( - f"Worker running — crawler every {crawler_interval} min, " - f"scraper every {scraper_interval} min, " + f"Worker running — crawler every {crawler_interval}s after completion, " + f"scraper every {scraper_interval}s after completion, " f"translation every {translation_interval} min. Press Ctrl+C to stop." ) try: - scheduler.start() + stop.wait() except (KeyboardInterrupt, SystemExit): + stop.set() + scheduler.shutdown(wait=False) self.stdout.write("Worker stopped.") diff --git a/backend/content/scrapers/ollama_client.py b/backend/content/scrapers/ollama_client.py index 3df9654..75d9ef8 100644 --- a/backend/content/scrapers/ollama_client.py +++ b/backend/content/scrapers/ollama_client.py @@ -17,6 +17,11 @@ # at the start of run N+1 before the cooldown has expired. _SHARED_MODEL_COOLDOWN: dict[str, float] = {} +# Models whose cooldown was triggered by a connection error (server down), +# not a 429 rate limit. These skip the cooldown wait — no point waiting +# for a server that isn't running. +_CONNECTION_ERROR_MODELS: set[str] = set() + # Buffer for cooldown events emitted during _wait_for_available_model. # Flushed into run.log by ScrapeService after each LLM call. _COOLDOWN_LOG_BUFFER: list[dict] = [] @@ -58,6 +63,12 @@ def _wait_for_available_model(self) -> list[str]: return models if not _SHARED_MODEL_COOLDOWN: return [] + # If every cooled-down model failed due to a connection error (server + # unreachable), skip the wait and fall through to deterministic immediately. + cooled_models = set(_SHARED_MODEL_COOLDOWN.keys()) + if cooled_models and cooled_models.issubset(_CONNECTION_ERROR_MODELS): + LOGGER.debug("Ollama unreachable — skipping LLM, using deterministic fallback") + return [] soonest = min(_SHARED_MODEL_COOLDOWN.values()) wait = max(0.0, soonest - time.time()) if wait > self.cooldown_max_wait_seconds: @@ -109,7 +120,10 @@ def extract_fallback( status_code = exc.status_code if isinstance(exc, ollama.ResponseError) else None _SHARED_MODEL_COOLDOWN[model] = time.time() + self.model_cooldown_seconds if status_code != 429: + _CONNECTION_ERROR_MODELS.add(model) LOGGER.warning("Ollama fallback unavailable for %s using %s: %s", source.key, model, exc) + else: + _CONNECTION_ERROR_MODELS.discard(model) continue text = response.response @@ -122,6 +136,7 @@ def extract_fallback( confidence = parsed.get("confidence", deterministic_payload.confidence) self.last_switch_count = index + _CONNECTION_ERROR_MODELS.discard(model) LOGGER.info( "LLM fallback success — source=%s model=%s conf=%.2f title=%r", source.key, model, float(confidence), str(title)[:60], @@ -179,7 +194,10 @@ def assess_and_extract( status_code = exc.status_code if isinstance(exc, ollama.ResponseError) else None _SHARED_MODEL_COOLDOWN[model] = time.time() + self.model_cooldown_seconds if status_code != 429: + _CONNECTION_ERROR_MODELS.add(model) LOGGER.warning("Ollama relevance check unavailable for %s using %s: %s", source.key, model, exc) + else: + _CONNECTION_ERROR_MODELS.discard(model) continue text = response.response @@ -200,6 +218,7 @@ def assess_and_extract( ) self.last_switch_count = index + _CONNECTION_ERROR_MODELS.discard(model) LOGGER.info( "LLM assess success — source=%s model=%s is_offer=%s offer_type=%s conf=%.2f reason=%r", source.key, model, is_relevant, offer_type or "(will classify)", confidence, reason[:80] if reason else "", diff --git a/backend/content/scrapers/queue_service.py b/backend/content/scrapers/queue_service.py index 85d442f..27997fd 100644 --- a/backend/content/scrapers/queue_service.py +++ b/backend/content/scrapers/queue_service.py @@ -3,6 +3,7 @@ from dataclasses import replace from datetime import timedelta from types import SimpleNamespace +from urllib.parse import urlparse import requests from django.db import transaction @@ -30,7 +31,7 @@ class CrawlerService(ScrapeService): def run(self) -> dict: sources = get_sources(self.source_keys) - stats = {"sources": 0, "discovered": 0, "new": 0, "already_known": 0, "errors": 0} + stats = {"sources": 0, "discovered": 0, "new": 0, "already_known": 0, "bumped": 0, "manual_archived": 0, "errors": 0} for source in sources: stats["sources"] += 1 @@ -40,9 +41,12 @@ def run(self) -> dict: stats["discovered"] += result["discovered"] stats["new"] += result["new"] stats["already_known"] += result["already_known"] + stats["bumped"] += result["bumped"] + stats["manual_archived"] += result["manual_archived"] LOGGER.info( - "[%s] Crawler done — discovered=%d new=%d known=%d", + "[%s] Crawler done — discovered=%d new=%d known=%d bumped=%d manual_archived=%d", source.key, result["discovered"], result["new"], result["already_known"], + result["bumped"], result["manual_archived"], ) except Exception: stats["errors"] += 1 @@ -52,6 +56,7 @@ def run(self) -> dict: def _crawl_source(self, source: SourceDefinition) -> dict: urls, _ = self._discover_urls_bfs(source) + live_url_set = set(urls) new_count = 0 known_count = 0 @@ -70,7 +75,58 @@ def _crawl_source(self, source: SourceDefinition) -> dict: elif crawl_url.status != CrawlUrl.UrlStatus.ARCHIVED: known_count += 1 - return {"discovered": len(urls), "new": new_count, "already_known": known_count} + # Bump CrawlUrls no longer in listing → immediate recheck → scraper fetches → 404 → archive. + # Network errors raise exceptions (caught in run()) so an empty live_url_set here means + # the listing genuinely has no offers — proceed with deletion detection. + bumped = CrawlUrl.objects.filter( + source_key=source.key, + status__in=[CrawlUrl.UrlStatus.DONE, CrawlUrl.UrlStatus.ERROR], + ).exclude(url__in=live_url_set).update(next_check_at=timezone.now()) + + # HEAD-check manually created offers (no CrawlUrl) from this org not in live set + manual_archived = self._check_manual_offers(source, live_url_set) + + return { + "discovered": len(urls), + "new": new_count, + "already_known": known_count, + "bumped": bumped, + "manual_archived": manual_archived, + } + + def _check_manual_offers(self, source: SourceDefinition, live_url_set: set[str]) -> int: + """HEAD-check published/draft offers from this org that have no CrawlUrl entry, + are not in the current live URL set, and whose link matches the source domain. + Archives published offers and deletes drafts if their link returns 404/410.""" + if not source.organization_id: + return 0 + + source_netloc = urlparse(source.url).netloc + archived_count = 0 + + manual_offers = ( + Offer.objects.filter( + organization_id=source.organization_id, + status__in=[Offer.OfferStatus.PUBLISHED, Offer.OfferStatus.DRAFT], + ) + .exclude(link__in=live_url_set) + .filter(link__icontains=source_netloc) + .filter(crawl_urls__isnull=True) + ) + + for offer in manual_offers: + status_code = self._fetch_status_code(offer.link) + if status_code in {404, 410}: + archived_count += 1 + if offer.status == Offer.OfferStatus.PUBLISHED: + offer.status = Offer.OfferStatus.ARCHIVED + offer.save(update_fields=["status", "updated_at"]) + LOGGER.info("[%s] ARCHIVED manual offer (HTTP %s) — %s", source.key, status_code, offer.link) + elif offer.status == Offer.OfferStatus.DRAFT: + offer.delete() + LOGGER.info("[%s] DELETED manual draft offer (HTTP %s) — %s", source.key, status_code, offer.link) + + return archived_count class UrlScraperService(ScrapeService): @@ -120,16 +176,20 @@ def run_batch(self) -> dict: self._scrape_one(crawl_url, source, source_type, ingestion_user, stats, logs, matched_offer_ids) - run.status = ScrapingRun.RunStatus.SUCCESS - run.offers_processed = stats["processed"] - run.offers_created = stats["created"] - run.offers_updated = stats["updated"] - run.offers_unchanged = stats["unchanged"] - run.errors_count = stats["errors"] - run.urls_neglected = stats["neglected"] - run.log = logs - run.completed_at = timezone.now() - run.save() + # Skip saving runs where nothing noteworthy happened (all neglected, no log entries). + if not logs: + run.delete() + else: + run.status = ScrapingRun.RunStatus.SUCCESS + run.offers_processed = stats["processed"] + run.offers_created = stats["created"] + run.offers_updated = stats["updated"] + run.offers_unchanged = stats["unchanged"] + run.errors_count = stats["errors"] + run.urls_neglected = stats["neglected"] + run.log = logs + run.completed_at = timezone.now() + run.save() LOGGER.info( "URL scraper batch done — processed=%d created=%d updated=%d archived=%d errors=%d neglected=%d", @@ -182,9 +242,14 @@ def _scrape_one( except requests.exceptions.HTTPError as exc: http_status = exc.response.status_code if exc.response is not None else None self._handle_http_error(crawl_url, http_status, stats) - logs.append({"ts": _ts(), "event": "url_failed", "level": "warn", - "source_key": source.key, "url": crawl_url.url, - "http_status": http_status, "reason": "http_error"}) + if http_status is not None and 400 <= http_status < 500: + logs.append({"ts": _ts(), "event": "url_archived", "level": "info", + "source_key": source.key, "url": crawl_url.url, + "http_status": http_status, "reason": "client_error"}) + else: + logs.append({"ts": _ts(), "event": "url_failed", "level": "warn", + "source_key": source.key, "url": crawl_url.url, + "http_status": http_status, "reason": "http_error"}) return except requests.RequestException as exc: self._handle_transient_error(crawl_url, str(exc), None, stats) @@ -196,11 +261,20 @@ def _scrape_one( extracted = extract_deterministic(html, page_source) if is_generic_page(extracted.title): - stats["neglected"] += 1 - LOGGER.info("[%s] NEGLECT %s — generic_page_title", source.key, crawl_url.url) - logs.append({"ts": _ts(), "event": "url_neglected", "level": "info", - "source_key": source.key, "url": crawl_url.url, "reason": "generic_page_title"}) - self._mark_done(crawl_url) + if crawl_url.offer_id is not None: + self._archive_offer(crawl_url) + crawl_url.status = CrawlUrl.UrlStatus.ARCHIVED + crawl_url.save() + stats["archived"] += 1 + LOGGER.info("[%s] ARCHIVE %s — generic_page (offer existed)", source.key, crawl_url.url) + logs.append({"ts": _ts(), "event": "url_archived", "level": "info", + "source_key": source.key, "url": crawl_url.url, "reason": "generic_page"}) + else: + stats["neglected"] += 1 + LOGGER.info("[%s] NEGLECT %s — generic_page_title", source.key, crawl_url.url) + logs.append({"ts": _ts(), "event": "url_neglected", "level": "info", + "source_key": source.key, "url": crawl_url.url, "reason": "generic_page_title"}) + self._mark_done(crawl_url) return if self.use_llm_fallback and source.llm_fallback_enabled: @@ -353,8 +427,8 @@ def _mark_done(self, crawl_url: CrawlUrl) -> None: crawl_url.save() def _handle_http_error(self, crawl_url: CrawlUrl, http_status: int | None, stats: dict) -> None: - if http_status in {404, 410}: - LOGGER.info("URL permanently gone (HTTP %s) — %s", http_status, crawl_url.url) + if http_status is not None and 400 <= http_status < 500: + LOGGER.info("URL client error (HTTP %s) — %s", http_status, crawl_url.url) self._archive_offer(crawl_url) crawl_url.status = CrawlUrl.UrlStatus.ARCHIVED crawl_url.last_http_status = http_status @@ -388,13 +462,11 @@ def _archive_offer(self, crawl_url: CrawlUrl) -> None: return try: offer = crawl_url.offer - if offer.status == Offer.OfferStatus.PUBLISHED: - offer.status = Offer.OfferStatus.ARCHIVED - offer.save(update_fields=["status", "updated_at"]) - LOGGER.info("Archived offer — %s", crawl_url.url) - elif offer.status == Offer.OfferStatus.DRAFT: - offer.delete() - LOGGER.info("Deleted draft offer — %s", crawl_url.url) + if offer.status not in {Offer.OfferStatus.PUBLISHED, Offer.OfferStatus.DRAFT}: + return + offer.status = Offer.OfferStatus.ARCHIVED + offer.save(update_fields=["status", "updated_at"]) + LOGGER.info("Archived offer (was %s) — %s", offer.status, crawl_url.url) except Offer.DoesNotExist: pass diff --git a/backend/content/scrapers/service.py b/backend/content/scrapers/service.py index 524a5ed..9b4d703 100644 --- a/backend/content/scrapers/service.py +++ b/backend/content/scrapers/service.py @@ -540,7 +540,7 @@ def _upsert_offer( resolved_offer_type_name = extracted.offer_type if not resolved_offer_type_name: - classifier_text = f"{extracted.title} {extracted.summary}".strip() + classifier_text = f"{extracted.title} {extracted.summary} {extracted.details or ''}".strip() resolved_offer_type_name, confidence = _classifier.classify(classifier_text) if resolved_offer_type_name: LOGGER.info( diff --git a/backend/content/templates/content/mock_website_detail.html b/backend/content/templates/content/mock_website_detail.html index f8f2ee6..aae7c25 100644 --- a/backend/content/templates/content/mock_website_detail.html +++ b/backend/content/templates/content/mock_website_detail.html @@ -13,7 +13,8 @@

{{ opportunity.title }}

{{ opportunity.description }}

diff --git a/backend/ui/src/app/pages/scrapper-admin-page.component.css b/backend/ui/src/app/pages/scrapper-admin-page.component.css index 09ad36a..f376fa6 100644 --- a/backend/ui/src/app/pages/scrapper-admin-page.component.css +++ b/backend/ui/src/app/pages/scrapper-admin-page.component.css @@ -249,6 +249,8 @@ header h1 { .failed-row:hover td { background: #fff0f0; } .skipped-row td { background: #fffcf0; } .skipped-row:hover td { background: #fff8e0; } +.archived-row td { background: #f4f4f4; } +.archived-row:hover td { background: #ebebeb; } .url-table-wrap { overflow-x: auto; border-radius: 10px; border: 1px solid var(--border, #e4e4e0); } @@ -462,9 +464,10 @@ header h1 { .warning-text { color: #856404; } .chip-inline { padding: 0.05rem 0.4rem; border-radius: 6px; font-size: 0.7rem; font-weight: 600; } -.chip-ok { background: #e7f4ea; color: #1d7347; } -.chip-warn { background: #fff3cd; color: #856404; } -.chip-fail { background: #ffe4e4; color: #9e2a2a; } +.chip-ok { background: #e7f4ea; color: #1d7347; } +.chip-warn { background: #fff3cd; color: #856404; } +.chip-fail { background: #ffe4e4; color: #9e2a2a; } +.chip-archived { background: #e9ecef; color: #495057; } .top-errors { background: #fff; diff --git a/backend/ui/src/app/pages/scrapper-admin-page.component.html b/backend/ui/src/app/pages/scrapper-admin-page.component.html index 2e23d67..6c13087 100644 --- a/backend/ui/src/app/pages/scrapper-admin-page.component.html +++ b/backend/ui/src/app/pages/scrapper-admin-page.component.html @@ -235,7 +235,7 @@

{{ run.source_key }}

- + {{ r.url }} @@ -244,10 +244,11 @@

{{ run.source_key }}

{{ r.confidence != null ? (r.confidence | number:'1.2-2') : '—' }} - {{ r.failed ? ('scrapper.statusError' | translate) : r.neglected ? ('scrapper.skipped' | translate) : (r.action ?? ('scrapper.ok' | translate)) }} + {{ r.failed ? ('scrapper.statusError' | translate) : r.neglected ? ('scrapper.skipped' | translate) : r.archived ? ('scrapper.archived' | translate) : (r.action ?? ('scrapper.ok' | translate)) }} {{ r.message ?? r.reason ?? '' }} diff --git a/backend/ui/src/app/shared/run-log.parser.ts b/backend/ui/src/app/shared/run-log.parser.ts index 280c104..4f650cd 100644 --- a/backend/ui/src/app/shared/run-log.parser.ts +++ b/backend/ui/src/app/shared/run-log.parser.ts @@ -24,6 +24,7 @@ export type UrlResult = { duration_ms?: number; failed: boolean; neglected: boolean; + archived: boolean; message?: string; reason?: string; action?: string; @@ -79,6 +80,8 @@ export function toUrlResults(entries: RunLogEntry[]): UrlResult[] { entry.event === 'url_neglected' || (entry.event === 'url_failed' && entry.level === 'info'); + const isArchived = entry.event === 'url_archived'; + if (!existing) { urlMap.set(entry.url, { url: entry.url, @@ -88,6 +91,7 @@ export function toUrlResults(entries: RunLogEntry[]): UrlResult[] { duration_ms: entry.duration_ms, failed: entry.event === 'url_failed' && !isNeglect, neglected: isNeglect, + archived: isArchived, message: entry.message, reason: entry.reason, action: entry.action, @@ -101,6 +105,7 @@ export function toUrlResults(entries: RunLogEntry[]): UrlResult[] { duration_ms: entry.duration_ms ?? existing.duration_ms, failed: false, neglected: false, + archived: false, action: entry.action ?? existing.action, ts: entry.ts ?? existing.ts, }); @@ -109,6 +114,17 @@ export function toUrlResults(entries: RunLogEntry[]): UrlResult[] { ...existing, neglected: true, failed: false, + archived: false, + reason: entry.reason ?? existing.reason, + ts: entry.ts ?? existing.ts, + }); + } else if (isArchived) { + urlMap.set(entry.url, { + ...existing, + archived: true, + failed: false, + neglected: false, + http_status: entry.http_status ?? existing.http_status, reason: entry.reason ?? existing.reason, ts: entry.ts ?? existing.ts, }); @@ -117,6 +133,7 @@ export function toUrlResults(entries: RunLogEntry[]): UrlResult[] { ...existing, failed: true, neglected: false, + archived: false, http_status: entry.http_status ?? existing.http_status, message: entry.message ?? existing.message, reason: entry.reason ?? existing.reason, diff --git a/docker-compose.yml b/docker-compose.yml index fad6d7e..3c440f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,9 +98,9 @@ services: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 POSTGRES_SCHEMA: ${POSTGRES_SCHEMA:-content} - SCRAPER_TIMEOUT_SECONDS: ${SCRAPER_TIMEOUT_SECONDS:-30} - CRAWLER_INTERVAL_MINUTES: ${CRAWLER_INTERVAL_MINUTES:-5} - SCRAPER_INTERVAL_MINUTES: ${SCRAPER_INTERVAL_MINUTES:-5} + SCRAPER_TIMEOUT_SECONDS: ${SCRAPER_TIMEOUT_SECONDS:-1} + CRAWLER_INTERVAL_SECONDS: ${CRAWLER_INTERVAL_SECONDS:-1} + SCRAPER_INTERVAL_SECONDS: ${SCRAPER_INTERVAL_SECONDS:-1} SCRAPER_BATCH_SIZE: ${SCRAPER_BATCH_SIZE:-10} SCRAPER_REVISIT_DAYS: ${SCRAPER_REVISIT_DAYS:-7} SCRAPER_MAX_CONSECUTIVE_ERRORS: ${SCRAPER_MAX_CONSECUTIVE_ERRORS:-3} diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 65c9d0b..212020d 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -14,7 +14,7 @@ }, "devDependencies": { "@playwright/test": "^1.49.1", - "@vitejs/plugin-vue": "^6.0.5", + "@vitejs/plugin-vue": "^6.0.7", "vite": "^8.0.4" } }, @@ -293,9 +293,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -313,9 +310,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -333,9 +327,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -353,9 +344,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -373,9 +361,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -393,9 +378,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -476,7 +458,9 @@ } }, "node_modules/@rolldown/pluginutils": { - "version": "1.0.0-rc.2", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", "dev": true, "license": "MIT" }, @@ -492,11 +476,13 @@ } }, "node_modules/@vitejs/plugin-vue": { - "version": "6.0.5", + "version": "6.0.7", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-6.0.7.tgz", + "integrity": "sha512-km+p+XdSz9Sxm5rqUbqcSfZYaAniKxWBj1KURl+Jr7UaPvvX7BmaWMdP69I5rrFDeQGyxAG7NXdc57vz+snhWg==", "dev": true, "license": "MIT", "dependencies": { - "@rolldown/pluginutils": "1.0.0-rc.2" + "@rolldown/pluginutils": "^1.0.1" }, "engines": { "node": "^20.19.0 || >=22.12.0" @@ -1057,13 +1043,6 @@ "@rolldown/binding-win32-x64-msvc": "1.1.2" } }, - "node_modules/rolldown/node_modules/@rolldown/pluginutils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", - "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", - "dev": true, - "license": "MIT" - }, "node_modules/source-map-js": { "version": "1.2.1", "license": "BSD-3-Clause", @@ -1451,7 +1430,9 @@ "optional": true }, "@rolldown/pluginutils": { - "version": "1.0.0-rc.2", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", "dev": true }, "@tybys/wasm-util": { @@ -1465,10 +1446,12 @@ } }, "@vitejs/plugin-vue": { - "version": "6.0.5", + "version": "6.0.7", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-6.0.7.tgz", + "integrity": "sha512-km+p+XdSz9Sxm5rqUbqcSfZYaAniKxWBj1KURl+Jr7UaPvvX7BmaWMdP69I5rrFDeQGyxAG7NXdc57vz+snhWg==", "dev": true, "requires": { - "@rolldown/pluginutils": "1.0.0-rc.2" + "@rolldown/pluginutils": "^1.0.1" } }, "@vue/compiler-core": { @@ -1740,14 +1723,6 @@ "@rolldown/binding-win32-arm64-msvc": "1.1.2", "@rolldown/binding-win32-x64-msvc": "1.1.2", "@rolldown/pluginutils": "^1.0.0" - }, - "dependencies": { - "@rolldown/pluginutils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", - "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", - "dev": true - } } }, "source-map-js": { diff --git a/frontend/package.json b/frontend/package.json index 9bfc8df..ad402bd 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -17,7 +17,7 @@ }, "devDependencies": { "@playwright/test": "^1.49.1", - "@vitejs/plugin-vue": "^6.0.5", + "@vitejs/plugin-vue": "^6.0.7", "vite": "^8.0.4" } }