From 4d9ecd02d2ac2d7babf7b3dc55c1518770d813a6 Mon Sep 17 00:00:00 2001 From: AshrafHosam Date: Wed, 24 Jun 2026 21:35:37 +0300 Subject: [PATCH 01/12] feat: deletion detection + second-granularity scraper intervals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When crawler scans a source listing, any CrawlUrl no longer present is bumped to next_check_at=now so the scraper picks it up in the next batch, fetches it, gets 404, and archives the linked offer. Manually created offers (no CrawlUrl) are HEAD-checked directly by the crawler: filtered by organization, source domain, and crawl_urls__isnull. Published offers → ARCHIVED, drafts → deleted on 404/410. Guard: if live_url_set is empty (network flake), both deletion steps are skipped to prevent mass false-archiving. Scheduler intervals switched from minutes to seconds via env vars CRAWLER_INTERVAL_SECONDS / SCRAPER_INTERVAL_SECONDS (default 1s), updated in docker-compose.yml, backend/.env, and backend/.env.example. --- backend/.env | 3 +- backend/.env.example | 3 +- .../management/commands/run_scraper_worker.py | 12 ++-- backend/content/scrapers/queue_service.py | 67 ++++++++++++++++++- docker-compose.yml | 4 +- 5 files changed, 76 insertions(+), 13 deletions(-) diff --git a/backend/.env b/backend/.env index b4bff02..d335ceb 100644 --- a/backend/.env +++ b/backend/.env @@ -14,7 +14,8 @@ POSTGRES_PORT=5432 POSTGRES_SCHEMA=content SCRAPER_TIMEOUT_SECONDS=30 -SCRAPER_INTERVAL_MINUTES=360 +CRAWLER_INTERVAL_SECONDS=1 +SCRAPER_INTERVAL_SECONDS=1 SCRAPER_RUN_ON_START=true SCRAPER_LLM_FALLBACK_THRESHOLD=0.60 SCRAPER_USER_AGENT=SUNRISE-OSS-Scraper/1.0 diff --git a/backend/.env.example b/backend/.env.example index b4bff02..d335ceb 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -14,7 +14,8 @@ POSTGRES_PORT=5432 POSTGRES_SCHEMA=content SCRAPER_TIMEOUT_SECONDS=30 -SCRAPER_INTERVAL_MINUTES=360 +CRAWLER_INTERVAL_SECONDS=1 +SCRAPER_INTERVAL_SECONDS=1 SCRAPER_RUN_ON_START=true SCRAPER_LLM_FALLBACK_THRESHOLD=0.60 SCRAPER_USER_AGENT=SUNRISE-OSS-Scraper/1.0 diff --git a/backend/content/management/commands/run_scraper_worker.py b/backend/content/management/commands/run_scraper_worker.py index 5195f76..61edaab 100644 --- a/backend/content/management/commands/run_scraper_worker.py +++ b/backend/content/management/commands/run_scraper_worker.py @@ -36,8 +36,8 @@ def add_arguments(self, parser): ) def handle(self, *args, **options): - crawler_interval = int(os.getenv("CRAWLER_INTERVAL_MINUTES", "360")) - scraper_interval = int(os.getenv("SCRAPER_INTERVAL_MINUTES", "5")) + crawler_interval = int(os.getenv("CRAWLER_INTERVAL_SECONDS", "1")) + scraper_interval = int(os.getenv("SCRAPER_INTERVAL_SECONDS", "1")) translation_interval = int(os.getenv("TRANSLATION_INTERVAL_MINUTES", "10")) translation_batch = int(os.getenv("TRANSLATION_BATCH_SIZE", "20")) run_on_start = os.getenv("SCRAPER_RUN_ON_START", "true").lower() == "true" @@ -57,7 +57,7 @@ def handle(self, *args, **options): run_crawler, "interval", id="crawl-discover-urls", - minutes=crawler_interval, + seconds=crawler_interval, max_instances=1, coalesce=True, misfire_grace_time=300, @@ -66,7 +66,7 @@ def handle(self, *args, **options): run_url_scraper_batch, "interval", id="scrape-url-queue", - minutes=scraper_interval, + seconds=scraper_interval, max_instances=1, coalesce=True, misfire_grace_time=60, @@ -112,8 +112,8 @@ def handle(self, *args, **options): self.stdout.write(f"Translation: {translation_summary}") self.stdout.write( - f"Worker running — crawler every {crawler_interval} min, " - f"scraper every {scraper_interval} min, " + f"Worker running — crawler every {crawler_interval}s, " + f"scraper every {scraper_interval}s, " f"translation every {translation_interval} min. Press Ctrl+C to stop." ) diff --git a/backend/content/scrapers/queue_service.py b/backend/content/scrapers/queue_service.py index 85d442f..8f0e8bc 100644 --- a/backend/content/scrapers/queue_service.py +++ b/backend/content/scrapers/queue_service.py @@ -3,6 +3,7 @@ from dataclasses import replace from datetime import timedelta from types import SimpleNamespace +from urllib.parse import urlparse import requests from django.db import transaction @@ -30,7 +31,7 @@ class CrawlerService(ScrapeService): def run(self) -> dict: sources = get_sources(self.source_keys) - stats = {"sources": 0, "discovered": 0, "new": 0, "already_known": 0, "errors": 0} + stats = {"sources": 0, "discovered": 0, "new": 0, "already_known": 0, "bumped": 0, "manual_archived": 0, "errors": 0} for source in sources: stats["sources"] += 1 @@ -40,9 +41,12 @@ def run(self) -> dict: stats["discovered"] += result["discovered"] stats["new"] += result["new"] stats["already_known"] += result["already_known"] + stats["bumped"] += result["bumped"] + stats["manual_archived"] += result["manual_archived"] LOGGER.info( - "[%s] Crawler done — discovered=%d new=%d known=%d", + "[%s] Crawler done — discovered=%d new=%d known=%d bumped=%d manual_archived=%d", source.key, result["discovered"], result["new"], result["already_known"], + result["bumped"], result["manual_archived"], ) except Exception: stats["errors"] += 1 @@ -52,6 +56,7 @@ def run(self) -> dict: def _crawl_source(self, source: SourceDefinition) -> dict: urls, _ = self._discover_urls_bfs(source) + live_url_set = set(urls) new_count = 0 known_count = 0 @@ -70,7 +75,63 @@ def _crawl_source(self, source: SourceDefinition) -> dict: elif crawl_url.status != CrawlUrl.UrlStatus.ARCHIVED: known_count += 1 - return {"discovered": len(urls), "new": new_count, "already_known": known_count} + # Guard: skip deletion logic if listing returned 0 URLs (network flake / empty page) + # to avoid mass-archiving all known offers for this source. + if live_url_set: + # Bump CrawlUrls no longer in listing → immediate recheck → scraper fetches → 404 → archive + bumped = CrawlUrl.objects.filter( + source_key=source.key, + status__in=[CrawlUrl.UrlStatus.DONE, CrawlUrl.UrlStatus.ERROR], + ).exclude(url__in=live_url_set).update(next_check_at=timezone.now()) + + # HEAD-check manually created offers (no CrawlUrl) from this org not in live set + manual_archived = self._check_manual_offers(source, live_url_set) + else: + bumped = 0 + manual_archived = 0 + LOGGER.warning("[%s] Live URL set empty — skipping deletion detection", source.key) + + return { + "discovered": len(urls), + "new": new_count, + "already_known": known_count, + "bumped": bumped, + "manual_archived": manual_archived, + } + + def _check_manual_offers(self, source: SourceDefinition, live_url_set: set[str]) -> int: + """HEAD-check published/draft offers from this org that have no CrawlUrl entry, + are not in the current live URL set, and whose link matches the source domain. + Archives published offers and deletes drafts if their link returns 404/410.""" + if not source.organization_id: + return 0 + + source_netloc = urlparse(source.url).netloc + archived_count = 0 + + manual_offers = ( + Offer.objects.filter( + organization_id=source.organization_id, + status__in=[Offer.OfferStatus.PUBLISHED, Offer.OfferStatus.DRAFT], + ) + .exclude(link__in=live_url_set) + .filter(link__icontains=source_netloc) + .filter(crawl_urls__isnull=True) + ) + + for offer in manual_offers: + status_code = self._fetch_status_code(offer.link) + if status_code in {404, 410}: + archived_count += 1 + if offer.status == Offer.OfferStatus.PUBLISHED: + offer.status = Offer.OfferStatus.ARCHIVED + offer.save(update_fields=["status", "updated_at"]) + LOGGER.info("[%s] ARCHIVED manual offer (HTTP %s) — %s", source.key, status_code, offer.link) + elif offer.status == Offer.OfferStatus.DRAFT: + offer.delete() + LOGGER.info("[%s] DELETED manual draft offer (HTTP %s) — %s", source.key, status_code, offer.link) + + return archived_count class UrlScraperService(ScrapeService): diff --git a/docker-compose.yml b/docker-compose.yml index fad6d7e..82dc0fd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,8 +99,8 @@ services: POSTGRES_PORT: 5432 POSTGRES_SCHEMA: ${POSTGRES_SCHEMA:-content} SCRAPER_TIMEOUT_SECONDS: ${SCRAPER_TIMEOUT_SECONDS:-30} - CRAWLER_INTERVAL_MINUTES: ${CRAWLER_INTERVAL_MINUTES:-5} - SCRAPER_INTERVAL_MINUTES: ${SCRAPER_INTERVAL_MINUTES:-5} + CRAWLER_INTERVAL_SECONDS: ${CRAWLER_INTERVAL_SECONDS:-1} + SCRAPER_INTERVAL_SECONDS: ${SCRAPER_INTERVAL_SECONDS:-1} SCRAPER_BATCH_SIZE: ${SCRAPER_BATCH_SIZE:-10} SCRAPER_REVISIT_DAYS: ${SCRAPER_REVISIT_DAYS:-7} SCRAPER_MAX_CONSECUTIVE_ERRORS: ${SCRAPER_MAX_CONSECUTIVE_ERRORS:-3} From f1d0e7401e6b1054cb080ecfaae6a153bb7a5d7a Mon Sep 17 00:00:00 2001 From: AshrafHosam Date: Wed, 24 Jun 2026 21:46:42 +0300 Subject: [PATCH 02/12] fix: update @vitejs/plugin-vue to 6.0.7 for Vite 8 rolldown compatibility --- frontend/package-lock.json | 55 +++++++++++--------------------------- frontend/package.json | 2 +- 2 files changed, 16 insertions(+), 41 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 65c9d0b..212020d 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -14,7 +14,7 @@ }, "devDependencies": { "@playwright/test": "^1.49.1", - "@vitejs/plugin-vue": "^6.0.5", + "@vitejs/plugin-vue": "^6.0.7", "vite": "^8.0.4" } }, @@ -293,9 +293,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -313,9 +310,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -333,9 +327,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -353,9 +344,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -373,9 +361,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -393,9 +378,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -476,7 +458,9 @@ } }, "node_modules/@rolldown/pluginutils": { - "version": "1.0.0-rc.2", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", "dev": true, "license": "MIT" }, @@ -492,11 +476,13 @@ } }, "node_modules/@vitejs/plugin-vue": { - "version": "6.0.5", + "version": "6.0.7", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-6.0.7.tgz", + "integrity": "sha512-km+p+XdSz9Sxm5rqUbqcSfZYaAniKxWBj1KURl+Jr7UaPvvX7BmaWMdP69I5rrFDeQGyxAG7NXdc57vz+snhWg==", "dev": true, "license": "MIT", "dependencies": { - "@rolldown/pluginutils": "1.0.0-rc.2" + "@rolldown/pluginutils": "^1.0.1" }, "engines": { "node": "^20.19.0 || >=22.12.0" @@ -1057,13 +1043,6 @@ "@rolldown/binding-win32-x64-msvc": "1.1.2" } }, - "node_modules/rolldown/node_modules/@rolldown/pluginutils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", - "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", - "dev": true, - "license": "MIT" - }, "node_modules/source-map-js": { "version": "1.2.1", "license": "BSD-3-Clause", @@ -1451,7 +1430,9 @@ "optional": true }, "@rolldown/pluginutils": { - "version": "1.0.0-rc.2", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", "dev": true }, "@tybys/wasm-util": { @@ -1465,10 +1446,12 @@ } }, "@vitejs/plugin-vue": { - "version": "6.0.5", + "version": "6.0.7", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-6.0.7.tgz", + "integrity": "sha512-km+p+XdSz9Sxm5rqUbqcSfZYaAniKxWBj1KURl+Jr7UaPvvX7BmaWMdP69I5rrFDeQGyxAG7NXdc57vz+snhWg==", "dev": true, "requires": { - "@rolldown/pluginutils": "1.0.0-rc.2" + "@rolldown/pluginutils": "^1.0.1" } }, "@vue/compiler-core": { @@ -1740,14 +1723,6 @@ "@rolldown/binding-win32-arm64-msvc": "1.1.2", "@rolldown/binding-win32-x64-msvc": "1.1.2", "@rolldown/pluginutils": "^1.0.0" - }, - "dependencies": { - "@rolldown/pluginutils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", - "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", - "dev": true - } } }, "source-map-js": { diff --git a/frontend/package.json b/frontend/package.json index 9bfc8df..ad402bd 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -17,7 +17,7 @@ }, "devDependencies": { "@playwright/test": "^1.49.1", - "@vitejs/plugin-vue": "^6.0.5", + "@vitejs/plugin-vue": "^6.0.7", "vite": "^8.0.4" } } From 9c8e922af5fd77892bf454a829158860f90d0d58 Mon Sep 17 00:00:00 2001 From: AshrafHosam Date: Wed, 24 Jun 2026 22:19:07 +0300 Subject: [PATCH 03/12] fix: deletion detection guard removal, FK clear after draft delete, type-aware mock template - Remove empty-live-set guard: network errors throw exceptions (caught in run()); a 200 response with 0 URLs is a genuine empty listing, not a flake - Clear crawl_url.offer_id after offer.delete() to avoid Django's 'save() prohibited to prevent data loss due to unsaved related object' ValueError - Update mock_website_detail.html meta description to use type-specific keywords so TF-IDF classifier reliably scores above 0.30 threshold (tested: 0.59-0.67) --- backend/content/scrapers/queue_service.py | 24 +++++++---------- .../content/mock_website_detail.html | 26 +++++++++++++++++++ 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/backend/content/scrapers/queue_service.py b/backend/content/scrapers/queue_service.py index 8f0e8bc..412f5a7 100644 --- a/backend/content/scrapers/queue_service.py +++ b/backend/content/scrapers/queue_service.py @@ -75,21 +75,16 @@ def _crawl_source(self, source: SourceDefinition) -> dict: elif crawl_url.status != CrawlUrl.UrlStatus.ARCHIVED: known_count += 1 - # Guard: skip deletion logic if listing returned 0 URLs (network flake / empty page) - # to avoid mass-archiving all known offers for this source. - if live_url_set: - # Bump CrawlUrls no longer in listing → immediate recheck → scraper fetches → 404 → archive - bumped = CrawlUrl.objects.filter( - source_key=source.key, - status__in=[CrawlUrl.UrlStatus.DONE, CrawlUrl.UrlStatus.ERROR], - ).exclude(url__in=live_url_set).update(next_check_at=timezone.now()) + # Bump CrawlUrls no longer in listing → immediate recheck → scraper fetches → 404 → archive. + # Network errors raise exceptions (caught in run()) so an empty live_url_set here means + # the listing genuinely has no offers — proceed with deletion detection. + bumped = CrawlUrl.objects.filter( + source_key=source.key, + status__in=[CrawlUrl.UrlStatus.DONE, CrawlUrl.UrlStatus.ERROR], + ).exclude(url__in=live_url_set).update(next_check_at=timezone.now()) - # HEAD-check manually created offers (no CrawlUrl) from this org not in live set - manual_archived = self._check_manual_offers(source, live_url_set) - else: - bumped = 0 - manual_archived = 0 - LOGGER.warning("[%s] Live URL set empty — skipping deletion detection", source.key) + # HEAD-check manually created offers (no CrawlUrl) from this org not in live set + manual_archived = self._check_manual_offers(source, live_url_set) return { "discovered": len(urls), @@ -455,6 +450,7 @@ def _archive_offer(self, crawl_url: CrawlUrl) -> None: LOGGER.info("Archived offer — %s", crawl_url.url) elif offer.status == Offer.OfferStatus.DRAFT: offer.delete() + crawl_url.offer_id = None # clear in-memory FK after deletion (SET_NULL already applied in DB) LOGGER.info("Deleted draft offer — %s", crawl_url.url) except Offer.DoesNotExist: pass diff --git a/backend/content/templates/content/mock_website_detail.html b/backend/content/templates/content/mock_website_detail.html index f8f2ee6..1c91f38 100644 --- a/backend/content/templates/content/mock_website_detail.html +++ b/backend/content/templates/content/mock_website_detail.html @@ -3,7 +3,33 @@ {{ opportunity.title }} — Demo University + {% if opportunity.offer_type == "internship" %} + + {% elif opportunity.offer_type == "thesis" %} + + {% elif opportunity.offer_type == "research_group" %} + + {% elif opportunity.offer_type == "project_opportunity" %} + + {% elif opportunity.offer_type == "training" %} + + {% elif opportunity.offer_type == "hackathon" %} + + {% elif opportunity.offer_type == "challenge" %} + + {% elif opportunity.offer_type == "lab" %} + + {% elif opportunity.offer_type == "funding_partner" %} + + {% elif opportunity.offer_type == "testbed" %} + + {% elif opportunity.offer_type == "service" %} + + {% elif opportunity.offer_type == "co_creation" %} + + {% else %} + {% endif %}
From 0af145f6b252d07e5a1acc1e09602e4bc59911e0 Mon Sep 17 00:00:00 2001 From: AshrafHosam Date: Wed, 24 Jun 2026 22:46:50 +0300 Subject: [PATCH 04/12] fix: distinguish 404-archived URLs from true scraper errors in tracking UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 404/410 during deletion detection is expected success — the scraper correctly archives the offer. Previously it emitted event=url_failed (shown as red "error" chip) while errors_count stayed 0, causing a confusing mismatch. Now 404/410 emits event=url_archived, the URL table shows a grey "archived" chip, and only genuine errors (5xx, network timeouts) increment errors_count and show as red. --- backend/content/scrapers/queue_service.py | 11 ++++++++--- .../app/pages/scrapper-admin-page.component.css | 9 ++++++--- .../pages/scrapper-admin-page.component.html | 7 ++++--- backend/ui/src/app/shared/run-log.parser.ts | 17 +++++++++++++++++ 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/backend/content/scrapers/queue_service.py b/backend/content/scrapers/queue_service.py index 412f5a7..3c501b0 100644 --- a/backend/content/scrapers/queue_service.py +++ b/backend/content/scrapers/queue_service.py @@ -238,9 +238,14 @@ def _scrape_one( except requests.exceptions.HTTPError as exc: http_status = exc.response.status_code if exc.response is not None else None self._handle_http_error(crawl_url, http_status, stats) - logs.append({"ts": _ts(), "event": "url_failed", "level": "warn", - "source_key": source.key, "url": crawl_url.url, - "http_status": http_status, "reason": "http_error"}) + if http_status in {404, 410}: + logs.append({"ts": _ts(), "event": "url_archived", "level": "info", + "source_key": source.key, "url": crawl_url.url, + "http_status": http_status, "reason": "gone"}) + else: + logs.append({"ts": _ts(), "event": "url_failed", "level": "warn", + "source_key": source.key, "url": crawl_url.url, + "http_status": http_status, "reason": "http_error"}) return except requests.RequestException as exc: self._handle_transient_error(crawl_url, str(exc), None, stats) diff --git a/backend/ui/src/app/pages/scrapper-admin-page.component.css b/backend/ui/src/app/pages/scrapper-admin-page.component.css index 09ad36a..f376fa6 100644 --- a/backend/ui/src/app/pages/scrapper-admin-page.component.css +++ b/backend/ui/src/app/pages/scrapper-admin-page.component.css @@ -249,6 +249,8 @@ header h1 { .failed-row:hover td { background: #fff0f0; } .skipped-row td { background: #fffcf0; } .skipped-row:hover td { background: #fff8e0; } +.archived-row td { background: #f4f4f4; } +.archived-row:hover td { background: #ebebeb; } .url-table-wrap { overflow-x: auto; border-radius: 10px; border: 1px solid var(--border, #e4e4e0); } @@ -462,9 +464,10 @@ header h1 { .warning-text { color: #856404; } .chip-inline { padding: 0.05rem 0.4rem; border-radius: 6px; font-size: 0.7rem; font-weight: 600; } -.chip-ok { background: #e7f4ea; color: #1d7347; } -.chip-warn { background: #fff3cd; color: #856404; } -.chip-fail { background: #ffe4e4; color: #9e2a2a; } +.chip-ok { background: #e7f4ea; color: #1d7347; } +.chip-warn { background: #fff3cd; color: #856404; } +.chip-fail { background: #ffe4e4; color: #9e2a2a; } +.chip-archived { background: #e9ecef; color: #495057; } .top-errors { background: #fff; diff --git a/backend/ui/src/app/pages/scrapper-admin-page.component.html b/backend/ui/src/app/pages/scrapper-admin-page.component.html index 2e23d67..6c13087 100644 --- a/backend/ui/src/app/pages/scrapper-admin-page.component.html +++ b/backend/ui/src/app/pages/scrapper-admin-page.component.html @@ -235,7 +235,7 @@

{{ run.source_key }}

- + {{ r.url }} @@ -244,10 +244,11 @@

{{ run.source_key }}

{{ r.confidence != null ? (r.confidence | number:'1.2-2') : '—' }} - {{ r.failed ? ('scrapper.statusError' | translate) : r.neglected ? ('scrapper.skipped' | translate) : (r.action ?? ('scrapper.ok' | translate)) }} + {{ r.failed ? ('scrapper.statusError' | translate) : r.neglected ? ('scrapper.skipped' | translate) : r.archived ? ('scrapper.archived' | translate) : (r.action ?? ('scrapper.ok' | translate)) }} {{ r.message ?? r.reason ?? '' }} diff --git a/backend/ui/src/app/shared/run-log.parser.ts b/backend/ui/src/app/shared/run-log.parser.ts index 280c104..4f650cd 100644 --- a/backend/ui/src/app/shared/run-log.parser.ts +++ b/backend/ui/src/app/shared/run-log.parser.ts @@ -24,6 +24,7 @@ export type UrlResult = { duration_ms?: number; failed: boolean; neglected: boolean; + archived: boolean; message?: string; reason?: string; action?: string; @@ -79,6 +80,8 @@ export function toUrlResults(entries: RunLogEntry[]): UrlResult[] { entry.event === 'url_neglected' || (entry.event === 'url_failed' && entry.level === 'info'); + const isArchived = entry.event === 'url_archived'; + if (!existing) { urlMap.set(entry.url, { url: entry.url, @@ -88,6 +91,7 @@ export function toUrlResults(entries: RunLogEntry[]): UrlResult[] { duration_ms: entry.duration_ms, failed: entry.event === 'url_failed' && !isNeglect, neglected: isNeglect, + archived: isArchived, message: entry.message, reason: entry.reason, action: entry.action, @@ -101,6 +105,7 @@ export function toUrlResults(entries: RunLogEntry[]): UrlResult[] { duration_ms: entry.duration_ms ?? existing.duration_ms, failed: false, neglected: false, + archived: false, action: entry.action ?? existing.action, ts: entry.ts ?? existing.ts, }); @@ -109,6 +114,17 @@ export function toUrlResults(entries: RunLogEntry[]): UrlResult[] { ...existing, neglected: true, failed: false, + archived: false, + reason: entry.reason ?? existing.reason, + ts: entry.ts ?? existing.ts, + }); + } else if (isArchived) { + urlMap.set(entry.url, { + ...existing, + archived: true, + failed: false, + neglected: false, + http_status: entry.http_status ?? existing.http_status, reason: entry.reason ?? existing.reason, ts: entry.ts ?? existing.ts, }); @@ -117,6 +133,7 @@ export function toUrlResults(entries: RunLogEntry[]): UrlResult[] { ...existing, failed: true, neglected: false, + archived: false, http_status: entry.http_status ?? existing.http_status, message: entry.message ?? existing.message, reason: entry.reason ?? existing.reason, From b1b1fe30888d44627d7402b5dc0b3d5dcb73b344 Mon Sep 17 00:00:00 2001 From: AshrafHosam Date: Wed, 24 Jun 2026 22:54:29 +0300 Subject: [PATCH 05/12] fix: always archive offers on deletion detection, never hard-delete Draft and published offers both move to ARCHIVED when their source URL returns 404/410. Hard-deleting drafts lost audit trail and any manual enrichment done before publishing. --- backend/content/scrapers/queue_service.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/backend/content/scrapers/queue_service.py b/backend/content/scrapers/queue_service.py index 3c501b0..fcf30a9 100644 --- a/backend/content/scrapers/queue_service.py +++ b/backend/content/scrapers/queue_service.py @@ -449,14 +449,11 @@ def _archive_offer(self, crawl_url: CrawlUrl) -> None: return try: offer = crawl_url.offer - if offer.status == Offer.OfferStatus.PUBLISHED: - offer.status = Offer.OfferStatus.ARCHIVED - offer.save(update_fields=["status", "updated_at"]) - LOGGER.info("Archived offer — %s", crawl_url.url) - elif offer.status == Offer.OfferStatus.DRAFT: - offer.delete() - crawl_url.offer_id = None # clear in-memory FK after deletion (SET_NULL already applied in DB) - LOGGER.info("Deleted draft offer — %s", crawl_url.url) + if offer.status not in {Offer.OfferStatus.PUBLISHED, Offer.OfferStatus.DRAFT}: + return + offer.status = Offer.OfferStatus.ARCHIVED + offer.save(update_fields=["status", "updated_at"]) + LOGGER.info("Archived offer (was %s) — %s", offer.status, crawl_url.url) except Offer.DoesNotExist: pass From 1430dd62c2995f75ca0e12d5aa673d909238efbb Mon Sep 17 00:00:00 2001 From: AshrafHosam Date: Wed, 24 Jun 2026 22:58:36 +0300 Subject: [PATCH 06/12] feat: archive offers on any 4xx or 200-with-empty-page, keep backoff for 5xx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Any 4xx client error means the resource is gone/inaccessible — archive immediately instead of only 404/410. 5xx server errors keep the existing exponential backoff since they indicate transient server issues. 200 responses returning a generic/empty page also archive the linked offer (if one exists); new URLs with no linked offer still just neglect. --- backend/content/scrapers/queue_service.py | 27 +++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/backend/content/scrapers/queue_service.py b/backend/content/scrapers/queue_service.py index fcf30a9..7c29e24 100644 --- a/backend/content/scrapers/queue_service.py +++ b/backend/content/scrapers/queue_service.py @@ -238,10 +238,10 @@ def _scrape_one( except requests.exceptions.HTTPError as exc: http_status = exc.response.status_code if exc.response is not None else None self._handle_http_error(crawl_url, http_status, stats) - if http_status in {404, 410}: + if http_status is not None and 400 <= http_status < 500: logs.append({"ts": _ts(), "event": "url_archived", "level": "info", "source_key": source.key, "url": crawl_url.url, - "http_status": http_status, "reason": "gone"}) + "http_status": http_status, "reason": "client_error"}) else: logs.append({"ts": _ts(), "event": "url_failed", "level": "warn", "source_key": source.key, "url": crawl_url.url, @@ -257,11 +257,20 @@ def _scrape_one( extracted = extract_deterministic(html, page_source) if is_generic_page(extracted.title): - stats["neglected"] += 1 - LOGGER.info("[%s] NEGLECT %s — generic_page_title", source.key, crawl_url.url) - logs.append({"ts": _ts(), "event": "url_neglected", "level": "info", - "source_key": source.key, "url": crawl_url.url, "reason": "generic_page_title"}) - self._mark_done(crawl_url) + if crawl_url.offer_id is not None: + self._archive_offer(crawl_url) + crawl_url.status = CrawlUrl.UrlStatus.ARCHIVED + crawl_url.save() + stats["archived"] += 1 + LOGGER.info("[%s] ARCHIVE %s — generic_page (offer existed)", source.key, crawl_url.url) + logs.append({"ts": _ts(), "event": "url_archived", "level": "info", + "source_key": source.key, "url": crawl_url.url, "reason": "generic_page"}) + else: + stats["neglected"] += 1 + LOGGER.info("[%s] NEGLECT %s — generic_page_title", source.key, crawl_url.url) + logs.append({"ts": _ts(), "event": "url_neglected", "level": "info", + "source_key": source.key, "url": crawl_url.url, "reason": "generic_page_title"}) + self._mark_done(crawl_url) return if self.use_llm_fallback and source.llm_fallback_enabled: @@ -414,8 +423,8 @@ def _mark_done(self, crawl_url: CrawlUrl) -> None: crawl_url.save() def _handle_http_error(self, crawl_url: CrawlUrl, http_status: int | None, stats: dict) -> None: - if http_status in {404, 410}: - LOGGER.info("URL permanently gone (HTTP %s) — %s", http_status, crawl_url.url) + if http_status is not None and 400 <= http_status < 500: + LOGGER.info("URL client error (HTTP %s) — %s", http_status, crawl_url.url) self._archive_offer(crawl_url) crawl_url.status = CrawlUrl.UrlStatus.ARCHIVED crawl_url.last_http_status = http_status From b2b469eae8757f32ee6cae183c8f4f7cc8be75b0 Mon Sep 17 00:00:00 2001 From: AshrafHosam Date: Wed, 24 Jun 2026 23:22:55 +0300 Subject: [PATCH 07/12] fix: store clean description, classify from body text not meta keywords Meta description was enriched with type keywords causing those keywords to be stored as the offer description. Reverted meta to real description only. Type keywords now come from body list items (offer_type field) which feed extracted.details. Classifier now uses title+summary+details so it picks up the body signal. --- backend/content/scrapers/service.py | 2 +- .../content/mock_website_detail.html | 29 ++----------------- 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/backend/content/scrapers/service.py b/backend/content/scrapers/service.py index 524a5ed..9b4d703 100644 --- a/backend/content/scrapers/service.py +++ b/backend/content/scrapers/service.py @@ -540,7 +540,7 @@ def _upsert_offer( resolved_offer_type_name = extracted.offer_type if not resolved_offer_type_name: - classifier_text = f"{extracted.title} {extracted.summary}".strip() + classifier_text = f"{extracted.title} {extracted.summary} {extracted.details or ''}".strip() resolved_offer_type_name, confidence = _classifier.classify(classifier_text) if resolved_offer_type_name: LOGGER.info( diff --git a/backend/content/templates/content/mock_website_detail.html b/backend/content/templates/content/mock_website_detail.html index 1c91f38..aae7c25 100644 --- a/backend/content/templates/content/mock_website_detail.html +++ b/backend/content/templates/content/mock_website_detail.html @@ -3,33 +3,7 @@ {{ opportunity.title }} — Demo University - {% if opportunity.offer_type == "internship" %} - - {% elif opportunity.offer_type == "thesis" %} - - {% elif opportunity.offer_type == "research_group" %} - - {% elif opportunity.offer_type == "project_opportunity" %} - - {% elif opportunity.offer_type == "training" %} - - {% elif opportunity.offer_type == "hackathon" %} - - {% elif opportunity.offer_type == "challenge" %} - - {% elif opportunity.offer_type == "lab" %} - - {% elif opportunity.offer_type == "funding_partner" %} - - {% elif opportunity.offer_type == "testbed" %} - - {% elif opportunity.offer_type == "service" %} - - {% elif opportunity.offer_type == "co_creation" %} - - {% else %} - {% endif %}
@@ -39,7 +13,8 @@

{{ opportunity.title }}

{{ opportunity.description }}

    -
  • Type: {{ opportunity.offer_type }}
    • +
  • Type: {{ opportunity.get_offer_type_display }}
    • +
  • Category: {{ opportunity.offer_type }}
  • For: {{ opportunity.target_profile }}
  • Institution: Demo University
From 1b176b748f166ae674d2d1a92bf6be4b91ce3f19 Mon Sep 17 00:00:00 2001 From: AshrafHosam Date: Wed, 24 Jun 2026 23:48:36 +0300 Subject: [PATCH 08/12] fix: replace APScheduler interval jobs with thread loops for crawler/scraper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit APScheduler skipped every tick when crawl took longer than the interval (1s) with multiple sources, flooding logs with 'maximum instances' warnings. Thread loops run fn → sleep N seconds → repeat, guaranteeing no overlap and no skip warnings regardless of how long a crawl takes. APScheduler kept only for infrequent cron jobs (translation, link-checker, archive). --- .../management/commands/run_scraper_worker.py | 76 ++++++++++--------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/backend/content/management/commands/run_scraper_worker.py b/backend/content/management/commands/run_scraper_worker.py index 61edaab..efa6cf0 100644 --- a/backend/content/management/commands/run_scraper_worker.py +++ b/backend/content/management/commands/run_scraper_worker.py @@ -1,6 +1,9 @@ +import logging import os +import threading +import time -from apscheduler.schedulers.blocking import BlockingScheduler +from apscheduler.schedulers.background import BackgroundScheduler from django.core.management.base import BaseCommand from django.utils import timezone @@ -8,6 +11,18 @@ from content.scrapers.link_checker import check_offer_links from content.scrapers.translation_service import run_offer_translation_batch +LOGGER = logging.getLogger(__name__) + + +def _loop(fn: callable, interval: float, stop: threading.Event) -> None: + """Run fn, sleep interval seconds, repeat — guarantees no overlap.""" + while not stop.is_set(): + try: + fn() + except Exception: + LOGGER.exception("Worker loop error in %s", fn.__name__) + stop.wait(interval) + def _archive_expired_offers(): from django.utils import timezone as tz @@ -51,26 +66,27 @@ def handle(self, *args, **options): self.stdout.write(f"Scraper: {scraper_summary}") return - scheduler = BlockingScheduler(timezone=timezone.get_current_timezone_name()) + stop = threading.Event() - scheduler.add_job( - run_crawler, - "interval", - id="crawl-discover-urls", - seconds=crawler_interval, - max_instances=1, - coalesce=True, - misfire_grace_time=300, - ) - scheduler.add_job( - run_url_scraper_batch, - "interval", - id="scrape-url-queue", - seconds=scraper_interval, - max_instances=1, - coalesce=True, - misfire_grace_time=60, - ) + if run_on_start: + self.stdout.write("Startup crawler run...") + self.stdout.write(f"Crawler: {run_crawler()}") + self.stdout.write("Startup scraper batch...") + self.stdout.write(f"Scraper: {run_url_scraper_batch()}") + self.stdout.write("Startup offer translation...") + self.stdout.write(f"Translation: {run_offer_translation_batch(limit=translation_batch)}") + + # Crawler and scraper run in threads: each waits N seconds AFTER the + # previous run completes, so no overlap and no APScheduler skip warnings. + threading.Thread( + target=_loop, args=(run_crawler, crawler_interval, stop), daemon=True, name="crawler" + ).start() + threading.Thread( + target=_loop, args=(run_url_scraper_batch, scraper_interval, stop), daemon=True, name="scraper" + ).start() + + # Cron jobs (infrequent) stay on APScheduler. + scheduler = BackgroundScheduler(timezone=timezone.get_current_timezone_name()) scheduler.add_job( run_offer_translation_batch, "interval", @@ -99,25 +115,17 @@ def handle(self, *args, **options): max_instances=1, coalesce=True, ) - - if run_on_start: - self.stdout.write("Startup crawler run...") - crawler_summary = run_crawler() - self.stdout.write(f"Crawler: {crawler_summary}") - self.stdout.write("Startup scraper batch...") - scraper_summary = run_url_scraper_batch() - self.stdout.write(f"Scraper: {scraper_summary}") - self.stdout.write("Startup offer translation...") - translation_summary = run_offer_translation_batch(limit=translation_batch) - self.stdout.write(f"Translation: {translation_summary}") + scheduler.start() self.stdout.write( - f"Worker running — crawler every {crawler_interval}s, " - f"scraper every {scraper_interval}s, " + f"Worker running — crawler every {crawler_interval}s after completion, " + f"scraper every {scraper_interval}s after completion, " f"translation every {translation_interval} min. Press Ctrl+C to stop." ) try: - scheduler.start() + stop.wait() except (KeyboardInterrupt, SystemExit): + stop.set() + scheduler.shutdown(wait=False) self.stdout.write("Worker stopped.") From 936686104792f4412ec7bd6c8d44290529d89073 Mon Sep 17 00:00:00 2001 From: AshrafHosam Date: Wed, 24 Jun 2026 23:53:27 +0300 Subject: [PATCH 09/12] chore: reduce scraper HTTP timeout from 30s to 1s --- backend/.env | 2 +- backend/.env.example | 2 +- docker-compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/.env b/backend/.env index d335ceb..e107eeb 100644 --- a/backend/.env +++ b/backend/.env @@ -13,7 +13,7 @@ POSTGRES_HOST=localhost POSTGRES_PORT=5432 POSTGRES_SCHEMA=content -SCRAPER_TIMEOUT_SECONDS=30 +SCRAPER_TIMEOUT_SECONDS=1 CRAWLER_INTERVAL_SECONDS=1 SCRAPER_INTERVAL_SECONDS=1 SCRAPER_RUN_ON_START=true diff --git a/backend/.env.example b/backend/.env.example index d335ceb..e107eeb 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -13,7 +13,7 @@ POSTGRES_HOST=localhost POSTGRES_PORT=5432 POSTGRES_SCHEMA=content -SCRAPER_TIMEOUT_SECONDS=30 +SCRAPER_TIMEOUT_SECONDS=1 CRAWLER_INTERVAL_SECONDS=1 SCRAPER_INTERVAL_SECONDS=1 SCRAPER_RUN_ON_START=true diff --git a/docker-compose.yml b/docker-compose.yml index 82dc0fd..3c440f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,7 +98,7 @@ services: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 POSTGRES_SCHEMA: ${POSTGRES_SCHEMA:-content} - SCRAPER_TIMEOUT_SECONDS: ${SCRAPER_TIMEOUT_SECONDS:-30} + SCRAPER_TIMEOUT_SECONDS: ${SCRAPER_TIMEOUT_SECONDS:-1} CRAWLER_INTERVAL_SECONDS: ${CRAWLER_INTERVAL_SECONDS:-1} SCRAPER_INTERVAL_SECONDS: ${SCRAPER_INTERVAL_SECONDS:-1} SCRAPER_BATCH_SIZE: ${SCRAPER_BATCH_SIZE:-10} From 8e35b9274eb4e0dcfc50c65afc9699bb5ffb6771 Mon Sep 17 00:00:00 2001 From: AshrafHosam Date: Thu, 25 Jun 2026 00:14:06 +0300 Subject: [PATCH 10/12] fix: Ollama connection errors skip cooldown wait, fall through to deterministic immediately MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When Ollama is unreachable (connection refused), models were put in 60s cooldown and _wait_for_available_model slept for ~58s before retrying — blocking the scraper thread indefinitely. Now connection errors (non-429) are tracked separately. If all cooled-down models failed due to connection errors, the wait is skipped entirely and the caller falls through to deterministic classification immediately. Rate-limit (429) cooldowns still wait as before. LLM enabled/disabled per-source via admin panel llm_fallback_enabled flag. --- backend/content/scrapers/ollama_client.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/backend/content/scrapers/ollama_client.py b/backend/content/scrapers/ollama_client.py index 3df9654..75d9ef8 100644 --- a/backend/content/scrapers/ollama_client.py +++ b/backend/content/scrapers/ollama_client.py @@ -17,6 +17,11 @@ # at the start of run N+1 before the cooldown has expired. _SHARED_MODEL_COOLDOWN: dict[str, float] = {} +# Models whose cooldown was triggered by a connection error (server down), +# not a 429 rate limit. These skip the cooldown wait — no point waiting +# for a server that isn't running. +_CONNECTION_ERROR_MODELS: set[str] = set() + # Buffer for cooldown events emitted during _wait_for_available_model. # Flushed into run.log by ScrapeService after each LLM call. _COOLDOWN_LOG_BUFFER: list[dict] = [] @@ -58,6 +63,12 @@ def _wait_for_available_model(self) -> list[str]: return models if not _SHARED_MODEL_COOLDOWN: return [] + # If every cooled-down model failed due to a connection error (server + # unreachable), skip the wait and fall through to deterministic immediately. + cooled_models = set(_SHARED_MODEL_COOLDOWN.keys()) + if cooled_models and cooled_models.issubset(_CONNECTION_ERROR_MODELS): + LOGGER.debug("Ollama unreachable — skipping LLM, using deterministic fallback") + return [] soonest = min(_SHARED_MODEL_COOLDOWN.values()) wait = max(0.0, soonest - time.time()) if wait > self.cooldown_max_wait_seconds: @@ -109,7 +120,10 @@ def extract_fallback( status_code = exc.status_code if isinstance(exc, ollama.ResponseError) else None _SHARED_MODEL_COOLDOWN[model] = time.time() + self.model_cooldown_seconds if status_code != 429: + _CONNECTION_ERROR_MODELS.add(model) LOGGER.warning("Ollama fallback unavailable for %s using %s: %s", source.key, model, exc) + else: + _CONNECTION_ERROR_MODELS.discard(model) continue text = response.response @@ -122,6 +136,7 @@ def extract_fallback( confidence = parsed.get("confidence", deterministic_payload.confidence) self.last_switch_count = index + _CONNECTION_ERROR_MODELS.discard(model) LOGGER.info( "LLM fallback success — source=%s model=%s conf=%.2f title=%r", source.key, model, float(confidence), str(title)[:60], @@ -179,7 +194,10 @@ def assess_and_extract( status_code = exc.status_code if isinstance(exc, ollama.ResponseError) else None _SHARED_MODEL_COOLDOWN[model] = time.time() + self.model_cooldown_seconds if status_code != 429: + _CONNECTION_ERROR_MODELS.add(model) LOGGER.warning("Ollama relevance check unavailable for %s using %s: %s", source.key, model, exc) + else: + _CONNECTION_ERROR_MODELS.discard(model) continue text = response.response @@ -200,6 +218,7 @@ def assess_and_extract( ) self.last_switch_count = index + _CONNECTION_ERROR_MODELS.discard(model) LOGGER.info( "LLM assess success — source=%s model=%s is_offer=%s offer_type=%s conf=%.2f reason=%r", source.key, model, is_relevant, offer_type or "(will classify)", confidence, reason[:80] if reason else "", From decb91c1f7319784907a2126f0824f9f592a84c1 Mon Sep 17 00:00:00 2001 From: AshrafHosam Date: Thu, 25 Jun 2026 01:15:23 +0300 Subject: [PATCH 11/12] fix: skip saving empty scraping runs, fix Archived offer log wording MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Empty batches (all neglected, nothing logged) no longer create a ScrapingRun DB row — they are deleted before saving. Only runs with actual log entries (created/updated/archived/errors) appear in the UI tracking tab. --- backend/content/scrapers/queue_service.py | 24 +++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/backend/content/scrapers/queue_service.py b/backend/content/scrapers/queue_service.py index 7c29e24..27997fd 100644 --- a/backend/content/scrapers/queue_service.py +++ b/backend/content/scrapers/queue_service.py @@ -176,16 +176,20 @@ def run_batch(self) -> dict: self._scrape_one(crawl_url, source, source_type, ingestion_user, stats, logs, matched_offer_ids) - run.status = ScrapingRun.RunStatus.SUCCESS - run.offers_processed = stats["processed"] - run.offers_created = stats["created"] - run.offers_updated = stats["updated"] - run.offers_unchanged = stats["unchanged"] - run.errors_count = stats["errors"] - run.urls_neglected = stats["neglected"] - run.log = logs - run.completed_at = timezone.now() - run.save() + # Skip saving runs where nothing noteworthy happened (all neglected, no log entries). + if not logs: + run.delete() + else: + run.status = ScrapingRun.RunStatus.SUCCESS + run.offers_processed = stats["processed"] + run.offers_created = stats["created"] + run.offers_updated = stats["updated"] + run.offers_unchanged = stats["unchanged"] + run.errors_count = stats["errors"] + run.urls_neglected = stats["neglected"] + run.log = logs + run.completed_at = timezone.now() + run.save() LOGGER.info( "URL scraper batch done — processed=%d created=%d updated=%d archived=%d errors=%d neglected=%d", From baa59fbd6685100ea9b646a4024cb2c4edeea913 Mon Sep 17 00:00:00 2001 From: AshrafHosam Date: Thu, 25 Jun 2026 02:41:54 +0300 Subject: [PATCH 12/12] docs: document crawler/scraper architecture, deletion detection, and archiving behaviour --- README.md | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index afed1b9..7c43a19 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,59 @@ docker-compose logs scraper-worker --- +## Scraper & Crawler Architecture + +The scraper worker runs two independent loops in parallel background threads: + +### CrawlerService — discovery loop + +Runs every `CRAWLER_INTERVAL_SECONDS` (default: 1s). + +1. BFS-crawls the source listing page to collect all live offer URLs. +2. Creates a `CrawlUrl` record (status `PENDING`) for any new URL found. +3. **Deletion detection** — any `CrawlUrl` no longer present in the live listing is bumped to `next_check_at = now`, so the scraper re-fetches it immediately and detects the 404. +4. **Manual offer cleanup** — offers created manually in the admin (no `CrawlUrl`) whose link matches the source domain but is absent from the live listing are HEAD-checked directly; those returning 404/410 are archived. + +### UrlScraperService — scrape loop + +Runs every `SCRAPER_INTERVAL_SECONDS` (default: 1s). + +1. Claims all `CrawlUrl` records where `next_check_at <= now`. +2. Fetches each URL (timeout: `SCRAPER_TIMEOUT_SECONDS`, default: 1s). +3. **4xx response** → offer is immediately archived (`PUBLISHED`/`DRAFT` → `ARCHIVED`). +4. **5xx response** → transient error; exponential backoff applied, retried later. +5. **200 with empty/generic page** → if a linked offer exists, it is archived; otherwise the URL is skipped. +6. **200 with valid content** → offer is created or updated. + +### Archiving behaviour + +Offers are **never hard-deleted** from the database. All removal triggers result in `status = ARCHIVED`. Archived offers are hidden from the public Vue frontend but remain visible in the admin dashboard. + +| Trigger | Result | +|---|---| +| URL returns 4xx (404, 410, etc.) | ARCHIVED immediately | +| URL returns 5xx | Backoff, retried later | +| URL returns 200 but page is empty/generic | ARCHIVED (if offer linked) | +| URL disappears from source listing | CrawlUrl bumped → re-fetched → 404 → ARCHIVED | +| Manually created offer URL gone from listing | HEAD-checked → 404/410 → ARCHIVED | + +### Scraper worker configuration + +| Variable | Default | Description | +|---|---|---| +| `CRAWLER_INTERVAL_SECONDS` | `1` | How often the crawler discovery loop runs | +| `SCRAPER_INTERVAL_SECONDS` | `1` | How often the scraper drain loop runs | +| `SCRAPER_TIMEOUT_SECONDS` | `1` | HTTP request timeout per URL | + +### LLM extraction + +Each scraped page is processed by one of two extraction methods: + +- **Ollama LLM (primary)** — structured JSON extraction with offer type classification. Configured per source in the admin panel. +- **Deterministic fallback** — `` as summary; TF-IDF cosine similarity for offer type classification. Used automatically when Ollama is not running or returns a connection error (no wait, no cooldown). + +--- + ## Useful Docker commands ```bash @@ -146,8 +199,11 @@ Each scraped offer is automatically classified into one of the offer types store | Variable | Default | Description | |---|---|---| -| `LLM_ENABLED` | `true` | Enable or disable the Ollama LLM | +| `LLM_ENABLED` | `true` | Enable or disable the Ollama LLM globally | | `SCRAPER_CLASSIFIER_THRESHOLD` | `0.15` | Minimum cosine similarity score for the TF-IDF fallback to accept a classification | +| `CRAWLER_INTERVAL_SECONDS` | `1` | Crawler discovery loop interval | +| `SCRAPER_INTERVAL_SECONDS` | `1` | Scraper drain loop interval | +| `SCRAPER_TIMEOUT_SECONDS` | `1` | HTTP request timeout per scraped URL | ### Adding or updating offer types