diff --git a/backend/content/migrations/0026_scrapingsource_auto_publish_enabled.py b/backend/content/migrations/0026_scrapingsource_auto_publish_enabled.py
new file mode 100644
index 0000000..7a3a8b6
--- /dev/null
+++ b/backend/content/migrations/0026_scrapingsource_auto_publish_enabled.py
@@ -0,0 +1,13 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [("content", "0025_contact_linkedin")]
+
+ operations = [
+ migrations.AddField(
+ model_name="scrapingsource",
+ name="auto_publish_enabled",
+ field=models.BooleanField(default=False),
+ ),
+ ]
diff --git a/backend/content/models.py b/backend/content/models.py
index 678e248..998f2c2 100644
--- a/backend/content/models.py
+++ b/backend/content/models.py
@@ -577,6 +577,7 @@ class ScrapingSource(models.Model):
crawl_max_pages = models.IntegerField(default=25)
crawl_match_patterns = models.JSONField(default=list)
crawl_exclude_patterns = models.JSONField(default=list)
+ auto_publish_enabled = models.BooleanField(default=False)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
diff --git a/backend/content/scrapers/service.py b/backend/content/scrapers/service.py
index 98f039b..f6088df 100644
--- a/backend/content/scrapers/service.py
+++ b/backend/content/scrapers/service.py
@@ -36,6 +36,9 @@
LOGGER = logging.getLogger(__name__)
+AUTO_PUBLISH_LLM_MIN_CONFIDENCE = 0.80
+AUTO_PUBLISH_DETERMINISTIC_MIN_CONFIDENCE = 0.90
+
def _ts() -> str:
return timezone.now().isoformat().replace("+00:00", "Z")
@@ -561,6 +564,11 @@ def _upsert_offer(
organization=organization,
offer_type=offer_type,
).first()
+ auto_publish_threshold = self._auto_publish_threshold(source)
+ auto_publish_allowed = self._should_auto_publish(source, extracted, offer_type)
+ auto_publish_result = (
+ "published" if auto_publish_allowed else "draft"
+ ) if existing is None else "preserved_existing_status"
current_timestamp = timezone.now().isoformat()
@@ -572,6 +580,10 @@ def _upsert_offer(
"confidence": extracted.confidence,
"last_seen_at": current_timestamp,
"stale_candidate": False,
+ "auto_publish_enabled": source.auto_publish_enabled,
+ "auto_publish_threshold": auto_publish_threshold,
+ "auto_publish_result": auto_publish_result,
+ "auto_publish_mode": "llm" if source.llm_fallback_enabled else "deterministic",
}
merged_details = {
@@ -583,6 +595,7 @@ def _upsert_offer(
if self.dry_run:
return "created", natural_key, None
+ offer_status = Offer.OfferStatus.PUBLISHED if auto_publish_allowed else Offer.OfferStatus.DRAFT
offer = Offer.objects.create(
title=extracted.title,
summary=extracted.summary,
@@ -592,7 +605,7 @@ def _upsert_offer(
source_type=source_type,
target_profile=target_profile,
organization=organization,
- status=Offer.OfferStatus.DRAFT,
+ status=offer_status,
created_by=ingestion_user,
updated_by=ingestion_user,
offer_type=offer_type,
@@ -645,6 +658,36 @@ def _upsert_offer(
self._replace_domains(existing, source.domain_names)
return "updated", natural_key, existing
+ @staticmethod
+ def _auto_publish_threshold(source: SourceDefinition) -> float:
+ return (
+ AUTO_PUBLISH_LLM_MIN_CONFIDENCE
+ if source.llm_fallback_enabled
+ else AUTO_PUBLISH_DETERMINISTIC_MIN_CONFIDENCE
+ )
+
+ def _should_auto_publish(
+ self,
+ source: SourceDefinition,
+ extracted: ExtractedPayload,
+ offer_type: OfferType | None,
+ ) -> bool:
+ if not source.auto_publish_enabled:
+ return False
+ if source.llm_fallback_enabled and extracted.method != "llm_primary":
+ return False
+ if extracted.confidence < self._auto_publish_threshold(source):
+ return False
+ if offer_type is None:
+ return False
+ if not extracted.title or not extracted.summary:
+ return False
+ if is_generic_page(extracted.title):
+ return False
+ if not source.llm_fallback_enabled and extracted.summary.startswith("Auto-extracted from"):
+ return False
+ return True
+
def _replace_domains(self, offer: Offer, domain_names: list[str]) -> None:
domain_map = {
domain.name: domain
@@ -771,7 +814,16 @@ def _normalized_details_for_compare(details: dict | None) -> dict:
normalized = deepcopy(details)
scraping = normalized.get("scraping")
if isinstance(scraping, dict):
- for key in ("last_seen_at", "stale_candidate", "stale_marked_at", "stale_reason"):
+ for key in (
+ "last_seen_at",
+ "stale_candidate",
+ "stale_marked_at",
+ "stale_reason",
+ "auto_publish_enabled",
+ "auto_publish_threshold",
+ "auto_publish_result",
+ "auto_publish_mode",
+ ):
scraping.pop(key, None)
normalized["scraping"] = scraping
return normalized
diff --git a/backend/content/scrapers/source_registry.py b/backend/content/scrapers/source_registry.py
index da6437b..e7cd760 100644
--- a/backend/content/scrapers/source_registry.py
+++ b/backend/content/scrapers/source_registry.py
@@ -191,6 +191,7 @@ def get_sources(source_keys: list[str] | None = None) -> list[SourceDefinition]:
crawl_max_pages=s.crawl_max_pages,
crawl_match_patterns=s.crawl_match_patterns,
crawl_exclude_patterns=s.crawl_exclude_patterns,
+ auto_publish_enabled=s.auto_publish_enabled,
)
for s in qs
]
diff --git a/backend/content/scrapers/types.py b/backend/content/scrapers/types.py
index 30aa95d..4237511 100644
--- a/backend/content/scrapers/types.py
+++ b/backend/content/scrapers/types.py
@@ -20,6 +20,7 @@ class SourceDefinition:
crawl_max_pages: int = 25
crawl_match_patterns: list[str] = field(default_factory=list)
crawl_exclude_patterns: list[str] = field(default_factory=list)
+ auto_publish_enabled: bool = False
@dataclass
diff --git a/backend/content/test_scraper_service.py b/backend/content/test_scraper_service.py
index 640f7d2..b3a0e46 100644
--- a/backend/content/test_scraper_service.py
+++ b/backend/content/test_scraper_service.py
@@ -1,4 +1,5 @@
import uuid
+from dataclasses import replace
from unittest.mock import Mock, patch
import requests
@@ -18,7 +19,7 @@
import ollama
from content.scrapers.extractors import extract_links_from_html
from content.scrapers.ollama_client import OllamaClient
-from content.scrapers.service import run_scrape
+from content.scrapers.service import run_scrape, ScrapeService
from content.scrapers.types import ExtractedPayload, SourceDefinition
from content.seeding import uuid_from_token
@@ -72,6 +73,210 @@ def _create_offer(self, details: dict) -> Offer:
OfferDomain.objects.create(offer=offer, domain=self.domain)
return offer
+ def _upsert_extracted(
+ self,
+ source: SourceDefinition,
+ extracted: ExtractedPayload,
+ ) -> Offer:
+ service = ScrapeService(use_llm_fallback=False)
+ action, _, offer = service._upsert_offer(
+ source,
+ self.source_type_scraping,
+ self.user,
+ extracted,
+ )
+ self.assertEqual(action, "created")
+ self.assertIsNotNone(offer)
+ return offer
+
+ def test_auto_publish_disabled_creates_draft(self):
+ source = replace(self.source, auto_publish_enabled=False)
+ offer = self._upsert_extracted(
+ source,
+ ExtractedPayload(
+ offer_type="training",
+ title="Research Training Programme",
+ summary="A structured opportunity for students.",
+ details={},
+ confidence=0.95,
+ method="llm_primary",
+ ),
+ )
+
+ self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+ self.assertFalse(offer.details["scraping"]["auto_publish_enabled"])
+ self.assertEqual(offer.details["scraping"]["auto_publish_result"], "draft")
+
+ def test_llm_auto_publish_above_threshold_creates_published_offer(self):
+ source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=True)
+ offer = self._upsert_extracted(
+ source,
+ ExtractedPayload(
+ offer_type="training",
+ title="Research Training Programme",
+ summary="A structured opportunity for students.",
+ details={},
+ confidence=0.85,
+ method="llm_primary",
+ ),
+ )
+
+ self.assertEqual(offer.status, Offer.OfferStatus.PUBLISHED)
+ self.assertEqual(offer.details["scraping"]["auto_publish_mode"], "llm")
+ self.assertEqual(offer.details["scraping"]["auto_publish_threshold"], 0.80)
+ self.assertEqual(offer.details["scraping"]["auto_publish_result"], "published")
+
+ def test_llm_auto_publish_below_threshold_creates_draft(self):
+ source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=True)
+ offer = self._upsert_extracted(
+ source,
+ ExtractedPayload(
+ offer_type="training",
+ title="Research Training Programme",
+ summary="A structured opportunity for students.",
+ details={},
+ confidence=0.79,
+ method="llm_primary",
+ ),
+ )
+
+ self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+ self.assertEqual(offer.details["scraping"]["auto_publish_result"], "draft")
+
+ def test_llm_auto_publish_requires_llm_primary_payload(self):
+ source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=True)
+ offer = self._upsert_extracted(
+ source,
+ ExtractedPayload(
+ offer_type="training",
+ title="Research Training Programme",
+ summary="A structured opportunity for students.",
+ details={},
+ confidence=0.95,
+ method="deterministic",
+ ),
+ )
+
+ self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+ self.assertEqual(offer.details["scraping"]["auto_publish_result"], "draft")
+
+ def test_deterministic_auto_publish_above_threshold_creates_published_offer(self):
+ source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=False)
+ offer = self._upsert_extracted(
+ source,
+ ExtractedPayload(
+ offer_type="training",
+ title="Research Training Programme",
+ summary="A structured opportunity for students.",
+ details={},
+ confidence=0.92,
+ method="deterministic",
+ ),
+ )
+
+ self.assertEqual(offer.status, Offer.OfferStatus.PUBLISHED)
+ self.assertEqual(offer.details["scraping"]["auto_publish_mode"], "deterministic")
+ self.assertEqual(offer.details["scraping"]["auto_publish_threshold"], 0.90)
+ self.assertEqual(offer.details["scraping"]["auto_publish_result"], "published")
+
+ def test_deterministic_auto_publish_below_threshold_creates_draft(self):
+ source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=False)
+ offer = self._upsert_extracted(
+ source,
+ ExtractedPayload(
+ offer_type="training",
+ title="Research Training Programme",
+ summary="A structured opportunity for students.",
+ details={},
+ confidence=0.89,
+ method="deterministic",
+ ),
+ )
+
+ self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+ self.assertEqual(offer.details["scraping"]["auto_publish_result"], "draft")
+
+ def test_deterministic_auto_publish_fallback_summary_creates_draft(self):
+ source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=False)
+ offer = self._upsert_extracted(
+ source,
+ ExtractedPayload(
+ offer_type="training",
+ title="Research Training Programme",
+ summary="Auto-extracted from https://example.edu/test-source",
+ details={},
+ confidence=0.95,
+ method="deterministic",
+ ),
+ )
+
+ self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+ self.assertEqual(offer.details["scraping"]["auto_publish_result"], "draft")
+
+ def test_auto_publish_generic_title_creates_draft(self):
+ source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=True)
+ offer = self._upsert_extracted(
+ source,
+ ExtractedPayload(
+ offer_type="training",
+ title="Contact Us",
+ summary="A structured opportunity for students.",
+ details={},
+ confidence=0.95,
+ method="llm_primary",
+ ),
+ )
+
+ self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+ self.assertEqual(offer.details["scraping"]["auto_publish_result"], "draft")
+
+ def test_auto_publish_missing_summary_creates_draft(self):
+ source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=True)
+ offer = self._upsert_extracted(
+ source,
+ ExtractedPayload(
+ offer_type="training",
+ title="Research Training Programme",
+ summary="",
+ details={},
+ confidence=0.95,
+ method="llm_primary",
+ ),
+ )
+
+ self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+ self.assertEqual(offer.details["scraping"]["auto_publish_result"], "draft")
+
+ def test_auto_publish_does_not_publish_existing_draft(self):
+ self._create_offer(
+ {
+ "source_name": "Test Source",
+ "scraping": {
+ "source_key": "test_source",
+ "stale_candidate": False,
+ },
+ }
+ )
+ source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=True)
+ service = ScrapeService(use_llm_fallback=False)
+ action, _, offer = service._upsert_offer(
+ source,
+ self.source_type_scraping,
+ self.user,
+ ExtractedPayload(
+ offer_type="training",
+ title="Updated Training Programme",
+ summary="A structured opportunity for students.",
+ details={},
+ confidence=0.95,
+ method="llm_primary",
+ ),
+ )
+
+ self.assertEqual(action, "updated")
+ self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+ self.assertEqual(offer.details["scraping"]["auto_publish_result"], "preserved_existing_status")
+
@patch("content.scrapers.service.get_sources")
@patch("content.scrapers.service.requests.get")
def test_http_404_marks_run_failed_and_deletes_invalid_source_offers(self, mock_get, mock_get_sources):
diff --git a/backend/content/test_sources.py b/backend/content/test_sources.py
index 04eeafc..f1da70c 100644
--- a/backend/content/test_sources.py
+++ b/backend/content/test_sources.py
@@ -73,6 +73,7 @@ def _source_payload(**overrides):
"crawl_max_pages": 25,
"crawl_match_patterns": [],
"crawl_exclude_patterns": [],
+ "auto_publish_enabled": False,
}
base.update(overrides)
return base
@@ -159,8 +160,25 @@ def test_create_success(self):
self.assertEqual(data["key"], "test_source")
self.assertEqual(data["name"], "Test Source")
self.assertEqual(data["url"], "https://example.com")
+ self.assertFalse(data["auto_publish_enabled"])
+ self.assertEqual(data["auto_publish_mode"], "llm")
self.assertTrue(ScrapingSource.objects.filter(key="test_source").exists())
+ def test_create_with_auto_publish_enabled(self):
+ payload = _source_payload(auto_publish_enabled=True, llm_fallback_enabled=False)
+ resp = self.client.post(
+ self.url,
+ data=json.dumps(payload),
+ content_type="application/json",
+ **self._auth(),
+ )
+ self.assertEqual(resp.status_code, 201)
+ data = json.loads(resp.content)
+ self.assertTrue(data["auto_publish_enabled"])
+ self.assertEqual(data["auto_publish_mode"], "deterministic")
+ source = ScrapingSource.objects.get(key="test_source")
+ self.assertTrue(source.auto_publish_enabled)
+
def test_create_missing_required_fields(self):
resp = self.client.post(
self.url,
@@ -208,6 +226,8 @@ def test_create_sets_defaults(self):
self.assertEqual(data["interval_minutes"], 360)
self.assertTrue(data["llm_fallback_enabled"])
self.assertTrue(data["enabled"])
+ self.assertFalse(data["auto_publish_enabled"])
+ self.assertEqual(data["auto_publish_mode"], "llm")
self.assertNotIn("crawl_enabled", data)
@@ -241,6 +261,8 @@ def test_get_detail_success(self):
data = json.loads(resp.content)
self.assertEqual(data["key"], "existing_src")
self.assertEqual(data["name"], "Existing Source")
+ self.assertFalse(data["auto_publish_enabled"])
+ self.assertEqual(data["auto_publish_mode"], "llm")
def test_get_detail_not_found(self):
resp = self.client.get("/api/admin/sources/nonexistent", **self._auth())
@@ -279,6 +301,7 @@ def test_patch_llm_fallback_toggle(self):
self.assertEqual(resp.status_code, 200)
data = json.loads(resp.content)
self.assertFalse(data["llm_fallback_enabled"])
+ self.assertEqual(data["auto_publish_mode"], "deterministic")
self.source.refresh_from_db()
self.assertFalse(self.source.llm_fallback_enabled)
@@ -292,6 +315,19 @@ def test_patch_enabled_toggle(self):
self.assertEqual(resp.status_code, 200)
self.assertFalse(json.loads(resp.content)["enabled"])
+ def test_patch_auto_publish_toggle(self):
+ resp = self.client.patch(
+ self.url,
+ data=json.dumps({"auto_publish_enabled": True}),
+ content_type="application/json",
+ **self._auth(),
+ )
+ self.assertEqual(resp.status_code, 200)
+ data = json.loads(resp.content)
+ self.assertTrue(data["auto_publish_enabled"])
+ self.source.refresh_from_db()
+ self.assertTrue(self.source.auto_publish_enabled)
+
def test_patch_interval_minutes(self):
resp = self.client.patch(
self.url,
@@ -407,6 +443,17 @@ def test_get_sources_returns_source_definition_objects(self):
self.assertIsInstance(src, SourceDefinition)
self.assertEqual(src.key, "typed_src")
self.assertEqual(src.country, "DE")
+ self.assertFalse(src.auto_publish_enabled)
+
+ def test_get_sources_maps_auto_publish_enabled(self):
+ from content.scrapers.source_registry import get_sources
+ ScrapingSource.objects.create(
+ key="auto_src", name="Auto", url="https://auto.com",
+ organization_token="org_x", auto_publish_enabled=True,
+ )
+ results = get_sources(source_keys=["auto_src"])
+ self.assertEqual(len(results), 1)
+ self.assertTrue(results[0].auto_publish_enabled)
def test_get_sources_empty_db(self):
from content.scrapers.source_registry import get_sources
diff --git a/backend/content/views/sources.py b/backend/content/views/sources.py
index c7a3f2e..d323e26 100644
--- a/backend/content/views/sources.py
+++ b/backend/content/views/sources.py
@@ -13,7 +13,7 @@
"name", "url", "target_profile",
"country", "domain_names", "interval_minutes", "llm_fallback_enabled",
"enabled", "quality", "crawl_depth", "crawl_max_pages",
- "crawl_match_patterns", "crawl_exclude_patterns",
+ "crawl_match_patterns", "crawl_exclude_patterns", "auto_publish_enabled",
}
@@ -42,6 +42,8 @@ def _serialize(s: ScrapingSource) -> dict:
"crawl_max_pages": s.crawl_max_pages,
"crawl_match_patterns": s.crawl_match_patterns,
"crawl_exclude_patterns": s.crawl_exclude_patterns,
+ "auto_publish_enabled": s.auto_publish_enabled,
+ "auto_publish_mode": "llm" if s.llm_fallback_enabled else "deterministic",
"created_at": s.created_at.isoformat(),
"updated_at": s.updated_at.isoformat(),
}
@@ -93,6 +95,7 @@ def admin_sources_collection(request):
crawl_max_pages=int(body.get("crawl_max_pages") or 25),
crawl_match_patterns=body.get("crawl_match_patterns") or [],
crawl_exclude_patterns=body.get("crawl_exclude_patterns") or [],
+ auto_publish_enabled=bool(body.get("auto_publish_enabled", False)),
)
source.refresh_from_db()
return JsonResponse(_serialize(source), status=201)
@@ -130,7 +133,7 @@ def admin_source_detail(request, key: str):
except (Organization.DoesNotExist, ValueError):
return JsonResponse({"detail": "Organization not found."}, status=404)
- bool_fields = {"llm_fallback_enabled", "enabled"}
+ bool_fields = {"llm_fallback_enabled", "enabled", "auto_publish_enabled"}
int_fields = {"interval_minutes", "crawl_depth", "crawl_max_pages"}
list_fields = {"domain_names", "crawl_match_patterns", "crawl_exclude_patterns"}
diff --git a/backend/ui/src/app/pages/sources-admin-page.component.css b/backend/ui/src/app/pages/sources-admin-page.component.css
index ed182a1..adb6df5 100644
--- a/backend/ui/src/app/pages/sources-admin-page.component.css
+++ b/backend/ui/src/app/pages/sources-admin-page.component.css
@@ -114,6 +114,23 @@ header h1 {
.data-table tr:hover td { background: #f8fbff; }
+.auto-publish-cell {
+ white-space: nowrap;
+}
+
+.criteria-link {
+ border: none;
+ background: transparent;
+ color: #246b9f;
+ cursor: pointer;
+ font-size: 0.74rem;
+ font-weight: 600;
+ margin-left: 0.35rem;
+ padding: 0.1rem 0.15rem;
+}
+
+.criteria-link:hover { text-decoration: underline; }
+
/* ── Modal form ──────────────────────────────────────────── */
.form-grid {
display: grid;
@@ -221,6 +238,70 @@ header h1 {
.modal-lg { max-width: 700px; width: 95vw; }
+.criteria-summary {
+ border: 1px solid #d7e5ef;
+ background: #f7fbfe;
+ border-radius: var(--r, 6px);
+ padding: 0.65rem 0.8rem;
+ margin: -0.15rem 0 1rem;
+}
+
+.criteria-summary-head {
+ display: flex;
+ justify-content: space-between;
+ gap: 0.75rem;
+ align-items: center;
+ color: #28465d;
+ font-size: 0.8rem;
+ margin-bottom: 0.35rem;
+}
+
+.criteria-summary ul,
+.criteria-list {
+ margin: 0;
+ padding-left: 1rem;
+ color: #3d505d;
+ font-size: 0.8rem;
+}
+
+.criteria-summary li,
+.criteria-list li {
+ margin: 0.18rem 0;
+}
+
+.criteria-modal { max-width: 460px; width: 92vw; }
+
+.criteria-modal h3 {
+ margin: 0.9rem 0 0.45rem;
+ color: var(--ink, #111110);
+ font-size: 0.9rem;
+}
+
+.criteria-facts {
+ display: grid;
+ grid-template-columns: repeat(2, minmax(0, 1fr));
+ gap: 0.6rem;
+}
+
+.criteria-facts div {
+ border: 1px solid #d7e5ef;
+ border-radius: var(--r, 6px);
+ padding: 0.55rem 0.65rem;
+ background: #f7fbfe;
+}
+
+.criteria-facts span {
+ display: block;
+ color: #6a7e8f;
+ font-size: 0.72rem;
+ margin-bottom: 0.2rem;
+}
+
+.criteria-facts strong {
+ color: #18384f;
+ font-size: 0.92rem;
+}
+
/* ── Misc ────────────────────────────────────────────────── */
.center { text-align: center; }
.mono { font-family: 'Courier New', monospace; }
diff --git a/backend/ui/src/app/pages/sources-admin-page.component.html b/backend/ui/src/app/pages/sources-admin-page.component.html
index b2db348..ab2137e 100644
--- a/backend/ui/src/app/pages/sources-admin-page.component.html
+++ b/backend/ui/src/app/pages/sources-admin-page.component.html
@@ -28,6 +28,7 @@ Scraping Sources
Organization |
Country |
Classification mode |
+ Auto-publish |
Enabled |
Actions |
@@ -39,6 +40,11 @@ Scraping Sources
{{ s.organization_name || '—' }} |
{{ s.country || '—' }} |
{{ s.llm_fallback_enabled ? 'LLM' : 'Deterministic' }} |
+
+
+
+ |
@@ -52,7 +58,7 @@ Scraping Sources
|
- | No sources configured. |
+ No sources configured. |
@@ -126,6 +132,20 @@ {{ sourceModalTarget ? 'Edit Source' : 'Add Source' }}
Enabled
+
+
+
+
+
+ Auto-publish criteria
+ {{ criteriaMode(sourceForm) }} · {{ criteriaThreshold(sourceForm) }}
+
+