Feat/scraped content auto-publish mode

backend/content/migrations/0026_scrapingsource_auto_publish_enabled.py

@@ -0,0 +1,13 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [("content", "0025_contact_linkedin")]
+
+    operations = [
+        migrations.AddField(
+            model_name="scrapingsource",
+            name="auto_publish_enabled",
+            field=models.BooleanField(default=False),
+        ),
+    ]

backend/content/models.py

@@ -577,6 +577,7 @@ class ScrapingSource(models.Model):
 	crawl_max_pages        = models.IntegerField(default=25)
 	crawl_match_patterns   = models.JSONField(default=list)
 	crawl_exclude_patterns = models.JSONField(default=list)
+	auto_publish_enabled   = models.BooleanField(default=False)
 	created_at             = models.DateTimeField(auto_now_add=True)
 	updated_at             = models.DateTimeField(auto_now=True)
 

backend/content/scrapers/service.py

@@ -36,6 +36,9 @@
 
 LOGGER = logging.getLogger(__name__)
 
+AUTO_PUBLISH_LLM_MIN_CONFIDENCE = 0.80
+AUTO_PUBLISH_DETERMINISTIC_MIN_CONFIDENCE = 0.90
+
 
 def _ts() -> str:
     return timezone.now().isoformat().replace("+00:00", "Z")
@@ -561,6 +564,11 @@ def _upsert_offer(
             organization=organization,
             offer_type=offer_type,
         ).first()
+        auto_publish_threshold = self._auto_publish_threshold(source)
+        auto_publish_allowed = self._should_auto_publish(source, extracted, offer_type)
+        auto_publish_result = (
+            "published" if auto_publish_allowed else "draft"
+        ) if existing is None else "preserved_existing_status"
 
         current_timestamp = timezone.now().isoformat()
 
@@ -572,6 +580,10 @@ def _upsert_offer(
             "confidence": extracted.confidence,
             "last_seen_at": current_timestamp,
             "stale_candidate": False,
+            "auto_publish_enabled": source.auto_publish_enabled,
+            "auto_publish_threshold": auto_publish_threshold,
+            "auto_publish_result": auto_publish_result,
+            "auto_publish_mode": "llm" if source.llm_fallback_enabled else "deterministic",
         }
 
         merged_details = {
@@ -583,6 +595,7 @@ def _upsert_offer(
             if self.dry_run:
                 return "created", natural_key, None
 
+            offer_status = Offer.OfferStatus.PUBLISHED if auto_publish_allowed else Offer.OfferStatus.DRAFT
             offer = Offer.objects.create(
                 title=extracted.title,
                 summary=extracted.summary,
@@ -592,7 +605,7 @@ def _upsert_offer(
                 source_type=source_type,
                 target_profile=target_profile,
                 organization=organization,
-                status=Offer.OfferStatus.DRAFT,
+                status=offer_status,
                 created_by=ingestion_user,
                 updated_by=ingestion_user,
                 offer_type=offer_type,
@@ -645,6 +658,36 @@ def _upsert_offer(
         self._replace_domains(existing, source.domain_names)
         return "updated", natural_key, existing
 
+    @staticmethod
+    def _auto_publish_threshold(source: SourceDefinition) -> float:
+        return (
+            AUTO_PUBLISH_LLM_MIN_CONFIDENCE
+            if source.llm_fallback_enabled
+            else AUTO_PUBLISH_DETERMINISTIC_MIN_CONFIDENCE
+        )
+
+    def _should_auto_publish(
+        self,
+        source: SourceDefinition,
+        extracted: ExtractedPayload,
+        offer_type: OfferType | None,
+    ) -> bool:
+        if not source.auto_publish_enabled:
+            return False
+        if source.llm_fallback_enabled and extracted.method != "llm_primary":
+            return False
+        if extracted.confidence < self._auto_publish_threshold(source):
+            return False
+        if offer_type is None:
+            return False
+        if not extracted.title or not extracted.summary:
+            return False
+        if is_generic_page(extracted.title):
+            return False
+        if not source.llm_fallback_enabled and extracted.summary.startswith("Auto-extracted from"):
+            return False
+        return True
+
     def _replace_domains(self, offer: Offer, domain_names: list[str]) -> None:
         domain_map = {
             domain.name: domain
@@ -771,7 +814,16 @@ def _normalized_details_for_compare(details: dict | None) -> dict:
         normalized = deepcopy(details)
         scraping = normalized.get("scraping")
         if isinstance(scraping, dict):
-            for key in ("last_seen_at", "stale_candidate", "stale_marked_at", "stale_reason"):
+            for key in (
+                "last_seen_at",
+                "stale_candidate",
+                "stale_marked_at",
+                "stale_reason",
+                "auto_publish_enabled",
+                "auto_publish_threshold",
+                "auto_publish_result",
+                "auto_publish_mode",
+            ):
                 scraping.pop(key, None)
             normalized["scraping"] = scraping
         return normalized

backend/content/scrapers/source_registry.py

@@ -191,6 +191,7 @@ def get_sources(source_keys: list[str] | None = None) -> list[SourceDefinition]:
             crawl_max_pages=s.crawl_max_pages,
             crawl_match_patterns=s.crawl_match_patterns,
             crawl_exclude_patterns=s.crawl_exclude_patterns,
+            auto_publish_enabled=s.auto_publish_enabled,
         )
         for s in qs
     ]

backend/content/scrapers/types.py

@@ -20,6 +20,7 @@ class SourceDefinition:
     crawl_max_pages: int = 25
     crawl_match_patterns: list[str] = field(default_factory=list)
     crawl_exclude_patterns: list[str] = field(default_factory=list)
+    auto_publish_enabled: bool = False
 
 
 @dataclass

backend/content/test_scraper_service.py

@@ -1,4 +1,5 @@
 import uuid
+from dataclasses import replace
 from unittest.mock import Mock, patch
 
 import requests
@@ -18,7 +19,7 @@
 import ollama
 from content.scrapers.extractors import extract_links_from_html
 from content.scrapers.ollama_client import OllamaClient
-from content.scrapers.service import run_scrape
+from content.scrapers.service import run_scrape, ScrapeService
 from content.scrapers.types import ExtractedPayload, SourceDefinition
 from content.seeding import uuid_from_token
 
@@ -72,6 +73,210 @@ def _create_offer(self, details: dict) -> Offer:
         OfferDomain.objects.create(offer=offer, domain=self.domain)
         return offer
 
+    def _upsert_extracted(
+        self,
+        source: SourceDefinition,
+        extracted: ExtractedPayload,
+    ) -> Offer:
+        service = ScrapeService(use_llm_fallback=False)
+        action, _, offer = service._upsert_offer(
+            source,
+            self.source_type_scraping,
+            self.user,
+            extracted,
+        )
+        self.assertEqual(action, "created")
+        self.assertIsNotNone(offer)
+        return offer
+
+    def test_auto_publish_disabled_creates_draft(self):
+        source = replace(self.source, auto_publish_enabled=False)
+        offer = self._upsert_extracted(
+            source,
+            ExtractedPayload(
+                offer_type="training",
+                title="Research Training Programme",
+                summary="A structured opportunity for students.",
+                details={},
+                confidence=0.95,
+                method="llm_primary",
+            ),
+        )
+
+        self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+        self.assertFalse(offer.details["scraping"]["auto_publish_enabled"])
+        self.assertEqual(offer.details["scraping"]["auto_publish_result"], "draft")
+
+    def test_llm_auto_publish_above_threshold_creates_published_offer(self):
+        source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=True)
+        offer = self._upsert_extracted(
+            source,
+            ExtractedPayload(
+                offer_type="training",
+                title="Research Training Programme",
+                summary="A structured opportunity for students.",
+                details={},
+                confidence=0.85,
+                method="llm_primary",
+            ),
+        )
+
+        self.assertEqual(offer.status, Offer.OfferStatus.PUBLISHED)
+        self.assertEqual(offer.details["scraping"]["auto_publish_mode"], "llm")
+        self.assertEqual(offer.details["scraping"]["auto_publish_threshold"], 0.80)
+        self.assertEqual(offer.details["scraping"]["auto_publish_result"], "published")
+
+    def test_llm_auto_publish_below_threshold_creates_draft(self):
+        source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=True)
+        offer = self._upsert_extracted(
+            source,
+            ExtractedPayload(
+                offer_type="training",
+                title="Research Training Programme",
+                summary="A structured opportunity for students.",
+                details={},
+                confidence=0.79,
+                method="llm_primary",
+            ),
+        )
+
+        self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+        self.assertEqual(offer.details["scraping"]["auto_publish_result"], "draft")
+
+    def test_llm_auto_publish_requires_llm_primary_payload(self):
+        source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=True)
+        offer = self._upsert_extracted(
+            source,
+            ExtractedPayload(
+                offer_type="training",
+                title="Research Training Programme",
+                summary="A structured opportunity for students.",
+                details={},
+                confidence=0.95,
+                method="deterministic",
+            ),
+        )
+
+        self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+        self.assertEqual(offer.details["scraping"]["auto_publish_result"], "draft")
+
+    def test_deterministic_auto_publish_above_threshold_creates_published_offer(self):
+        source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=False)
+        offer = self._upsert_extracted(
+            source,
+            ExtractedPayload(
+                offer_type="training",
+                title="Research Training Programme",
+                summary="A structured opportunity for students.",
+                details={},
+                confidence=0.92,
+                method="deterministic",
+            ),
+        )
+
+        self.assertEqual(offer.status, Offer.OfferStatus.PUBLISHED)
+        self.assertEqual(offer.details["scraping"]["auto_publish_mode"], "deterministic")
+        self.assertEqual(offer.details["scraping"]["auto_publish_threshold"], 0.90)
+        self.assertEqual(offer.details["scraping"]["auto_publish_result"], "published")
+
+    def test_deterministic_auto_publish_below_threshold_creates_draft(self):
+        source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=False)
+        offer = self._upsert_extracted(
+            source,
+            ExtractedPayload(
+                offer_type="training",
+                title="Research Training Programme",
+                summary="A structured opportunity for students.",
+                details={},
+                confidence=0.89,
+                method="deterministic",
+            ),
+        )
+
+        self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+        self.assertEqual(offer.details["scraping"]["auto_publish_result"], "draft")
+
+    def test_deterministic_auto_publish_fallback_summary_creates_draft(self):
+        source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=False)
+        offer = self._upsert_extracted(
+            source,
+            ExtractedPayload(
+                offer_type="training",
+                title="Research Training Programme",
+                summary="Auto-extracted from https://example.edu/test-source",
+                details={},
+                confidence=0.95,
+                method="deterministic",
+            ),
+        )
+
+        self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+        self.assertEqual(offer.details["scraping"]["auto_publish_result"], "draft")
+
+    def test_auto_publish_generic_title_creates_draft(self):
+        source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=True)
+        offer = self._upsert_extracted(
+            source,
+            ExtractedPayload(
+                offer_type="training",
+                title="Contact Us",
+                summary="A structured opportunity for students.",
+                details={},
+                confidence=0.95,
+                method="llm_primary",
+            ),
+        )
+
+        self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+        self.assertEqual(offer.details["scraping"]["auto_publish_result"], "draft")
+
+    def test_auto_publish_missing_summary_creates_draft(self):
+        source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=True)
+        offer = self._upsert_extracted(
+            source,
+            ExtractedPayload(
+                offer_type="training",
+                title="Research Training Programme",
+                summary="",
+                details={},
+                confidence=0.95,
+                method="llm_primary",
+            ),
+        )
+
+        self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+        self.assertEqual(offer.details["scraping"]["auto_publish_result"], "draft")
+
+    def test_auto_publish_does_not_publish_existing_draft(self):
+        self._create_offer(
+            {
+                "source_name": "Test Source",
+                "scraping": {
+                    "source_key": "test_source",
+                    "stale_candidate": False,
+                },
+            }
+        )
+        source = replace(self.source, auto_publish_enabled=True, llm_fallback_enabled=True)
+        service = ScrapeService(use_llm_fallback=False)
+        action, _, offer = service._upsert_offer(
+            source,
+            self.source_type_scraping,
+            self.user,
+            ExtractedPayload(
+                offer_type="training",
+                title="Updated Training Programme",
+                summary="A structured opportunity for students.",
+                details={},
+                confidence=0.95,
+                method="llm_primary",
+            ),
+        )
+
+        self.assertEqual(action, "updated")
+        self.assertEqual(offer.status, Offer.OfferStatus.DRAFT)
+        self.assertEqual(offer.details["scraping"]["auto_publish_result"], "preserved_existing_status")
+
     @patch("content.scrapers.service.get_sources")
     @patch("content.scrapers.service.requests.get")
     def test_http_404_marks_run_failed_and_deletes_invalid_source_offers(self, mock_get, mock_get_sources):