scam detector fix

src/cogs/scam_image_detector.py

@@ -21,6 +21,16 @@
 
 DEFAULT_ENABLED_BOT_IDS = {1508281890820460604, 1464966749643341847}
 REVIEW_CHANNEL_ID = 1517350483646484480
+MODERATION_BYPASS_ROLE_IDS = {
+    1259554509559169159,
+    587392891220131860,
+    182222541857751040,
+    1201290738953093240,
+    1207394456584847461,
+    299005509065900045,
+    814369761030701058,
+    939015562619678781,
+}
 IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp"}
 VIDEO_EXTENSIONS = {".mp4", ".mov", ".webm", ".m4v", ".mkv"}
 MAX_SCAN_BYTES = 8 * 1024 * 1024
@@ -104,7 +114,6 @@
     "free",
     "limited time",
     "congratulations",
-    "selected",
     "airdrop",
     "giving away",
     "give away",
@@ -137,7 +146,6 @@
     "visit",
     "page",
     "continue",
-    "follow",
     "subscribe",
 )
 PARODY_TERMS = (
@@ -163,6 +171,21 @@
     "shares",
     "ytd",
 )
+MODERATION_EVIDENCE_TERMS = (
+    "possible unsafe message detected",
+    "forwarded for human review",
+    "message deleted",
+    "jump to message",
+    "report summary",
+    "selected message",
+    "report category",
+    "submit report",
+)
+SELECTED_WINNER_RE = re.compile(
+    r"\b(?:you(?:'ve| have)?(?: been)?|you are|you were)\s+selected\b"
+    r"|\bselected\s+(?:winner|for\s+(?:a\s+)?(?:prize|reward|giveaway))\b",
+    re.IGNORECASE,
+)
 NSFW_CLASSES = {
     "ANUS_EXPOSED",
     "BUTTOCKS_EXPOSED",
@@ -208,6 +231,10 @@ def contains_any(text: str, terms: Iterable[str]) -> bool:
     return any(contains_term(text, term) for term in terms)
 
 
+def has_moderation_bypass_role(member) -> bool:
+    return any(getattr(role, "id", None) in MODERATION_BYPASS_ROLE_IDS for role in getattr(member, "roles", []))
+
+
 def score_to_level(score: int) -> str:
     if score >= ALERT_THRESHOLD:
         return HIGH_RISK
@@ -239,7 +266,7 @@ def assess_scam_text(
     has_brand = contains_any(normalized, BRAND_TERMS)
     has_money = bool(MONEY_RE.search(normalized))
     has_hook = bool(HOOK_RE.search(normalized))
-    has_reward_language = contains_any(normalized, GIVEAWAY_TERMS)
+    has_reward_language = contains_any(normalized, GIVEAWAY_TERMS) or bool(SELECTED_WINNER_RE.search(normalized))
     has_giveaway = has_reward_language or has_money or has_hook
     has_strong_action = contains_any(normalized, STRONG_ACTION_TERMS)
     has_weak_action = contains_any(normalized, WEAK_ACTION_TERMS)
@@ -320,6 +347,11 @@ def assess_scam_text(
         score = min(score, REVIEW_THRESHOLD - 1)
         reasons.append("looks like stock market information without a solicitation")
 
+    moderation_evidence_signals = sum(contains_term(normalized, term) for term in MODERATION_EVIDENCE_TERMS)
+    if moderation_evidence_signals >= 2 and not nsfw_detections:
+        score = min(score, REVIEW_THRESHOLD - 1)
+        reasons.append("looks like a screenshot of moderation or report evidence")
+
     score = min(score, 100)
     if score < REVIEW_THRESHOLD and not reasons:
         reasons.append("no strong scam signals found")
@@ -820,7 +852,11 @@ async def on_message(self, message: nextcord.Message) -> None:
         if getattr(message, "guild", None) is None:
             return
 
-        if getattr(getattr(message, "author", None), "bot", False):
+        author = getattr(message, "author", None)
+        if getattr(author, "bot", False):
+            return
+
+        if has_moderation_bypass_role(author):
             return
 
         source_channel_id = getattr(getattr(message, "channel", None), "id", None)

tests/test_scam_image_detector.py

@@ -5,10 +5,12 @@
 from cogs.scam_image_detector import (
     ALERT_THRESHOLD,
     DEFAULT_ENABLED_BOT_IDS,
+    MODERATION_BYPASS_ROLE_IDS,
     REVIEW_CHANNEL_ID,
     REVIEW_THRESHOLD,
     ScamImageDetector,
     assess_scam_text,
+    has_moderation_bypass_role,
     is_visual_attachment,
     is_visual_url,
     is_video_attachment,
@@ -99,6 +101,49 @@ def test_stock_price_screenshot_is_not_flagged():
     assert "asks users to claim, verify, log in, scan, or enter sensitive info" not in assessment.reasons
 
 
+def test_existing_moderation_alert_screenshot_is_not_flagged_again():
+    assessment = assess_scam_text(
+        """
+        Possible Unsafe Message Detected
+        This was forwarded for human review. The bot did not delete the original message.
+        Message Deleted
+        Jump to message
+        MrBeast giveaway claim your free prize and verify now
+        """,
+        has_visual_attachment=True,
+    )
+
+    assert assessment.should_alert is False
+    assert assessment.score < ALERT_THRESHOLD
+
+
+def test_discord_report_summary_screenshot_is_not_flagged():
+    assessment = assess_scam_text(
+        """
+        Report Summary
+        Review your report before submitting
+        Selected Message im only 10
+        Report Category Something else
+        This person is too young to use Discord
+        Please follow our Community Guidelines
+        Submit Report
+        """,
+        has_visual_attachment=True,
+    )
+
+    assert assessment.should_alert is False
+    assert assessment.score < ALERT_THRESHOLD
+
+
+def test_selected_winner_scam_still_alerts():
+    assessment = assess_scam_text(
+        "You have been selected for a free Nitro prize. Claim it now.",
+        has_visual_attachment=True,
+    )
+
+    assert assessment.should_alert is True
+
+
 def test_mrbeast_free_money_page_ad_alerts_without_url():
     assessment = assess_scam_text(
         "Ad MRBEAST everyone that visits our page gets $500 You're eligible for free $500",
@@ -151,6 +196,42 @@ def test_detector_can_be_forced_on_for_standalone_safety_bot():
     assert ScamImageDetector(bot).is_enabled_for_current_bot()
 
 
+def test_moderation_bypass_roles_match_all_requested_ids():
+    assert MODERATION_BYPASS_ROLE_IDS == {
+        1259554509559169159,
+        587392891220131860,
+        182222541857751040,
+        1201290738953093240,
+        1207394456584847461,
+        299005509065900045,
+        814369761030701058,
+        939015562619678781,
+    }
+    member = SimpleNamespace(roles=[SimpleNamespace(id=1259554509559169159)])
+    assert has_moderation_bypass_role(member) is True
+
+
+def test_moderation_bypass_role_skips_scanning_and_forwarding():
+    review_channel = SimpleNamespace(id=REVIEW_CHANNEL_ID, send=AsyncMock())
+    bot = SimpleNamespace(user=SimpleNamespace(id=next(iter(DEFAULT_ENABLED_BOT_IDS))))
+    cog = ScamImageDetector(bot)
+    cog.assess_message = AsyncMock()
+    message = SimpleNamespace(
+        guild=SimpleNamespace(id=1),
+        author=SimpleNamespace(
+            id=111,
+            bot=False,
+            roles=[SimpleNamespace(id=1259554509559169159)],
+        ),
+        channel=SimpleNamespace(id=123),
+    )
+
+    asyncio.run(cog.on_message(message))
+
+    cog.assess_message.assert_not_awaited()
+    review_channel.send.assert_not_awaited()
+
+
 def test_on_message_forwards_alert_without_ping_or_deletion():
     review_channel = SimpleNamespace(id=REVIEW_CHANNEL_ID, send=AsyncMock())
     bot = SimpleNamespace(