From 5178cd29b07123d9c064e7c35fd5bdcec63b8ebc Mon Sep 17 00:00:00 2001 From: pushiscool Date: Sat, 20 Jun 2026 16:01:11 -0400 Subject: [PATCH 1/2] Ignore moderation evidence screenshots --- src/cogs/scam_image_detector.py | 24 ++++++++++++++--- tests/test_scam_image_detector.py | 43 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/cogs/scam_image_detector.py b/src/cogs/scam_image_detector.py index e5dc450..9d801a0 100644 --- a/src/cogs/scam_image_detector.py +++ b/src/cogs/scam_image_detector.py @@ -104,7 +104,6 @@ "free", "limited time", "congratulations", - "selected", "airdrop", "giving away", "give away", @@ -137,7 +136,6 @@ "visit", "page", "continue", - "follow", "subscribe", ) PARODY_TERMS = ( @@ -163,6 +161,21 @@ "shares", "ytd", ) +MODERATION_EVIDENCE_TERMS = ( + "possible unsafe message detected", + "forwarded for human review", + "message deleted", + "jump to message", + "report summary", + "selected message", + "report category", + "submit report", +) +SELECTED_WINNER_RE = re.compile( + r"\b(?:you(?:'ve| have)?(?: been)?|you are|you were)\s+selected\b" + r"|\bselected\s+(?:winner|for\s+(?:a\s+)?(?:prize|reward|giveaway))\b", + re.IGNORECASE, +) NSFW_CLASSES = { "ANUS_EXPOSED", "BUTTOCKS_EXPOSED", @@ -239,7 +252,7 @@ def assess_scam_text( has_brand = contains_any(normalized, BRAND_TERMS) has_money = bool(MONEY_RE.search(normalized)) has_hook = bool(HOOK_RE.search(normalized)) - has_reward_language = contains_any(normalized, GIVEAWAY_TERMS) + has_reward_language = contains_any(normalized, GIVEAWAY_TERMS) or bool(SELECTED_WINNER_RE.search(normalized)) has_giveaway = has_reward_language or has_money or has_hook has_strong_action = contains_any(normalized, STRONG_ACTION_TERMS) has_weak_action = contains_any(normalized, WEAK_ACTION_TERMS) @@ -320,6 +333,11 @@ def assess_scam_text( score = min(score, REVIEW_THRESHOLD - 1) reasons.append("looks like stock market information without a solicitation") + moderation_evidence_signals = sum(contains_term(normalized, term) for term in MODERATION_EVIDENCE_TERMS) + if moderation_evidence_signals >= 2 and not nsfw_detections: + score = min(score, REVIEW_THRESHOLD - 1) + reasons.append("looks like a screenshot of moderation or report evidence") + score = min(score, 100) if score < REVIEW_THRESHOLD and not reasons: reasons.append("no strong scam signals found") diff --git a/tests/test_scam_image_detector.py b/tests/test_scam_image_detector.py index 97ec5cd..44a6317 100644 --- a/tests/test_scam_image_detector.py +++ b/tests/test_scam_image_detector.py @@ -99,6 +99,49 @@ def test_stock_price_screenshot_is_not_flagged(): assert "asks users to claim, verify, log in, scan, or enter sensitive info" not in assessment.reasons +def test_existing_moderation_alert_screenshot_is_not_flagged_again(): + assessment = assess_scam_text( + """ + Possible Unsafe Message Detected + This was forwarded for human review. The bot did not delete the original message. + Message Deleted + Jump to message + MrBeast giveaway claim your free prize and verify now + """, + has_visual_attachment=True, + ) + + assert assessment.should_alert is False + assert assessment.score < ALERT_THRESHOLD + + +def test_discord_report_summary_screenshot_is_not_flagged(): + assessment = assess_scam_text( + """ + Report Summary + Review your report before submitting + Selected Message im only 10 + Report Category Something else + This person is too young to use Discord + Please follow our Community Guidelines + Submit Report + """, + has_visual_attachment=True, + ) + + assert assessment.should_alert is False + assert assessment.score < ALERT_THRESHOLD + + +def test_selected_winner_scam_still_alerts(): + assessment = assess_scam_text( + "You have been selected for a free Nitro prize. Claim it now.", + has_visual_attachment=True, + ) + + assert assessment.should_alert is True + + def test_mrbeast_free_money_page_ad_alerts_without_url(): assessment = assess_scam_text( "Ad MRBEAST everyone that visits our page gets $500 You're eligible for free $500", From 5c3d765faf9b8c7dfa65aec9a75d27220f7c26a6 Mon Sep 17 00:00:00 2001 From: pushiscool Date: Sat, 20 Jun 2026 16:06:37 -0400 Subject: [PATCH 2/2] Skip trusted moderation roles --- src/cogs/scam_image_detector.py | 20 +++++++++++++++- tests/test_scam_image_detector.py | 38 +++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/cogs/scam_image_detector.py b/src/cogs/scam_image_detector.py index 9d801a0..b870198 100644 --- a/src/cogs/scam_image_detector.py +++ b/src/cogs/scam_image_detector.py @@ -21,6 +21,16 @@ DEFAULT_ENABLED_BOT_IDS = {1508281890820460604, 1464966749643341847} REVIEW_CHANNEL_ID = 1517350483646484480 +MODERATION_BYPASS_ROLE_IDS = { + 1259554509559169159, + 587392891220131860, + 182222541857751040, + 1201290738953093240, + 1207394456584847461, + 299005509065900045, + 814369761030701058, + 939015562619678781, +} IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp"} VIDEO_EXTENSIONS = {".mp4", ".mov", ".webm", ".m4v", ".mkv"} MAX_SCAN_BYTES = 8 * 1024 * 1024 @@ -221,6 +231,10 @@ def contains_any(text: str, terms: Iterable[str]) -> bool: return any(contains_term(text, term) for term in terms) +def has_moderation_bypass_role(member) -> bool: + return any(getattr(role, "id", None) in MODERATION_BYPASS_ROLE_IDS for role in getattr(member, "roles", [])) + + def score_to_level(score: int) -> str: if score >= ALERT_THRESHOLD: return HIGH_RISK @@ -838,7 +852,11 @@ async def on_message(self, message: nextcord.Message) -> None: if getattr(message, "guild", None) is None: return - if getattr(getattr(message, "author", None), "bot", False): + author = getattr(message, "author", None) + if getattr(author, "bot", False): + return + + if has_moderation_bypass_role(author): return source_channel_id = getattr(getattr(message, "channel", None), "id", None) diff --git a/tests/test_scam_image_detector.py b/tests/test_scam_image_detector.py index 44a6317..9a54d9f 100644 --- a/tests/test_scam_image_detector.py +++ b/tests/test_scam_image_detector.py @@ -5,10 +5,12 @@ from cogs.scam_image_detector import ( ALERT_THRESHOLD, DEFAULT_ENABLED_BOT_IDS, + MODERATION_BYPASS_ROLE_IDS, REVIEW_CHANNEL_ID, REVIEW_THRESHOLD, ScamImageDetector, assess_scam_text, + has_moderation_bypass_role, is_visual_attachment, is_visual_url, is_video_attachment, @@ -194,6 +196,42 @@ def test_detector_can_be_forced_on_for_standalone_safety_bot(): assert ScamImageDetector(bot).is_enabled_for_current_bot() +def test_moderation_bypass_roles_match_all_requested_ids(): + assert MODERATION_BYPASS_ROLE_IDS == { + 1259554509559169159, + 587392891220131860, + 182222541857751040, + 1201290738953093240, + 1207394456584847461, + 299005509065900045, + 814369761030701058, + 939015562619678781, + } + member = SimpleNamespace(roles=[SimpleNamespace(id=1259554509559169159)]) + assert has_moderation_bypass_role(member) is True + + +def test_moderation_bypass_role_skips_scanning_and_forwarding(): + review_channel = SimpleNamespace(id=REVIEW_CHANNEL_ID, send=AsyncMock()) + bot = SimpleNamespace(user=SimpleNamespace(id=next(iter(DEFAULT_ENABLED_BOT_IDS)))) + cog = ScamImageDetector(bot) + cog.assess_message = AsyncMock() + message = SimpleNamespace( + guild=SimpleNamespace(id=1), + author=SimpleNamespace( + id=111, + bot=False, + roles=[SimpleNamespace(id=1259554509559169159)], + ), + channel=SimpleNamespace(id=123), + ) + + asyncio.run(cog.on_message(message)) + + cog.assess_message.assert_not_awaited() + review_channel.send.assert_not_awaited() + + def test_on_message_forwards_alert_without_ping_or_deletion(): review_channel = SimpleNamespace(id=REVIEW_CHANNEL_ID, send=AsyncMock()) bot = SimpleNamespace(