Basic Kafka Implementation

configs/dev.yaml

@@ -11,3 +11,11 @@ S3_ACCESS_KEY: "dev-access"
 S3_SECRET_KEY: "5ece9238b18dc29d5e0724b3ae577cdf90b8f78b5eaf62ca"
 S3_BUCKET_NAME: "dev"
 S3_VERIFY: "true"
+
+KAFKA_BROKERS: "localhost:9092"
+KAFKA_REQUEST_TOPIC: "moderation-request"
+KAFKA_RESULT_TOPIC: "moderation-result"
+KAFKA_CONSUMER_GROUP_ID: "moderation-consumer-group"
+KAFKA_MAX_RETRIES: "3"
+KAFKA_RETRY_BACKOFF_MS: "1000"
+KAFKA_ENABLE_SSL: false

justfile

@@ -10,6 +10,10 @@ TMP_DIR := '.proto'
 SOURCE := 'modservice'
 OUT_DIR := 'src/' + SOURCE + '/grpc'
 
+MODERATION_PROTO_TAG := 'v0.1.3'
+MODERATION_PROTO_NAME := 'moderation.proto'
+MODERATION_TMP_DIR := '.proto_moderation'
+
 MKDIR_TOOLS := 'mkdir -p tools'
 
 FETCH_COMMON_JUST := 'curl -fsSL ' + COMMON_JUST_URL + ' -o tools/common.just'

migrations/20260308000000_add_on_moderation_status.sql

@@ -0,0 +1,22 @@
+-- +goose Up
+-- +goose NO TRANSACTION
+
+-- Add ON_MODERATION to existing mod_status enum
+ALTER TYPE mod_status ADD VALUE IF NOT EXISTS 'ON_MODERATION';
+
+-- +goose StatementBegin
+-- Update existing UPLOADING mods to ON_MODERATION
+UPDATE mods SET status = 'ON_MODERATION' WHERE status = 'UPLOADING';
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+
+-- Revert ON_MODERATION mods back to UPLOADING
+UPDATE mods SET status = 'UPLOADING' WHERE status = 'ON_MODERATION';
+
+-- Note: PostgreSQL does not support removing enum values directly
+-- If rollback is needed, the enum type would need to be recreated
+-- For now, we just revert the data changes
+
+-- +goose StatementEnd

pyproject.toml

@@ -124,6 +124,8 @@ dependencies = [
     "aioboto3>=13.0.0",
     "python-dotenv>=1.0.0",
     "asyncpg>=0.30.0",
+    "confluent-kafka>=2.12.2",
+    "protobuf==6.32.1",
 ]
 requires-python = ">=3.13"
 readme = "README.md"
@@ -144,6 +146,7 @@ dev = [
     "pytest-mock==3.15.1",
     "pytest-faker==2.0.0",
     "Faker==37.11.0",
+    "hypothesis>=6.151.9",
     "black==25.9.0",
     "isort==6.0.1",
     "flake8==7.3.0",

src/modservice/constants.py

@@ -1,4 +1,5 @@
 STATUS_UPLOADING = "UPLOADING"
+STATUS_ON_MODERATION = "ON_MODERATION"
 STATUS_UPLOADED = "UPLOADED"
 STATUS_BANNED = "BANNED"
 STATUS_HIDDEN = "HIDDEN"

src/modservice/grpc/__init__.py

@@ -0,0 +1 @@
+# Generated gRPC files

src/modservice/grpc/moderation_pb2.pyi

@@ -0,0 +1,33 @@
+from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
+
+DESCRIPTOR: _descriptor.FileDescriptor
+
+class ObjectType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    OBJECT_TYPE_UNSPECIFIED: _ClassVar[ObjectType]
+    OBJECT_TYPE_MOD_DESCRIPTION: _ClassVar[ObjectType]
+    OBJECT_TYPE_COMMENT_TEXT: _ClassVar[ObjectType]
+    OBJECT_TYPE_USER_NAME: _ClassVar[ObjectType]
+OBJECT_TYPE_UNSPECIFIED: ObjectType
+OBJECT_TYPE_MOD_DESCRIPTION: ObjectType
+OBJECT_TYPE_COMMENT_TEXT: ObjectType
+OBJECT_TYPE_USER_NAME: ObjectType
+
+class ModerateObjectRequest(_message.Message):
+    __slots__ = ("id", "type", "text")
+    ID_FIELD_NUMBER: _ClassVar[int]
+    TYPE_FIELD_NUMBER: _ClassVar[int]
+    TEXT_FIELD_NUMBER: _ClassVar[int]
+    id: int
+    type: ObjectType
+    text: str
+    def __init__(self, id: _Optional[int] = ..., type: _Optional[_Union[ObjectType, str]] = ..., text: _Optional[str] = ...) -> None: ...
+
+class ModerateObjectResponse(_message.Message):
+    __slots__ = ("success",)
+    SUCCESS_FIELD_NUMBER: _ClassVar[int]
+    success: bool
+    def __init__(self, success: bool = ...) -> None: ...

src/modservice/grpc/moderation_pb2_grpc.py

@@ -0,0 +1,97 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+import warnings
+
+from . import moderation_pb2 as moderation__pb2
+
+GRPC_GENERATED_VERSION = '1.75.1'
+GRPC_VERSION = grpc.__version__
+_version_not_supported = False
+
+try:
+    from grpc._utilities import first_version_is_lower
+    _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION)
+except ImportError:
+    _version_not_supported = True
+
+if _version_not_supported:
+    raise RuntimeError(
+        f'The grpc package installed is at version {GRPC_VERSION},'
+        + f' but the generated code in moderation_pb2_grpc.py depends on'
+        + f' grpcio>={GRPC_GENERATED_VERSION}.'
+        + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}'
+        + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.'
+    )
+
+
+class ModerationServiceStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.ModerateObject = channel.unary_unary(
+                '/moderation.ModerationService/ModerateObject',
+                request_serializer=moderation__pb2.ModerateObjectRequest.SerializeToString,
+                response_deserializer=moderation__pb2.ModerateObjectResponse.FromString,
+                _registered_method=True)
+
+
+class ModerationServiceServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def ModerateObject(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_ModerationServiceServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'ModerateObject': grpc.unary_unary_rpc_method_handler(
+                    servicer.ModerateObject,
+                    request_deserializer=moderation__pb2.ModerateObjectRequest.FromString,
+                    response_serializer=moderation__pb2.ModerateObjectResponse.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'moderation.ModerationService', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+    server.add_registered_method_handlers('moderation.ModerationService', rpc_method_handlers)
+
+
+ # This class is part of an EXPERIMENTAL API.
+class ModerationService(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def ModerateObject(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            '/moderation.ModerationService/ModerateObject',
+            moderation__pb2.ModerateObjectRequest.SerializeToString,
+            moderation__pb2.ModerateObjectResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)

src/modservice/handler/get_mods.py

@@ -1,10 +1,11 @@
 import grpc
 
-from modservice.constants import STATUS_BANNED, STATUS_HIDDEN, STATUS_UPLOADED
+from modservice.constants import STATUS_BANNED, STATUS_HIDDEN, STATUS_ON_MODERATION, STATUS_UPLOADED
 from modservice.grpc import mod_pb2
 from modservice.service.service import ModService
 
 STATUS_TO_PROTO: dict[str, mod_pb2.ModStatus] = {
+    STATUS_ON_MODERATION: mod_pb2.ModStatus.MOD_STATUS_UNSPECIFIED,
     STATUS_UPLOADED: mod_pb2.ModStatus.MOD_STATUS_UPLOADED,
     STATUS_BANNED: mod_pb2.ModStatus.MOD_STATUS_BANNED,
     STATUS_HIDDEN: mod_pb2.ModStatus.MOD_STATUS_HIDDEN,

src/modservice/kafka/__init__.py

@@ -0,0 +1,6 @@
+"""Kafka components for moderation integration."""
+
+from modservice.kafka.consumer import KafkaConsumer
+from modservice.kafka.producer import KafkaProducer
+
+__all__ = ["KafkaConsumer", "KafkaProducer"]

src/modservice/kafka/config.py

@@ -0,0 +1,37 @@
+import os
+
+
+class KafkaConfig:
+
+    def __init__(self):
+        self.brokers = os.getenv('KAFKA_BROKERS', 'localhost:9092')
+        self.request_topic = os.getenv('KAFKA_REQUEST_TOPIC', 'moderation-request')
+        self.result_topic = os.getenv('KAFKA_RESULT', 'moderation-result')
+        self.consumer_group_id = os.getenv('KAFKA_CONSUMER_GROUP_ID', 'mod-service-group')
+        self.max_retries = int(os.getenv('KAFKA_MAX_RETRIES', '3'))
+        self.retry_backoff_ms = int(os.getenv('KAFKA_RETRY_BACKOFF_MS', '1000'))
+        self.enable_ssl = os.getenv('KAFKA_ENABLE_SSL', 'false').lower() == 'true'
+
+    def get_producer_config(self):
+        config = {
+            'bootstrap.servers': self.brokers,
+            'client.id': 'mod_service_producer',
+            'acks': 'all',
+            'retries': self.max_retries,
+            'retry.backoff.ms': self.retry_backoff_ms,
+            'enable.idempotence': True,
+            'compression.type': 'none'
+        }
+        return config
+
+    def get_consumer_config(self):
+        config = {
+            'bootstrap.servers': self.brokers,
+            'group.id': self.consumer_group_id,
+            'auto.offset.reset': 'earliest',
+            'enable.auto.commit': True,
+            'auto.commit.interval.ms': 1000,
+            'session.timeout.ms': 30000,
+            'heartbeat.interval.ms': 10000
+        }
+        return config