Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 2 additions & 10 deletions .clang-tidy
Original file line number Diff line number Diff line change
@@ -1,12 +1,4 @@
Checks: >
clang-analyzer-*,
bugprone-*,
performance-*,
modernize-*,
readability-*,
-modernize-use-trailing-return-type

Checks: 'clang-analyzer-*,bugprone-*,performance-*,modernize-*,readability-*,-modernize-use-trailing-return-type,-bugprone-suspicious-include'
WarningsAsErrors: ''

HeaderFilterRegex: '^(src|include|tests|benchmark)/'
HeaderFilterRegex: '.*/moderation-service/(src|tests|benchmark|include)/.*'
FormatStyle: file
22 changes: 22 additions & 0 deletions .github/workflows/lint-and-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
name: "lint and test"

on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
types: [ opened, synchronize, reopened ]

permissions:
contents: read
pull-requests: write

jobs:
lint-and-test:
uses: esclient/tools/.github/workflows/lint-and-test-cpp.yml@main
with:
sonar-inclusions: "src/**,tests/**"
sonar-exclusions: "**/generated/**,**/grpc/**,tests/mocks/**,tests/unit/**"
sonar-coverage-exclusions: "tests/mocks/**,tests/unit/**,src/moderationservice/main.cpp,src/moderationservice/server/server.cpp"
secrets:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
25 changes: 16 additions & 9 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
cmake_minimum_required(VERSION 3.20)
project(moderation-service)
enable_testing()
set(CMAKE_CXX_STANDARD 17)

if(MSVC)
Expand All @@ -18,14 +19,21 @@ if(WIN32)
add_compile_definitions(U_STATIC_IMPLEMENTATION)
endif()

option(ENABLE_COVERAGE "Enable coverage reporting" OFF)
if(ENABLE_COVERAGE)
add_compile_options(--coverage -fprofile-arcs -ftest-coverage)
add_link_options(--coverage)
endif()

include(${CMAKE_BINARY_DIR}/conan_toolchain.cmake OPTIONAL)

find_package(Protobuf REQUIRED CONFIG)
find_package(protobuf REQUIRED CONFIG)
find_package(gRPC REQUIRED CONFIG)
find_package(ICU REQUIRED COMPONENTS uc i18n data)
find_package(RdKafka REQUIRED)
find_package(Threads REQUIRED)
find_package(libpqxx REQUIRED)
find_package(absl CONFIG REQUIRED)

set(CPM_DOWNLOAD_VERSION 0.40.2)
if(CPM_SOURCE_CACHE)
Expand Down Expand Up @@ -82,11 +90,9 @@ add_custom_command(
# Create library from generated sources
add_library(${GENPROTO_LIB} ${PROTO_SRCS} ${PROTO_HDRS} ${GRPC_SRCS} ${GRPC_HDRS})

target_include_directories(
${GENPROTO_LIB} PUBLIC ${GENPROTO_DIR} ${protobuf_INCLUDE_DIRS_RELEASE}
${grpc_INCLUDE_DIRS_RELEASE} ${icu_INCLUDE_DIRS_RELEASE})
target_include_directories(${GENPROTO_LIB} PUBLIC ${GENPROTO_DIR})

target_link_libraries(${GENPROTO_LIB} PUBLIC protobuf::libprotobuf gRPC::grpc++)
target_link_libraries(${GENPROTO_LIB} PUBLIC protobuf::libprotobuf gRPC::grpc++ abseil::abseil)

# link source files
add_executable(
Expand Down Expand Up @@ -117,7 +123,8 @@ target_link_libraries(
Threads::Threads
RdKafka::rdkafka
RdKafka::rdkafka++
libpqxx::pqxx)
libpqxx::pqxx
abseil::abseil)

if(WIN32)
target_compile_definitions(${PROJECT_NAME} PRIVATE U_STATIC_IMPLEMENTATION)
Expand Down Expand Up @@ -256,9 +263,9 @@ if(CPPCHECK_EXE
COMMAND ${CMAKE_COMMAND} -E echo "=== Running cppcheck ==="
COMMAND
${CPPCHECK_EXE} --enable=warning,style,performance,portability --inline-suppr
--suppress=*:${CMAKE_BINARY_DIR}/generated/* -I${PROJECT_SOURCE_DIR}/include
-I${PROJECT_SOURCE_DIR}/src/moderationservice --library=gnu
${PROJECT_SOURCE_DIR}/src/moderationservice ${PROJECT_SOURCE_DIR}/include
--suppress=*:${CMAKE_BINARY_DIR}/generated/* --suppress=*:${CMAKE_BINARY_DIR}/_deps/*
-I${PROJECT_SOURCE_DIR}/include -I${PROJECT_SOURCE_DIR}/src/moderationservice
--library=gnu ${PROJECT_SOURCE_DIR}/src/moderationservice ${PROJECT_SOURCE_DIR}/include
${PROJECT_SOURCE_DIR}/tests ${PROJECT_SOURCE_DIR}/benchmark
COMMAND ${CMAKE_COMMAND} -E echo ""
COMMAND ${CMAKE_COMMAND} -E echo "=== Running cpplint ==="
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
<img width="5573" height="6341" alt="Unbenannt-2025-10-16-0020(1)" src="https://github.com/user-attachments/assets/d9891188-500e-43e7-a8a3-6d11cbbd1d0e" />



To build and run you need the following installed:

-Chocolatey (Windows)
Expand All @@ -17,3 +16,4 @@ To build and run you need the following installed:
-sh/bash (Windows only)
-Internet connection (Optional)
-Electricity (Recommended)
-Antidepressants(Recommended)
50 changes: 20 additions & 30 deletions include/config/config.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,44 +18,38 @@ struct Config {
const char* host = std::getenv("HOST");
if (host == nullptr) {
throw std::runtime_error("HOST environment variable not set");
} else {
config.host = host;
}
config.host = host;

const char* port = std::getenv("PORT");
if (port == nullptr) {
throw std::runtime_error("PORT environment variable not set");
} else {
config.port = port;
}
config.port = port;

const char* database_url = std::getenv("DATABASE_URL");
if (database_url == nullptr) {
throw std::runtime_error("DATABASE_URL environment variable not set");
} else {
config.database_url = database_url;
}
config.database_url = database_url;

const char* log_level = std::getenv("LOG_LEVEL");
if (log_level == nullptr) {
throw std::runtime_error("LOG_LEVEL environment variable not set");
} else {
config.log_level = log_level;
}
config.log_level = log_level;

const char* log_format = std::getenv("LOG_FORMAT");
if (log_format == nullptr) {
throw std::runtime_error("LOG_FORMAT environment variable not set");
} else {
config.log_format = log_format;
}
config.log_format = log_format;

const char* log_datefmt = std::getenv("LOG_DATEFMT");
if (log_datefmt == nullptr) {
throw std::runtime_error("LOG_DATEFMT environment variable not set");
} else {
config.log_datefmt = log_datefmt;
}
config.log_datefmt = log_datefmt;

return config;
}
Expand All @@ -76,49 +70,45 @@ struct KafkaConfig {
const char* kafka_brokers = std::getenv("KAFKA_BROKERS");
if (kafka_brokers == nullptr) {
throw std::runtime_error("KAFKA_BROKERS environment variable not set");
} else {
kafkaConfig.brokers = kafka_brokers;
}
kafkaConfig.brokers = kafka_brokers;

const char* kafka_request_topic = std::getenv("KAFKA_REQUEST_TOPIC");
if (kafka_request_topic == nullptr) {
throw std::runtime_error("KAFKA_REQUEST_TOPIC environment variable not set");
} else {
kafkaConfig.request_topic = kafka_request_topic;
}
kafkaConfig.request_topic = kafka_request_topic;

const char* kafka_result_topic = std::getenv("KAFKA_RESULT_TOPIC");
if (kafka_result_topic == nullptr) {
throw std::runtime_error("KAFKA_RESULT_TOPIC environment variable not set");
} else {
kafkaConfig.result_topic = kafka_result_topic;
}
kafkaConfig.result_topic = kafka_result_topic;

const char* kafka_consumer_group_id = std::getenv("KAFKA_CONSUMER_GROUP_ID");
if (kafka_consumer_group_id == nullptr) {
throw std::runtime_error("KAFKA_CONSUMER_GROUP_ID environment variable not set");
} else {
kafkaConfig.consumer_group_id = kafka_consumer_group_id;
}
kafkaConfig.consumer_group_id = kafka_consumer_group_id;

const char* kafka_max_retries = std::getenv("KAFKA_MAX_RETRIES");
if (kafka_max_retries == nullptr) {
throw std::runtime_error("KAFKA_MAX_RETRIES environment variable not set");
} else {
kafkaConfig.max_retries = std::stoi(kafka_max_retries);
}
kafkaConfig.max_retries = std::stoi(kafka_max_retries);

const char* kafka_retry_backoff_ms = std::getenv("KAFKA_RETRY_BACKOFF_MS");
if (kafka_retry_backoff_ms == nullptr) {
throw std::runtime_error("KAFKA_RETRY_BACKOFF_MS environment variable not set");
} else {
kafkaConfig.retry_backoff_ms = std::stoi(kafka_retry_backoff_ms);
}
kafkaConfig.retry_backoff_ms = std::stoi(kafka_retry_backoff_ms);

const char* kafka_enable_ssl = std::getenv("KAFKA_ENABLE_SSL");
if (kafka_enable_ssl == nullptr) {
throw std::runtime_error("KAFKA_ENABLE_SSL environment variable not set");
} else {
if (std::string(kafka_enable_ssl) == "true" || std::string(kafka_enable_ssl) == "1") {
kafkaConfig.enable_ssl = true;
} else {
kafkaConfig.enable_ssl = false;
}
}
const std::string enableSslStr(kafka_enable_ssl);
kafkaConfig.enable_ssl = (enableSslStr == "true" || enableSslStr == "1");

return kafkaConfig;
}
Expand Down
6 changes: 2 additions & 4 deletions include/config/timeouts.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,10 @@

#include <cstdint>

namespace moderation {
namespace config {
namespace moderation::config {

// Kafka timeouts in milliseconds
constexpr int32_t KAFKA_FLUSH_TIMEOUT_MS = 10000;
constexpr int32_t KAFKA_CONSUME_TIMEOUT_MS = 1000;

} // namespace config
} // namespace moderation
} // namespace moderation::config
2 changes: 1 addition & 1 deletion src/moderationservice/kafka/kafkaclient.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
#include <utility>
#include <vector>

KafkaClient::KafkaClient(const KafkaConfig& config) : config_(config), initialized_(false) {}
KafkaClient::KafkaClient(KafkaConfig config) : config_(std::move(config)) {}
void KafkaClient::Initialize(std::function<void(const moderation::ModerateObjectResponse&, int64_t,
const std::string&, moderation::ObjectType)>
result_callback) {
Expand Down
14 changes: 8 additions & 6 deletions src/moderationservice/kafka/kafkaclient.hpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#pragma once

#include "config/config.hpp"
#include "config/timeouts.hpp"
#include "moderation.pb.h"
#include <atomic>
#include <functional>
Expand All @@ -10,6 +11,7 @@
#include <mutex>
#include <string>
#include <thread>
#include <utility>
#include <vector>

class KafkaProducer;
Expand All @@ -27,8 +29,8 @@ class ConsumerEventCb : public RdKafka::EventCb {

class KafkaClient {
public:
explicit KafkaClient(const KafkaConfig& config);
~KafkaClient();
explicit KafkaClient(KafkaConfig config);
virtual ~KafkaClient();

virtual void Initialize(std::function<void(const moderation::ModerateObjectResponse&, int64_t,
const std::string&, moderation::ObjectType)>
Expand All @@ -44,7 +46,7 @@ class KafkaClient {
KafkaConfig config_;
std::unique_ptr<KafkaProducer> producer_;
std::unique_ptr<KafkaConsumer> consumer_;
bool initialized_;
bool initialized_{false};
std::mutex mutex_;
};

Expand All @@ -57,8 +59,8 @@ class KafkaProducer {
const std::string& topic);
bool SendResponseAsync(const moderation::ModerateObjectResponse& response,
const std::string& topic, const std::string& key);
bool Flush(int timeoutMs = 10000);
bool isHealthy() const { return producer_ != nullptr; }
bool Flush(int timeoutMs = moderation::config::KAFKA_FLUSH_TIMEOUT_MS);
[[nodiscard]] bool isHealthy() const { return producer_ != nullptr; }

private:
KafkaConfig config_;
Expand All @@ -79,7 +81,7 @@ class KafkaConsumer {

void Start();
void Stop();
bool isRunning() const { return running_.load(); }
[[nodiscard]] bool isRunning() const { return running_.load(); }

private:
void ConsumeLoop();
Expand Down
3 changes: 3 additions & 0 deletions src/moderationservice/model/constants.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,7 @@ extern tsl::htrie_set<char> forbiddenWords;
void InitializeForbiddenWords();

} // namespace HashTrieMaps
namespace Thresholds {
inline constexpr unsigned int FORBIDDEN_WORD_THRESHOLD = 3;
} // namespace Thresholds
} // namespace TextProcessingConstants
6 changes: 2 additions & 4 deletions src/moderationservice/model/model_utils.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,8 @@
#include "moderation.pb.h"
#include <string>

namespace moderation {
namespace utils {
namespace moderation::utils {

std::string ObjectTypeToString(moderation::ObjectType type);

} // namespace utils
} // namespace moderation
} // namespace moderation::utils
12 changes: 12 additions & 0 deletions src/moderationservice/model/unicode_constants.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,16 @@ constexpr UChar32 PARAGRAPH_SEPARATOR = 0x2029;
constexpr UChar32 BIDI_MARKERS_START = 0x202A;
constexpr UChar32 BIDI_MARKERS_END = 0x202E;

// Cyrillic block
constexpr UChar32 CYRILLIC_BLOCK_START = 0x0400;
constexpr UChar32 CYRILLIC_BLOCK_END = 0x04FF;

// Latin uppercase A–Z
constexpr UChar32 LATIN_UPPERCASE_START = 0x0041;
constexpr UChar32 LATIN_UPPERCASE_END = 0x005A;

// Latin lowercase a–z
constexpr UChar32 LATIN_LOWERCASE_START = 0x0061;
constexpr UChar32 LATIN_LOWERCASE_END = 0x007A;

} // namespace unicode_constants
Loading