From 5215b9aa482581c010266e0b2ceee68b34ef4f8b Mon Sep 17 00:00:00 2001 From: Samuel Date: Sun, 22 Feb 2026 23:11:36 +0100 Subject: [PATCH 01/21] implemented workflow (not tested yet) --- .github/workflows/lint-and-test.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index e69de29..ddd95f8 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -0,0 +1,23 @@ +name: "lint and test" + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + types: [ opened, synchronize, reopened ] + +permissions: + contents: read + pull-requests: write + +jobs: + lint-and-test: + uses: esclient/tools/.github/workflows/lint-and-test-cpp.yml@v1.0.0 + with: + source: "moderationservice" + sonar-inclusions: "src/**,tests/**" + sonar-exclusions: "**/generated/**,**/grpc/**" + sonar-coverage-exclusions: "src/moderationservice/main.cpp,src/moderationservice/server/server.cpp" + secrets: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file From b1c6463ef752d82f693b3d99bfc0b2e7e1c20158 Mon Sep 17 00:00:00 2001 From: Samuel Date: Mon, 23 Feb 2026 16:26:11 +0100 Subject: [PATCH 02/21] testing via branch --- .github/workflows/lint-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index ddd95f8..5a90f94 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -13,7 +13,7 @@ permissions: jobs: lint-and-test: - uses: esclient/tools/.github/workflows/lint-and-test-cpp.yml@v1.0.0 + uses: esclient/tools/.github/workflows/lint-and-test-cpp.yml@cpp-workflow with: source: "moderationservice" sonar-inclusions: "src/**,tests/**" From c751ae782f91968a052f7f6416722c2df7811731 Mon Sep 17 00:00:00 2001 From: Samuel Date: Mon, 23 Feb 2026 17:16:47 +0100 Subject: [PATCH 03/21] Test --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c96f674..9c393b6 100644 --- a/README.md +++ b/README.md @@ -17,3 +17,4 @@ To build and run you need the following installed: -sh/bash (Windows only) -Internet connection (Optional) -Electricity (Recommended) +-Antidepressants(Recommended) \ No newline at end of file From 25cd322b14e8b961dfa7b245ef0cfe1a4dfdc1f8 Mon Sep 17 00:00:00 2001 From: Samuel Date: Mon, 23 Feb 2026 17:20:09 +0100 Subject: [PATCH 04/21] test -> change the URL later --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index d4e710b..2debfdd 100644 --- a/justfile +++ b/justfile @@ -1,7 +1,7 @@ set windows-shell := ["sh", "-c"] set dotenv-load := true -COMMON_JUST_URL := 'https://raw.githubusercontent.com/esclient/tools/refs/heads/main/cpp/common.just' +COMMON_JUST_URL := 'https://raw.githubusercontent.com/esclient/tools/refs/heads/cpp-workflow/cpp/common.just' LOAD_ENVS_URL := 'https://raw.githubusercontent.com/esclient/tools/refs/heads/main/load_envs.sh' PROTO_TAG := 'v0.1.3' From b539b827bc8188548b687b7aa1e25141ec0120b2 Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 24 Feb 2026 23:33:25 +0100 Subject: [PATCH 05/21] Fixed linter warnings, fixed absl bug --- .clang-tidy | 12 +---- CMakeLists.txt | 19 ++++--- include/config/config.hpp | 50 ++++++++----------- include/config/timeouts.hpp | 6 +-- src/moderationservice/kafka/kafkaclient.cpp | 2 +- src/moderationservice/kafka/kafkaclient.hpp | 12 +++-- src/moderationservice/model/constants.hpp | 3 ++ src/moderationservice/model/model_utils.hpp | 6 +-- .../model/unicode_constants.hpp | 12 +++++ .../service/leetspeak_normalization.cpp | 30 ++++++----- .../service/word_checker.cpp | 15 +++--- tests/mocks/fake_moderation_repository.hpp | 4 +- tests/mocks/mock_kafka_client.hpp | 2 +- tests/mocks/test_utils.hpp | 2 +- tests/unit/service/text_processor_test.cpp | 3 +- tests/unit/service/word_checker_test.cpp | 2 +- 16 files changed, 93 insertions(+), 87 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 230fa6f..1e7cc46 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,12 +1,4 @@ -Checks: > - clang-analyzer-*, - bugprone-*, - performance-*, - modernize-*, - readability-*, - -modernize-use-trailing-return-type - +Checks: 'clang-analyzer-*,bugprone-*,performance-*,modernize-*,readability-*,-modernize-use-trailing-return-type,-bugprone-suspicious-include' WarningsAsErrors: '' - -HeaderFilterRegex: '^(src|include|tests|benchmark)/' +HeaderFilterRegex: '.*/moderation-service/(src|tests|benchmark|include)/.*' FormatStyle: file \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index ac1f9e0..76eff04 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,7 @@ find_package(ICU REQUIRED COMPONENTS uc i18n data) find_package(RdKafka REQUIRED) find_package(Threads REQUIRED) find_package(libpqxx REQUIRED) +find_package(absl CONFIG REQUIRED) set(CPM_DOWNLOAD_VERSION 0.40.2) if(CPM_SOURCE_CACHE) @@ -82,11 +83,12 @@ add_custom_command( # Create library from generated sources add_library(${GENPROTO_LIB} ${PROTO_SRCS} ${PROTO_HDRS} ${GRPC_SRCS} ${GRPC_HDRS}) -target_include_directories( - ${GENPROTO_LIB} PUBLIC ${GENPROTO_DIR} ${protobuf_INCLUDE_DIRS_RELEASE} - ${grpc_INCLUDE_DIRS_RELEASE} ${icu_INCLUDE_DIRS_RELEASE}) +target_include_directories(${GENPROTO_LIB} + PUBLIC + ${GENPROTO_DIR} +) -target_link_libraries(${GENPROTO_LIB} PUBLIC protobuf::libprotobuf gRPC::grpc++) +target_link_libraries(${GENPROTO_LIB} PUBLIC protobuf::libprotobuf gRPC::grpc++ absl::strings) # link source files add_executable( @@ -117,7 +119,8 @@ target_link_libraries( Threads::Threads RdKafka::rdkafka RdKafka::rdkafka++ - libpqxx::pqxx) + libpqxx::pqxx + absl::strings) if(WIN32) target_compile_definitions(${PROJECT_NAME} PRIVATE U_STATIC_IMPLEMENTATION) @@ -256,9 +259,9 @@ if(CPPCHECK_EXE COMMAND ${CMAKE_COMMAND} -E echo "=== Running cppcheck ===" COMMAND ${CPPCHECK_EXE} --enable=warning,style,performance,portability --inline-suppr - --suppress=*:${CMAKE_BINARY_DIR}/generated/* -I${PROJECT_SOURCE_DIR}/include - -I${PROJECT_SOURCE_DIR}/src/moderationservice --library=gnu - ${PROJECT_SOURCE_DIR}/src/moderationservice ${PROJECT_SOURCE_DIR}/include + --suppress=*:${CMAKE_BINARY_DIR}/generated/* --suppress=*:${CMAKE_BINARY_DIR}/_deps/* + -I${PROJECT_SOURCE_DIR}/include -I${PROJECT_SOURCE_DIR}/src/moderationservice + --library=gnu ${PROJECT_SOURCE_DIR}/src/moderationservice ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/tests ${PROJECT_SOURCE_DIR}/benchmark COMMAND ${CMAKE_COMMAND} -E echo "" COMMAND ${CMAKE_COMMAND} -E echo "=== Running cpplint ===" diff --git a/include/config/config.hpp b/include/config/config.hpp index 0ba090d..f5157d2 100644 --- a/include/config/config.hpp +++ b/include/config/config.hpp @@ -18,44 +18,38 @@ struct Config { const char* host = std::getenv("HOST"); if (host == nullptr) { throw std::runtime_error("HOST environment variable not set"); - } else { - config.host = host; } + config.host = host; const char* port = std::getenv("PORT"); if (port == nullptr) { throw std::runtime_error("PORT environment variable not set"); - } else { - config.port = port; } + config.port = port; const char* database_url = std::getenv("DATABASE_URL"); if (database_url == nullptr) { throw std::runtime_error("DATABASE_URL environment variable not set"); - } else { - config.database_url = database_url; } + config.database_url = database_url; const char* log_level = std::getenv("LOG_LEVEL"); if (log_level == nullptr) { throw std::runtime_error("LOG_LEVEL environment variable not set"); - } else { - config.log_level = log_level; } + config.log_level = log_level; const char* log_format = std::getenv("LOG_FORMAT"); if (log_format == nullptr) { throw std::runtime_error("LOG_FORMAT environment variable not set"); - } else { - config.log_format = log_format; } + config.log_format = log_format; const char* log_datefmt = std::getenv("LOG_DATEFMT"); if (log_datefmt == nullptr) { throw std::runtime_error("LOG_DATEFMT environment variable not set"); - } else { - config.log_datefmt = log_datefmt; } + config.log_datefmt = log_datefmt; return config; } @@ -76,49 +70,45 @@ struct KafkaConfig { const char* kafka_brokers = std::getenv("KAFKA_BROKERS"); if (kafka_brokers == nullptr) { throw std::runtime_error("KAFKA_BROKERS environment variable not set"); - } else { - kafkaConfig.brokers = kafka_brokers; } + kafkaConfig.brokers = kafka_brokers; + const char* kafka_request_topic = std::getenv("KAFKA_REQUEST_TOPIC"); if (kafka_request_topic == nullptr) { throw std::runtime_error("KAFKA_REQUEST_TOPIC environment variable not set"); - } else { - kafkaConfig.request_topic = kafka_request_topic; } + kafkaConfig.request_topic = kafka_request_topic; + const char* kafka_result_topic = std::getenv("KAFKA_RESULT_TOPIC"); if (kafka_result_topic == nullptr) { throw std::runtime_error("KAFKA_RESULT_TOPIC environment variable not set"); - } else { - kafkaConfig.result_topic = kafka_result_topic; } + kafkaConfig.result_topic = kafka_result_topic; + const char* kafka_consumer_group_id = std::getenv("KAFKA_CONSUMER_GROUP_ID"); if (kafka_consumer_group_id == nullptr) { throw std::runtime_error("KAFKA_CONSUMER_GROUP_ID environment variable not set"); - } else { - kafkaConfig.consumer_group_id = kafka_consumer_group_id; } + kafkaConfig.consumer_group_id = kafka_consumer_group_id; + const char* kafka_max_retries = std::getenv("KAFKA_MAX_RETRIES"); if (kafka_max_retries == nullptr) { throw std::runtime_error("KAFKA_MAX_RETRIES environment variable not set"); - } else { - kafkaConfig.max_retries = std::stoi(kafka_max_retries); } + kafkaConfig.max_retries = std::stoi(kafka_max_retries); + const char* kafka_retry_backoff_ms = std::getenv("KAFKA_RETRY_BACKOFF_MS"); if (kafka_retry_backoff_ms == nullptr) { throw std::runtime_error("KAFKA_RETRY_BACKOFF_MS environment variable not set"); - } else { - kafkaConfig.retry_backoff_ms = std::stoi(kafka_retry_backoff_ms); } + kafkaConfig.retry_backoff_ms = std::stoi(kafka_retry_backoff_ms); + const char* kafka_enable_ssl = std::getenv("KAFKA_ENABLE_SSL"); if (kafka_enable_ssl == nullptr) { throw std::runtime_error("KAFKA_ENABLE_SSL environment variable not set"); - } else { - if (std::string(kafka_enable_ssl) == "true" || std::string(kafka_enable_ssl) == "1") { - kafkaConfig.enable_ssl = true; - } else { - kafkaConfig.enable_ssl = false; - } } + const std::string enableSslStr(kafka_enable_ssl); + kafkaConfig.enable_ssl = (enableSslStr == "true" || enableSslStr == "1"); return kafkaConfig; } diff --git a/include/config/timeouts.hpp b/include/config/timeouts.hpp index a6aae3f..b4101aa 100644 --- a/include/config/timeouts.hpp +++ b/include/config/timeouts.hpp @@ -2,12 +2,10 @@ #include -namespace moderation { -namespace config { +namespace moderation::config { // Kafka timeouts in milliseconds constexpr int32_t KAFKA_FLUSH_TIMEOUT_MS = 10000; constexpr int32_t KAFKA_CONSUME_TIMEOUT_MS = 1000; -} // namespace config -} // namespace moderation +} // namespace moderation::config diff --git a/src/moderationservice/kafka/kafkaclient.cpp b/src/moderationservice/kafka/kafkaclient.cpp index c1d7ea1..e15e8e5 100644 --- a/src/moderationservice/kafka/kafkaclient.cpp +++ b/src/moderationservice/kafka/kafkaclient.cpp @@ -9,7 +9,7 @@ #include #include -KafkaClient::KafkaClient(const KafkaConfig& config) : config_(config), initialized_(false) {} +KafkaClient::KafkaClient(KafkaConfig config) : config_(std::move(config)) {} void KafkaClient::Initialize(std::function result_callback) { diff --git a/src/moderationservice/kafka/kafkaclient.hpp b/src/moderationservice/kafka/kafkaclient.hpp index 98c6f1d..0eeb178 100644 --- a/src/moderationservice/kafka/kafkaclient.hpp +++ b/src/moderationservice/kafka/kafkaclient.hpp @@ -1,6 +1,7 @@ #pragma once #include "config/config.hpp" +#include "config/timeouts.hpp" #include "moderation.pb.h" #include #include @@ -10,6 +11,7 @@ #include #include #include +#include #include class KafkaProducer; @@ -27,7 +29,7 @@ class ConsumerEventCb : public RdKafka::EventCb { class KafkaClient { public: - explicit KafkaClient(const KafkaConfig& config); + explicit KafkaClient(KafkaConfig config); ~KafkaClient(); virtual void Initialize(std::function producer_; std::unique_ptr consumer_; - bool initialized_; + bool initialized_{false}; std::mutex mutex_; }; @@ -57,8 +59,8 @@ class KafkaProducer { const std::string& topic); bool SendResponseAsync(const moderation::ModerateObjectResponse& response, const std::string& topic, const std::string& key); - bool Flush(int timeoutMs = 10000); - bool isHealthy() const { return producer_ != nullptr; } + bool Flush(int timeoutMs = moderation::config::KAFKA_FLUSH_TIMEOUT_MS); + [[nodiscard]] bool isHealthy() const { return producer_ != nullptr; } private: KafkaConfig config_; @@ -79,7 +81,7 @@ class KafkaConsumer { void Start(); void Stop(); - bool isRunning() const { return running_.load(); } + [[nodiscard]] bool isRunning() const { return running_.load(); } private: void ConsumeLoop(); diff --git a/src/moderationservice/model/constants.hpp b/src/moderationservice/model/constants.hpp index ac994be..2ce8407 100644 --- a/src/moderationservice/model/constants.hpp +++ b/src/moderationservice/model/constants.hpp @@ -21,4 +21,7 @@ extern tsl::htrie_set forbiddenWords; void InitializeForbiddenWords(); } // namespace HashTrieMaps +namespace Thresholds { +inline constexpr unsigned int FORBIDDEN_WORD_THRESHOLD = 3; +} // namespace Thresholds } // namespace TextProcessingConstants diff --git a/src/moderationservice/model/model_utils.hpp b/src/moderationservice/model/model_utils.hpp index c1c8b7e..ac4ecf6 100644 --- a/src/moderationservice/model/model_utils.hpp +++ b/src/moderationservice/model/model_utils.hpp @@ -3,10 +3,8 @@ #include "moderation.pb.h" #include -namespace moderation { -namespace utils { +namespace moderation::utils { std::string ObjectTypeToString(moderation::ObjectType type); -} // namespace utils -} // namespace moderation +} // namespace moderation::utils diff --git a/src/moderationservice/model/unicode_constants.hpp b/src/moderationservice/model/unicode_constants.hpp index d273bee..9e4b874 100644 --- a/src/moderationservice/model/unicode_constants.hpp +++ b/src/moderationservice/model/unicode_constants.hpp @@ -29,4 +29,16 @@ constexpr UChar32 PARAGRAPH_SEPARATOR = 0x2029; constexpr UChar32 BIDI_MARKERS_START = 0x202A; constexpr UChar32 BIDI_MARKERS_END = 0x202E; +// Cyrillic block +constexpr UChar32 CYRILLIC_BLOCK_START = 0x0400; +constexpr UChar32 CYRILLIC_BLOCK_END = 0x04FF; + +// Latin uppercase A–Z +constexpr UChar32 LATIN_UPPERCASE_START = 0x0041; +constexpr UChar32 LATIN_UPPERCASE_END = 0x005A; + +// Latin lowercase a–z +constexpr UChar32 LATIN_LOWERCASE_START = 0x0061; +constexpr UChar32 LATIN_LOWERCASE_END = 0x007A; + } // namespace unicode_constants diff --git a/src/moderationservice/service/leetspeak_normalization.cpp b/src/moderationservice/service/leetspeak_normalization.cpp index d5bbb5a..ca03d6f 100644 --- a/src/moderationservice/service/leetspeak_normalization.cpp +++ b/src/moderationservice/service/leetspeak_normalization.cpp @@ -1,5 +1,6 @@ #include "service/leetspeak_normalization.hpp" #include "model/constants.hpp" +#include "model/unicode_constants.hpp" #include namespace LeetspeakNormalization { @@ -11,27 +12,28 @@ const auto& homoglyphMap = TextProcessingConstants::LeetspeakMaps::homoglyphMap; std::string LeetspeakNormalization(const std::string& textN, bool isCyrillic) { icu::UnicodeString text = icu::UnicodeString::fromUTF8(textN); if (text.length() == 0) { - text = icu::UnicodeString(textN.c_str(), textN.length(), "UTF-8"); + const auto length = static_cast(textN.length()); + text = icu::UnicodeString(textN.c_str(), length, "UTF-8"); } icu::UnicodeString result; for (int32_t i = 0; i < text.length(); i++) { - UChar32 c = text.char32At(i); - UChar32 normalized = c; + UChar32 character = text.char32At(i); + UChar32 normalized = character; if (isCyrillic) { - auto homoglyphText = homoglyphMap.find(c); + auto homoglyphText = homoglyphMap.find(character); if (homoglyphText != homoglyphMap.end()) { normalized = homoglyphText->second; } else { - auto leetedTextCyrillic = cyrillicMap.find(c); + auto leetedTextCyrillic = cyrillicMap.find(character); if (leetedTextCyrillic != cyrillicMap.end()) { normalized = leetedTextCyrillic->second; } } } else { - auto leetedText = latinMap.find(c); + auto leetedText = latinMap.find(character); if (leetedText != latinMap.end()) { normalized = leetedText->second; } @@ -52,21 +54,25 @@ bool IsPrimarilyCyrillic(const std::string& textN) { icu::UnicodeString text = icu::UnicodeString::fromUTF8(textN); if (text.length() == 0) { - text = icu::UnicodeString(textN.c_str(), textN.length(), "UTF-8"); + const auto length = static_cast(textN.length()); + text = icu::UnicodeString(textN.c_str(), length, "UTF-8"); } int cyrillicCount = 0; int latinCount = 0; for (int32_t i = 0; i < text.length(); i++) { - UChar32 c = text.char32At(i); + UChar32 character = text.char32At(i); - // Cyrillic range: U+0400 to U+04FF - if (c >= 0x0400 && c <= 0x04FF) { + if (character >= unicode_constants::CYRILLIC_BLOCK_START && + character <= unicode_constants::CYRILLIC_BLOCK_END) { cyrillicCount++; - } else if ((c >= 0x0041 && c <= 0x005A) || (c >= 0x0061 && c <= 0x007A)) { + } else if ((character >= unicode_constants::LATIN_UPPERCASE_START && + character <= unicode_constants::LATIN_UPPERCASE_END) || + (character >= unicode_constants::LATIN_LOWERCASE_START && + character <= unicode_constants::LATIN_LOWERCASE_END)) { latinCount++; - } // Latin range + } if (U16_IS_LEAD(text.charAt(i))) { i++; diff --git a/src/moderationservice/service/word_checker.cpp b/src/moderationservice/service/word_checker.cpp index 0895c8f..55854a0 100644 --- a/src/moderationservice/service/word_checker.cpp +++ b/src/moderationservice/service/word_checker.cpp @@ -11,15 +11,16 @@ std::vector Tokenize(const std::string& textN) { std::vector tokens; icu::UnicodeString text = icu::UnicodeString::fromUTF8(textN); if (text.length() == 0) { - text = icu::UnicodeString(textN.c_str(), textN.length(), "UTF-8"); + const auto length = static_cast(textN.length()); + text = icu::UnicodeString(textN.c_str(), length, "UTF-8"); } std::string currentWord; for (int32_t i = 0; i < text.length(); i++) { - UChar32 c = text.char32At(i); + UChar32 character = text.char32At(i); - if (u_isalnum(c)) { - icu::UnicodeString charStr(c); + if (u_isalnum(character)) { + icu::UnicodeString charStr(character); std::string utf8char; charStr.toUTF8String(utf8char); currentWord += utf8char; @@ -44,13 +45,13 @@ std::vector Tokenize(const std::string& textN) { bool WordChecking(const std::vector& textN) { unsigned int forbiddenWordCount = 0; - for (int i = 0; i < textN.size(); i++) { - if (forbiddenWords.find(textN[i]) != forbiddenWords.end()) { + for (const auto& token : textN) { + if (forbiddenWords.find(token) != forbiddenWords.end()) { forbiddenWordCount++; } } - return (forbiddenWordCount > 3); + return (forbiddenWordCount > TextProcessingConstants::Thresholds::FORBIDDEN_WORD_THRESHOLD); } } // namespace WordChecker diff --git a/tests/mocks/fake_moderation_repository.hpp b/tests/mocks/fake_moderation_repository.hpp index 3d9782d..6acbaae 100644 --- a/tests/mocks/fake_moderation_repository.hpp +++ b/tests/mocks/fake_moderation_repository.hpp @@ -6,8 +6,8 @@ class FakeModerationRepository : public IModerationRepository { public: - bool SaveModerationResult(const ModerationRecord& result); - const std::vector& GetStoredRecords() const { return stored_; } + bool SaveModerationResult(const ModerationRecord& result) override; + [[nodiscard]] const std::vector& GetStoredRecords() const { return stored_; } void Clear() { stored_.clear(); } void SetSaveResult(bool value) { save_result_ = value; } diff --git a/tests/mocks/mock_kafka_client.hpp b/tests/mocks/mock_kafka_client.hpp index 854c61c..c8d7a9d 100644 --- a/tests/mocks/mock_kafka_client.hpp +++ b/tests/mocks/mock_kafka_client.hpp @@ -18,7 +18,7 @@ class MockKafkaClient : public KafkaClient { bool SendRequestAsync(const moderation::ModerateObjectRequest& request) override; void SetSendResult(bool value) { send_result_ = value; } - const std::vector& GetSentRequests() const { + [[nodiscard]] const std::vector& GetSentRequests() const { return sent_requests_; } void ClearSentRequests() { sent_requests_.clear(); } diff --git a/tests/mocks/test_utils.hpp b/tests/mocks/test_utils.hpp index a2c8f0d..a9075fe 100644 --- a/tests/mocks/test_utils.hpp +++ b/tests/mocks/test_utils.hpp @@ -12,7 +12,7 @@ ModerationRecord MakeRecord(int64_t object_id, const std::string& text, bool is_ moderation::ObjectType type = moderation::OBJECT_TYPE_UNSPECIFIED); moderation::ModerateObjectRequest -MakeRequest(int64_t id, const std::string& text, +MakeRequest(int64_t request_id, const std::string& text, moderation::ObjectType type = moderation::OBJECT_TYPE_UNSPECIFIED); moderation::ModerateObjectResponse MakeResponse(bool success); diff --git a/tests/unit/service/text_processor_test.cpp b/tests/unit/service/text_processor_test.cpp index 9d23fd7..b3f0b39 100644 --- a/tests/unit/service/text_processor_test.cpp +++ b/tests/unit/service/text_processor_test.cpp @@ -14,7 +14,8 @@ TEST_F(TextProcessorTest, CleanTextReturnsFalse) { TEST_F(TextProcessorTest, ManyForbiddenWordsReturnsTrue) { std::string bad; - for (int i = 0; i < 5; ++i) { + constexpr int testForbiddenWordRepetitions = 5; + for (int i = 0; i < testForbiddenWordRepetitions; ++i) { bad += "gay "; } diff --git a/tests/unit/service/word_checker_test.cpp b/tests/unit/service/word_checker_test.cpp index 1fe8c9a..ed04a85 100644 --- a/tests/unit/service/word_checker_test.cpp +++ b/tests/unit/service/word_checker_test.cpp @@ -11,7 +11,7 @@ class WordCheckerTest : public ::testing::Test { TEST_F(WordCheckerTest, TokenizeSimple) { auto tokens = WordChecker::Tokenize("hello world"); - ASSERT_EQ(2u, tokens.size()); + ASSERT_EQ(2U, tokens.size()); EXPECT_EQ(tokens[0], "hello"); EXPECT_EQ(tokens[1], "world"); } From 8cdfa6359b62fac05c3071526bc9ecb2384dd40f Mon Sep 17 00:00:00 2001 From: Samuel Date: Wed, 25 Feb 2026 14:38:46 +0100 Subject: [PATCH 06/21] test with new abseil --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 76eff04..0ac95d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -88,7 +88,7 @@ target_include_directories(${GENPROTO_LIB} ${GENPROTO_DIR} ) -target_link_libraries(${GENPROTO_LIB} PUBLIC protobuf::libprotobuf gRPC::grpc++ absl::strings) +target_link_libraries(${GENPROTO_LIB} PUBLIC protobuf::libprotobuf gRPC::grpc++ abseil::abseil) # link source files add_executable( @@ -120,7 +120,7 @@ target_link_libraries( RdKafka::rdkafka RdKafka::rdkafka++ libpqxx::pqxx - absl::strings) + abseil::abseil) if(WIN32) target_compile_definitions(${PROJECT_NAME} PRIVATE U_STATIC_IMPLEMENTATION) From 99973db68c76fd4229e63af6b0486a311a141e4b Mon Sep 17 00:00:00 2001 From: Samuel Date: Wed, 25 Feb 2026 14:43:16 +0100 Subject: [PATCH 07/21] test --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9c393b6..b54f52d 100644 --- a/README.md +++ b/README.md @@ -17,4 +17,6 @@ To build and run you need the following installed: -sh/bash (Windows only) -Internet connection (Optional) -Electricity (Recommended) --Antidepressants(Recommended) \ No newline at end of file +-Antidepressants(Recommended) + +test \ No newline at end of file From b98697c83fa45cd56f76c16982ac64dab7cd2d8e Mon Sep 17 00:00:00 2001 From: Samuel Date: Wed, 25 Feb 2026 14:49:57 +0100 Subject: [PATCH 08/21] fixed a bug with false passing parameter --- .github/workflows/lint-and-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 5a90f94..71c9435 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -15,7 +15,6 @@ jobs: lint-and-test: uses: esclient/tools/.github/workflows/lint-and-test-cpp.yml@cpp-workflow with: - source: "moderationservice" sonar-inclusions: "src/**,tests/**" sonar-exclusions: "**/generated/**,**/grpc/**" sonar-coverage-exclusions: "src/moderationservice/main.cpp,src/moderationservice/server/server.cpp" From 481c6e68bca7ebc262e6ea1dae380160c4147d0e Mon Sep 17 00:00:00 2001 From: Samuel Date: Wed, 25 Feb 2026 15:46:21 +0100 Subject: [PATCH 09/21] fix attempts --- CMakeLists.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ac95d4..dc09ab0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ endif() include(${CMAKE_BINARY_DIR}/conan_toolchain.cmake OPTIONAL) -find_package(Protobuf REQUIRED CONFIG) +find_package(protobuf REQUIRED CONFIG) find_package(gRPC REQUIRED CONFIG) find_package(ICU REQUIRED COMPONENTS uc i18n data) find_package(RdKafka REQUIRED) @@ -86,9 +86,12 @@ add_library(${GENPROTO_LIB} ${PROTO_SRCS} ${PROTO_HDRS} ${GRPC_SRCS} ${GRPC_HDRS target_include_directories(${GENPROTO_LIB} PUBLIC ${GENPROTO_DIR} + ${protobuf_INCLUDE_DIRS} + ${grpc_INCLUDE_DIRS} + ${ICU_INCLUDE_DIRS} ) -target_link_libraries(${GENPROTO_LIB} PUBLIC protobuf::libprotobuf gRPC::grpc++ abseil::abseil) +target_link_libraries(${GENPROTO_LIB} PUBLIC protobuf::protobuf gRPC::grpc++ abseil::abseil) # link source files add_executable( @@ -111,7 +114,7 @@ target_link_libraries( PRIVATE ${GENPROTO_LIB} gRPC::grpc++ gRPC::grpc++_reflection - protobuf::libprotobuf + protobuf::protobuf ICU::uc ICU::i18n ICU::data From bbb5635e34d643f062cf4c9e298fdc0970cde40f Mon Sep 17 00:00:00 2001 From: Samuel Date: Thu, 26 Feb 2026 17:08:14 +0100 Subject: [PATCH 10/21] Fixed compatibility bug with linux --- tools/common.just | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/common.just b/tools/common.just index ac58f66..20cbcda 100644 --- a/tools/common.just +++ b/tools/common.just @@ -2,6 +2,8 @@ MKDIR := 'mkdir -p' RM := 'rm -rf ' + TMP_DIR DOWN := 'curl -fsSL' DOWN_OUT := '-o' +EXE := if os() == "windows" { ".exe" } else { "" } +TEST_DIR := if os() == "windows" { "./build/tests/Release" } else { "./build/tests" } fetch-proto: {{MKDIR}} "{{TMP_DIR}}" @@ -83,12 +85,12 @@ upgrade-python-tools: pipx upgrade cmakelang test: - ./build/tests/Release/fake_repository_test.exe - ./build/tests/Release/handler_test.exe - ./build/tests/Release/moderation_service_test.exe - ./build/tests/Release/text_normalization_test.exe - ./build/tests/Release/text_processor_test.exe - ./build/tests/Release/word_checker_test.exe + {{TEST_DIR}}/fake_repository_test{{EXE}} + {{TEST_DIR}}/handler_test{{EXE}} + {{TEST_DIR}}/moderation_service_test{{EXE}} + {{TEST_DIR}}/text_normalization_test{{EXE}} + {{TEST_DIR}}/text_processor_test{{EXE}} + {{TEST_DIR}}/word_checker_test{{EXE}} benchmark: just build-windows From 0d83ede13acfb33175a31bbd0a2c27f7fc792908 Mon Sep 17 00:00:00 2001 From: Samuel Date: Thu, 26 Feb 2026 17:34:19 +0100 Subject: [PATCH 11/21] test --- tests/CMakeLists.txt | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0b2d0ff..d572d4b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -46,7 +46,7 @@ function(add_service_test TEST_NAME) RdKafka::rdkafka++ ${TEST_DEPS} ) - add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) + gtest_discover_tests(${TEST_NAME} DISCOVERY_MODE PRE_TEST) endfunction() # ============================================================================ @@ -56,22 +56,18 @@ add_service_test(fake_repository_test ${CMAKE_CURRENT_SOURCE_DIR}/unit/repository/repository_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/mocks/fake_moderation_repository.cpp ) -gtest_discover_tests(fake_repository_test) add_service_test(text_normalization_test ${CMAKE_CURRENT_SOURCE_DIR}/unit/service/text_normalization_test.cpp ) -gtest_discover_tests(text_normalization_test) add_service_test(word_checker_test ${CMAKE_CURRENT_SOURCE_DIR}/unit/service/word_checker_test.cpp ) -gtest_discover_tests(word_checker_test) add_service_test(text_processor_test ${CMAKE_CURRENT_SOURCE_DIR}/unit/service/text_processor_test.cpp ) -gtest_discover_tests(text_processor_test) add_service_test(moderation_service_test ${CMAKE_CURRENT_SOURCE_DIR}/unit/service/moderation_service_test.cpp @@ -81,7 +77,6 @@ add_service_test(moderation_service_test ${CMAKE_CURRENT_SOURCE_DIR}/mocks/test_utils.cpp ${CMAKE_SOURCE_DIR}/src/moderationservice/kafka/kafkaclient.cpp ) -gtest_discover_tests(moderation_service_test) add_service_test(handler_test ${CMAKE_CURRENT_SOURCE_DIR}/unit/handler/handler_test.cpp @@ -92,4 +87,3 @@ add_service_test(handler_test ${CMAKE_CURRENT_SOURCE_DIR}/mocks/mock_kafka_client.cpp ${CMAKE_CURRENT_SOURCE_DIR}/mocks/test_utils.cpp ) -gtest_discover_tests(handler_test) From b64bbb78c66983a063d3b14fb09c281d1642222b Mon Sep 17 00:00:00 2001 From: Samuel Date: Thu, 26 Feb 2026 18:03:48 +0100 Subject: [PATCH 12/21] test --- CMakeLists.txt | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dc09ab0..c20231d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,6 @@ cmake_minimum_required(VERSION 3.20) project(moderation-service) +enable_testing() set(CMAKE_CXX_STANDARD 17) if(MSVC) @@ -18,6 +19,12 @@ if(WIN32) add_compile_definitions(U_STATIC_IMPLEMENTATION) endif() +option(ENABLE_COVERAGE "Enable coverage reporting" OFF) +if(ENABLE_COVERAGE) + add_compile_options(--coverage -fprofile-arcs -ftest-coverage) + add_link_options(--coverage) +endif() + include(${CMAKE_BINARY_DIR}/conan_toolchain.cmake OPTIONAL) find_package(protobuf REQUIRED CONFIG) @@ -86,12 +93,9 @@ add_library(${GENPROTO_LIB} ${PROTO_SRCS} ${PROTO_HDRS} ${GRPC_SRCS} ${GRPC_HDRS target_include_directories(${GENPROTO_LIB} PUBLIC ${GENPROTO_DIR} - ${protobuf_INCLUDE_DIRS} - ${grpc_INCLUDE_DIRS} - ${ICU_INCLUDE_DIRS} ) -target_link_libraries(${GENPROTO_LIB} PUBLIC protobuf::protobuf gRPC::grpc++ abseil::abseil) +target_link_libraries(${GENPROTO_LIB} PUBLIC protobuf::libprotobuf gRPC::grpc++ abseil::abseil) # link source files add_executable( @@ -114,7 +118,7 @@ target_link_libraries( PRIVATE ${GENPROTO_LIB} gRPC::grpc++ gRPC::grpc++_reflection - protobuf::protobuf + protobuf::libprotobuf ICU::uc ICU::i18n ICU::data From df95b222b81643d10065867a4e51da58632ae0cc Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 3 Mar 2026 15:03:40 +0100 Subject: [PATCH 13/21] test --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b54f52d..5496968 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ +test + To build and run you need the following installed: -Chocolatey (Windows) From bd3fbe918f21f3052878cd3d7fde3470772a9231 Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 3 Mar 2026 15:26:38 +0100 Subject: [PATCH 14/21] test --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5496968..6824242 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ +test test To build and run you need the following installed: From 296c1b824f9b198827a0606b05b3aba5b27259e6 Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 3 Mar 2026 16:35:30 +0100 Subject: [PATCH 15/21] added new tests to increase the coverage --- .github/workflows/lint-and-test.yml | 4 +- README.md | 6 --- src/moderationservice/kafka/kafkaclient.hpp | 2 +- .../service/leetspeak_normalization.cpp | 4 +- .../service/word_checker.cpp | 3 +- tests/mocks/mock_kafka_client.hpp | 1 + .../service/leetspeak_normalization_test.cpp | 44 +++++++++++++++++++ tests/unit/service/word_checker_test.cpp | 28 ++++++++++++ 8 files changed, 79 insertions(+), 13 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 71c9435..7a07efa 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -16,7 +16,7 @@ jobs: uses: esclient/tools/.github/workflows/lint-and-test-cpp.yml@cpp-workflow with: sonar-inclusions: "src/**,tests/**" - sonar-exclusions: "**/generated/**,**/grpc/**" - sonar-coverage-exclusions: "src/moderationservice/main.cpp,src/moderationservice/server/server.cpp" + sonar-exclusions: "**/generated/**,**/grpc/**,tests/mocks/**" + sonar-coverage-exclusions: "tests/mocks/**,src/moderationservice/main.cpp,src/moderationservice/server/server.cpp" secrets: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file diff --git a/README.md b/README.md index 6824242..be61625 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,6 @@ Unbenannt-2025-10-16-0020(1) - -test -test - To build and run you need the following installed: -Chocolatey (Windows) @@ -21,5 +17,3 @@ To build and run you need the following installed: -Internet connection (Optional) -Electricity (Recommended) -Antidepressants(Recommended) - -test \ No newline at end of file diff --git a/src/moderationservice/kafka/kafkaclient.hpp b/src/moderationservice/kafka/kafkaclient.hpp index 0eeb178..1f477f5 100644 --- a/src/moderationservice/kafka/kafkaclient.hpp +++ b/src/moderationservice/kafka/kafkaclient.hpp @@ -30,7 +30,7 @@ class ConsumerEventCb : public RdKafka::EventCb { class KafkaClient { public: explicit KafkaClient(KafkaConfig config); - ~KafkaClient(); + virtual ~KafkaClient(); virtual void Initialize(std::function diff --git a/src/moderationservice/service/leetspeak_normalization.cpp b/src/moderationservice/service/leetspeak_normalization.cpp index ca03d6f..2f7ebdc 100644 --- a/src/moderationservice/service/leetspeak_normalization.cpp +++ b/src/moderationservice/service/leetspeak_normalization.cpp @@ -62,9 +62,9 @@ bool IsPrimarilyCyrillic(const std::string& textN) { int latinCount = 0; for (int32_t i = 0; i < text.length(); i++) { - UChar32 character = text.char32At(i); - if (character >= unicode_constants::CYRILLIC_BLOCK_START && + if (UChar32 character = text.char32At(i); + character >= unicode_constants::CYRILLIC_BLOCK_START && character <= unicode_constants::CYRILLIC_BLOCK_END) { cyrillicCount++; } else if ((character >= unicode_constants::LATIN_UPPERCASE_START && diff --git a/src/moderationservice/service/word_checker.cpp b/src/moderationservice/service/word_checker.cpp index 55854a0..dbb55e9 100644 --- a/src/moderationservice/service/word_checker.cpp +++ b/src/moderationservice/service/word_checker.cpp @@ -17,9 +17,8 @@ std::vector Tokenize(const std::string& textN) { std::string currentWord; for (int32_t i = 0; i < text.length(); i++) { - UChar32 character = text.char32At(i); - if (u_isalnum(character)) { + if (UChar32 character = text.char32At(i); u_isalnum(character)) { icu::UnicodeString charStr(character); std::string utf8char; charStr.toUTF8String(utf8char); diff --git a/tests/mocks/mock_kafka_client.hpp b/tests/mocks/mock_kafka_client.hpp index c8d7a9d..e588145 100644 --- a/tests/mocks/mock_kafka_client.hpp +++ b/tests/mocks/mock_kafka_client.hpp @@ -16,6 +16,7 @@ class MockKafkaClient : public KafkaClient { const std::string&, moderation::ObjectType)> callback) override; bool SendRequestAsync(const moderation::ModerateObjectRequest& request) override; + ~MockKafkaClient() override = default; void SetSendResult(bool value) { send_result_ = value; } [[nodiscard]] const std::vector& GetSentRequests() const { diff --git a/tests/unit/service/leetspeak_normalization_test.cpp b/tests/unit/service/leetspeak_normalization_test.cpp index 391a977..9eba000 100644 --- a/tests/unit/service/leetspeak_normalization_test.cpp +++ b/tests/unit/service/leetspeak_normalization_test.cpp @@ -14,3 +14,47 @@ TEST(LeetspeakNormalization, IsPrimarilyCyrillic_CyrillicWin) { TEST(LeetspeakNormalization, IsPrimarilyCyrillic_LatinWin) { EXPECT_FALSE(LeetspeakNormalization::IsPrimarilyCyrillic("hello hello привет")); } + +TEST(LeetspeakNormalization, EmptyInputReturnsEmpty) { + std::string outLatin = LeetspeakNormalization::LeetspeakNormalization("", false); + std::string outCyrillic = LeetspeakNormalization::LeetspeakNormalization("", true); + + EXPECT_EQ(outLatin, ""); + EXPECT_EQ(outCyrillic, ""); +} + +TEST(LeetspeakNormalization, CyrillicHomoglyphMapping) { + std::string out = LeetspeakNormalization::LeetspeakNormalization("a", true); + // Latin 'a' (U+0061) mapped to Cyrillic 'а' (U+0430) + EXPECT_EQ(out, "а"); +} + +TEST(LeetspeakNormalization, CyrillicLeetDigitMapping) { + std::string out = LeetspeakNormalization::LeetspeakNormalization("0", true); + // '0' mapped to Cyrillic 'о' + EXPECT_EQ(out, "о"); +} + +TEST(LeetspeakNormalization, CyrillicNoMappingKeepsCharacter) { + std::string out = LeetspeakNormalization::LeetspeakNormalization("?", true); + EXPECT_EQ(out, "?"); +} + +TEST(LeetspeakNormalization, SurrogatePairIsHandledCorrectly) { + std::string input = "a😊b"; + std::string out = LeetspeakNormalization::LeetspeakNormalization(input, false); + EXPECT_EQ(out, input); +} + +TEST(LeetspeakNormalization, IsPrimarilyCyrillic_EqualCountsReturnsFalse) { + EXPECT_FALSE(LeetspeakNormalization::IsPrimarilyCyrillic("aа")); +} + +TEST(LeetspeakNormalization, IsPrimarilyCyrillic_NoLettersReturnsFalse) { + EXPECT_FALSE(LeetspeakNormalization::IsPrimarilyCyrillic("1234!@#$")); +} + +TEST(LeetspeakNormalization, IsPrimarilyCyrillic_SurrogatePairIgnoredForCounts) { + std::string text = "привет😊hello"; + EXPECT_TRUE(LeetspeakNormalization::IsPrimarilyCyrillic(text)); +} diff --git a/tests/unit/service/word_checker_test.cpp b/tests/unit/service/word_checker_test.cpp index ed04a85..22037b8 100644 --- a/tests/unit/service/word_checker_test.cpp +++ b/tests/unit/service/word_checker_test.cpp @@ -36,3 +36,31 @@ TEST_F(WordCheckerTest, Tokenize_SpecialCharacters) { EXPECT_TRUE(tokens.empty()); } + +TEST_F(WordCheckerTest, TokenizeUnicodeAndEmoji) { + std::string text = "привет😀мир"; + auto tokens = WordChecker::Tokenize(text); + + ASSERT_EQ(tokens.size(), 2U); + EXPECT_EQ(tokens[0], "привет"); + EXPECT_EQ(tokens[1], "мир"); +} + +TEST_F(WordCheckerTest, TokenizeEmojiBetweenAsciiWords) { + std::string text = "hello😀world"; + auto tokens = WordChecker::Tokenize(text); + + ASSERT_EQ(tokens.size(), 2U); + EXPECT_EQ(tokens[0], "hello"); + EXPECT_EQ(tokens[1], "world"); +} + +TEST_F(WordCheckerTest, WordCheckingExactlyAtThresholdIsFalse) { + std::vector at_threshold = {"gay", "gay", "gay"}; + EXPECT_FALSE(WordChecker::WordChecking(at_threshold)); +} + +TEST_F(WordCheckerTest, WordCheckingNoForbiddenWords) { + std::vector safe = {"hello", "world", "test"}; + EXPECT_FALSE(WordChecker::WordChecking(safe)); +} From a224a13a5e0c96f7b0fd819ebd632852b9597f30 Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 3 Mar 2026 16:56:49 +0100 Subject: [PATCH 16/21] fixed a bug where the leetspeak_normalization test wasnt being executed --- .github/workflows/lint-and-test.yml | 4 ++-- CMakeLists.txt | 5 +---- .../service/leetspeak_normalization.cpp | 12 +----------- .../service/text_normalization.cpp | 16 +--------------- src/moderationservice/service/word_checker.cpp | 5 +---- tests/CMakeLists.txt | 4 ++++ 6 files changed, 10 insertions(+), 36 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 7a07efa..cb5167c 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -16,7 +16,7 @@ jobs: uses: esclient/tools/.github/workflows/lint-and-test-cpp.yml@cpp-workflow with: sonar-inclusions: "src/**,tests/**" - sonar-exclusions: "**/generated/**,**/grpc/**,tests/mocks/**" - sonar-coverage-exclusions: "tests/mocks/**,src/moderationservice/main.cpp,src/moderationservice/server/server.cpp" + sonar-exclusions: "**/generated/**,**/grpc/**,tests/mocks/**, tests/unit/**" + sonar-coverage-exclusions: "tests/mocks/**, tests/unit/**,src/moderationservice/main.cpp,src/moderationservice/server/server.cpp" secrets: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index c20231d..4b4eec5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,10 +90,7 @@ add_custom_command( # Create library from generated sources add_library(${GENPROTO_LIB} ${PROTO_SRCS} ${PROTO_HDRS} ${GRPC_SRCS} ${GRPC_HDRS}) -target_include_directories(${GENPROTO_LIB} - PUBLIC - ${GENPROTO_DIR} -) +target_include_directories(${GENPROTO_LIB} PUBLIC ${GENPROTO_DIR}) target_link_libraries(${GENPROTO_LIB} PUBLIC protobuf::libprotobuf gRPC::grpc++ abseil::abseil) diff --git a/src/moderationservice/service/leetspeak_normalization.cpp b/src/moderationservice/service/leetspeak_normalization.cpp index 2f7ebdc..8ad9564 100644 --- a/src/moderationservice/service/leetspeak_normalization.cpp +++ b/src/moderationservice/service/leetspeak_normalization.cpp @@ -11,10 +11,6 @@ const auto& homoglyphMap = TextProcessingConstants::LeetspeakMaps::homoglyphMap; std::string LeetspeakNormalization(const std::string& textN, bool isCyrillic) { icu::UnicodeString text = icu::UnicodeString::fromUTF8(textN); - if (text.length() == 0) { - const auto length = static_cast(textN.length()); - text = icu::UnicodeString(textN.c_str(), length, "UTF-8"); - } icu::UnicodeString result; @@ -53,18 +49,12 @@ std::string LeetspeakNormalization(const std::string& textN, bool isCyrillic) { bool IsPrimarilyCyrillic(const std::string& textN) { icu::UnicodeString text = icu::UnicodeString::fromUTF8(textN); - if (text.length() == 0) { - const auto length = static_cast(textN.length()); - text = icu::UnicodeString(textN.c_str(), length, "UTF-8"); - } - int cyrillicCount = 0; int latinCount = 0; for (int32_t i = 0; i < text.length(); i++) { - if (UChar32 character = text.char32At(i); - character >= unicode_constants::CYRILLIC_BLOCK_START && + if (UChar32 character = text.char32At(i); character >= unicode_constants::CYRILLIC_BLOCK_START && character <= unicode_constants::CYRILLIC_BLOCK_END) { cyrillicCount++; } else if ((character >= unicode_constants::LATIN_UPPERCASE_START && diff --git a/src/moderationservice/service/text_normalization.cpp b/src/moderationservice/service/text_normalization.cpp index 2d19347..12b705b 100644 --- a/src/moderationservice/service/text_normalization.cpp +++ b/src/moderationservice/service/text_normalization.cpp @@ -17,10 +17,6 @@ std::string TextNormalization(const std::string& textN) { return ""; } - if (text.length() == 0) { - text = icu::UnicodeString(textN.c_str(), static_cast(textN.length()), "UTF-8"); - } - icu::UnicodeString normalized = normalizer->normalize(text, status); if (U_FAILURE(status) != 0) { @@ -42,10 +38,6 @@ std::string WhitespaceNormalization(const std::string& textN) { icu::UnicodeString text = icu::UnicodeString::fromUTF8(textN); icu::UnicodeString result; - if (text.length() == 0) { - text = icu::UnicodeString(textN.c_str(), static_cast(textN.length()), "UTF-8"); - } - bool prev_space = false; for (int32_t i = 0; i < text.length(); i++) { @@ -75,10 +67,6 @@ std::string RepetitionNormalization(const std::string& textN) { UChar32 prevChar = 0; - if (text.length() == 0) { - text = icu::UnicodeString(textN.c_str(), static_cast(textN.length()), "UTF-8"); - } - for (int32_t i = 0; i < text.length(); i++) { UChar32 character = text.char32At(i); @@ -99,9 +87,7 @@ std::string RepetitionNormalization(const std::string& textN) { std::string InvisibleCharacterNormalization(const std::string& textN) { icu::UnicodeString text = icu::UnicodeString::fromUTF8(textN); - if (text.length() == 0) { - text = icu::UnicodeString(textN.c_str(), static_cast(textN.length()), "UTF-8"); - } + icu::UnicodeString result; for (int32_t i = 0; i < text.length(); i++) { diff --git a/src/moderationservice/service/word_checker.cpp b/src/moderationservice/service/word_checker.cpp index dbb55e9..2d78613 100644 --- a/src/moderationservice/service/word_checker.cpp +++ b/src/moderationservice/service/word_checker.cpp @@ -10,10 +10,7 @@ const auto& forbiddenWords = TextProcessingConstants::HashTrieMaps::forbiddenWor std::vector Tokenize(const std::string& textN) { std::vector tokens; icu::UnicodeString text = icu::UnicodeString::fromUTF8(textN); - if (text.length() == 0) { - const auto length = static_cast(textN.length()); - text = icu::UnicodeString(textN.c_str(), length, "UTF-8"); - } + std::string currentWord; for (int32_t i = 0; i < text.length(); i++) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d572d4b..1c58d75 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -87,3 +87,7 @@ add_service_test(handler_test ${CMAKE_CURRENT_SOURCE_DIR}/mocks/mock_kafka_client.cpp ${CMAKE_CURRENT_SOURCE_DIR}/mocks/test_utils.cpp ) + +add_service_test(leetspeak_normalization_test + ${CMAKE_CURRENT_SOURCE_DIR}/unit/service/leetspeak_normalization_test.cpp +) From 5ba8ca6fe152d616768f92fefcc2de768f7dc9cc Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 3 Mar 2026 17:00:30 +0100 Subject: [PATCH 17/21] fix --- .github/workflows/lint-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index cb5167c..7f2c9de 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -16,7 +16,7 @@ jobs: uses: esclient/tools/.github/workflows/lint-and-test-cpp.yml@cpp-workflow with: sonar-inclusions: "src/**,tests/**" - sonar-exclusions: "**/generated/**,**/grpc/**,tests/mocks/**, tests/unit/**" - sonar-coverage-exclusions: "tests/mocks/**, tests/unit/**,src/moderationservice/main.cpp,src/moderationservice/server/server.cpp" + sonar-exclusions: "**/generated/**,**/grpc/**,tests/mocks/**,tests/unit/**" + sonar-coverage-exclusions: "tests/mocks/**,tests/unit/**,src/moderationservice/main.cpp,src/moderationservice/server/server.cpp" secrets: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file From 7e0a019cfb31d8dc6ded7762f58dfe35e6678d2d Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 3 Mar 2026 17:32:59 +0100 Subject: [PATCH 18/21] test --- .github/workflows/lint-and-test.yml | 2 +- justfile | 2 +- src/moderationservice/service/leetspeak_normalization.cpp | 3 ++- src/moderationservice/service/text_normalization.cpp | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 7f2c9de..daccead 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -13,7 +13,7 @@ permissions: jobs: lint-and-test: - uses: esclient/tools/.github/workflows/lint-and-test-cpp.yml@cpp-workflow + uses: esclient/tools/.github/workflows/lint-and-test-cpp.yml@main with: sonar-inclusions: "src/**,tests/**" sonar-exclusions: "**/generated/**,**/grpc/**,tests/mocks/**,tests/unit/**" diff --git a/justfile b/justfile index 2debfdd..d4e710b 100644 --- a/justfile +++ b/justfile @@ -1,7 +1,7 @@ set windows-shell := ["sh", "-c"] set dotenv-load := true -COMMON_JUST_URL := 'https://raw.githubusercontent.com/esclient/tools/refs/heads/cpp-workflow/cpp/common.just' +COMMON_JUST_URL := 'https://raw.githubusercontent.com/esclient/tools/refs/heads/main/cpp/common.just' LOAD_ENVS_URL := 'https://raw.githubusercontent.com/esclient/tools/refs/heads/main/load_envs.sh' PROTO_TAG := 'v0.1.3' diff --git a/src/moderationservice/service/leetspeak_normalization.cpp b/src/moderationservice/service/leetspeak_normalization.cpp index 8ad9564..815fbd9 100644 --- a/src/moderationservice/service/leetspeak_normalization.cpp +++ b/src/moderationservice/service/leetspeak_normalization.cpp @@ -54,7 +54,8 @@ bool IsPrimarilyCyrillic(const std::string& textN) { for (int32_t i = 0; i < text.length(); i++) { - if (UChar32 character = text.char32At(i); character >= unicode_constants::CYRILLIC_BLOCK_START && + if (UChar32 character = text.char32At(i); + character >= unicode_constants::CYRILLIC_BLOCK_START && character <= unicode_constants::CYRILLIC_BLOCK_END) { cyrillicCount++; } else if ((character >= unicode_constants::LATIN_UPPERCASE_START && diff --git a/src/moderationservice/service/text_normalization.cpp b/src/moderationservice/service/text_normalization.cpp index 12b705b..5a068ae 100644 --- a/src/moderationservice/service/text_normalization.cpp +++ b/src/moderationservice/service/text_normalization.cpp @@ -87,7 +87,7 @@ std::string RepetitionNormalization(const std::string& textN) { std::string InvisibleCharacterNormalization(const std::string& textN) { icu::UnicodeString text = icu::UnicodeString::fromUTF8(textN); - + icu::UnicodeString result; for (int32_t i = 0; i < text.length(); i++) { From 4f4f4342ee178d7df48ec32caea0dd965fad3a63 Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 3 Mar 2026 20:29:42 +0100 Subject: [PATCH 19/21] linter fix --- .../service/leetspeak_normalization.cpp | 1 - src/moderationservice/service/word_checker.cpp | 11 +++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/moderationservice/service/leetspeak_normalization.cpp b/src/moderationservice/service/leetspeak_normalization.cpp index 815fbd9..915b945 100644 --- a/src/moderationservice/service/leetspeak_normalization.cpp +++ b/src/moderationservice/service/leetspeak_normalization.cpp @@ -53,7 +53,6 @@ bool IsPrimarilyCyrillic(const std::string& textN) { int latinCount = 0; for (int32_t i = 0; i < text.length(); i++) { - if (UChar32 character = text.char32At(i); character >= unicode_constants::CYRILLIC_BLOCK_START && character <= unicode_constants::CYRILLIC_BLOCK_END) { diff --git a/src/moderationservice/service/word_checker.cpp b/src/moderationservice/service/word_checker.cpp index 2d78613..8d6228a 100644 --- a/src/moderationservice/service/word_checker.cpp +++ b/src/moderationservice/service/word_checker.cpp @@ -14,7 +14,6 @@ std::vector Tokenize(const std::string& textN) { std::string currentWord; for (int32_t i = 0; i < text.length(); i++) { - if (UChar32 character = text.char32At(i); u_isalnum(character)) { icu::UnicodeString charStr(character); std::string utf8char; @@ -39,13 +38,9 @@ std::vector Tokenize(const std::string& textN) { return tokens; } bool WordChecking(const std::vector& textN) { - unsigned int forbiddenWordCount = 0; - - for (const auto& token : textN) { - if (forbiddenWords.find(token) != forbiddenWords.end()) { - forbiddenWordCount++; - } - } + unsigned int forbiddenWordCount = std::count_if(textN.begin(), textN.end(), [](const std::string& word) { + return forbiddenWords.find(word) != forbiddenWords.end(); + }); return (forbiddenWordCount > TextProcessingConstants::Thresholds::FORBIDDEN_WORD_THRESHOLD); } From 66a3f5056c709bd7539a0cedc49b38989d6b4c76 Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 3 Mar 2026 20:34:27 +0100 Subject: [PATCH 20/21] test --- src/moderationservice/service/word_checker.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/moderationservice/service/word_checker.cpp b/src/moderationservice/service/word_checker.cpp index 8d6228a..34437ce 100644 --- a/src/moderationservice/service/word_checker.cpp +++ b/src/moderationservice/service/word_checker.cpp @@ -2,6 +2,7 @@ #include "model/constants.hpp" #include #include +#include namespace WordChecker { From d232d8ce9b883f35f2b4bad938a18a28cb8d5ec0 Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 3 Mar 2026 22:24:30 +0100 Subject: [PATCH 21/21] formatted --- src/moderationservice/service/word_checker.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/moderationservice/service/word_checker.cpp b/src/moderationservice/service/word_checker.cpp index 34437ce..a07662e 100644 --- a/src/moderationservice/service/word_checker.cpp +++ b/src/moderationservice/service/word_checker.cpp @@ -1,8 +1,8 @@ #include "service/word_checker.hpp" #include "model/constants.hpp" +#include #include #include -#include namespace WordChecker { @@ -39,9 +39,10 @@ std::vector Tokenize(const std::string& textN) { return tokens; } bool WordChecking(const std::vector& textN) { - unsigned int forbiddenWordCount = std::count_if(textN.begin(), textN.end(), [](const std::string& word) { - return forbiddenWords.find(word) != forbiddenWords.end(); - }); + unsigned int forbiddenWordCount = + std::count_if(textN.begin(), textN.end(), [](const std::string& word) { + return forbiddenWords.find(word) != forbiddenWords.end(); + }); return (forbiddenWordCount > TextProcessingConstants::Thresholds::FORBIDDEN_WORD_THRESHOLD); }