From fdfeea033909e9bef46c4ec8e4649a2dea53c4ba Mon Sep 17 00:00:00 2001 From: Samuel Date: Sun, 26 Apr 2026 13:38:12 +0200 Subject: [PATCH 1/8] basic tests added --- .gitignore | 4 +- pdm.lock | 2 +- pyproject.toml | 2 +- tests/behavioral/test_golden_path.py | 27 ++++++++++++++ tests/conftest.py | 47 ++++++++++++++++++++++++ tests/contract/test_grpc_contract.py | 16 ++++++++ tests/tokenizer/test_tokenizer_shapes.py | 4 ++ tools/common.just | 4 +- 8 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 tests/behavioral/test_golden_path.py create mode 100644 tests/conftest.py create mode 100644 tests/contract/test_grpc_contract.py create mode 100644 tests/tokenizer/test_tokenizer_shapes.py diff --git a/.gitignore b/.gitignore index ba1e1fd..fc9d187 100644 --- a/.gitignore +++ b/.gitignore @@ -219,5 +219,7 @@ data/raw/* patched-tokenizer/* deberta-int8/* deberta-onnx/* - models/* + +*.proto +.proto/* diff --git a/pdm.lock b/pdm.lock index d365cbe..6b8cb92 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "quantize"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:1d063bc87338415c1464de94251e8dbdd67ccc9e7058c9c1944e6afb2b42843c" +content_hash = "sha256:091bbe537cc7f8d027999c2849e85334761b181bb6ec234533139f03b92e8945" [[metadata.targets]] requires_python = ">=3.11" diff --git a/pyproject.toml b/pyproject.toml index 0d59cbe..1f401be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "ai-validator-service" version = "0.1.0" description = "Default template for PDM package" -dependencies = ["grpcio==1.75.1", "transformers>=4.57.6", "tokenizers>=0.21.4", "grpcio-reflection==1.75.1", "pydantic==2.11.9", "pydantic-settings==2.11.0", "psycopg2-binary==2.9.10", "watchfiles==1.1.0", "asyncpg>=0.30.0", "PyPika>=0.51.1", "grpcio-tools==1.75.1", "grpc-stubs==1.53.0.6", "protobuf==6.32.1", "pytest==8.4.2", "pytest-asyncio==0.24.0", "pytest-cov==5.0.0", "pytest-mock==3.15.1", "pytest-faker==2.0.0", "Faker==37.11.0", "black==25.9.0", "isort==6.0.1", "flake8==7.3.0", "flake8-pyproject==1.2.3", "ruff==0.13.2", "mypy==1.18.2", "types-protobuf==6.32.1.20250918", "types-psycopg2==2.9.21.20250915", "scikit-learn>=1.6.1", "torch>=2.7.1", "optimum[onnxruntime]>=2.1.0", "tiktoken>=0.12.0", "sentencepiece>=0.2.1", "huggingface-hub>=0.36.2", "onnxruntime>=1.25.0"] +dependencies = ["grpcio==1.75.1", "transformers>=4.57.6", "tokenizers>=0.21.4", "grpcio-reflection==1.75.1", "pydantic==2.11.9", "pydantic-settings==2.11.0", "psycopg2-binary==2.9.10", "watchfiles==1.1.0", "asyncpg>=0.30.0", "PyPika>=0.51.1", "grpcio-tools==1.75.1", "grpc-stubs==1.53.0.6", "protobuf==6.32.1", "pytest>=8.4.2", "pytest-asyncio==0.24.0", "pytest-cov==5.0.0", "pytest-mock==3.15.1", "pytest-faker==2.0.0", "Faker==37.11.0", "black==25.9.0", "isort==6.0.1", "flake8==7.3.0", "flake8-pyproject==1.2.3", "ruff==0.13.2", "mypy==1.18.2", "types-protobuf==6.32.1.20250918", "types-psycopg2==2.9.21.20250915", "scikit-learn>=1.6.1", "torch>=2.7.1", "optimum[onnxruntime]>=2.1.0", "tiktoken>=0.12.0", "sentencepiece>=0.2.1", "huggingface-hub>=0.36.2", "onnxruntime>=1.25.0"] requires-python = ">=3.11" readme = "README.md" license = {text = "GNU GENERAL PUBLIC LICENSE"} diff --git a/tests/behavioral/test_golden_path.py b/tests/behavioral/test_golden_path.py new file mode 100644 index 0000000..74a065b --- /dev/null +++ b/tests/behavioral/test_golden_path.py @@ -0,0 +1,27 @@ +import pytest + +@pytest.mark.asyncio +async def test_high_logit_returns_true(toxic_service): + result = await toxic_service.moderate("arbitrary text") + assert result is True + + +@pytest.mark.asyncio +async def test_low_logit_returns_false(clean_service): + result = await clean_service.moderate("arbitrary text") + assert result is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "text", + [ + "", + "Привет こんにちは مرحبا", + "word " * 300, + ], +) +async def test_crash_safety_inputs_return_boolean(clean_service, text): + result = await clean_service.moderate(text) + assert isinstance(result, bool) + diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..09728a9 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,47 @@ +import pytest +import torch +from unittest.mock import MagicMock +from aivalidatorservice.model import loader +from aivalidatorservice.model.loader import ModerationModel +from aivalidatorservice.service.service import ModerationService + +def make_fake_session(logit: float) -> MagicMock: + session = MagicMock() + outputs = MagicMock() + outputs.logits = torch.tensor([[0.0, logit]], dtype=torch.float32) + session.return_value = outputs + return session + +def make_fake_tokenizer(max_length: int = 128) -> MagicMock: + tokenizer = MagicMock() + tokenizer.return_value = { + "input_ids": torch.ones((1, max_length), dtype=torch.int64), + "attention_mask": torch.ones((1, max_length), dtype=torch.int64), + } + return tokenizer + +@pytest.fixture(scope="session") +def real_tokenizer(): + return loader.AutoTokenizer.from_pretrained("microsoft/deberta-v3-small") + +@pytest.fixture +def toxic_service(monkeypatch): + monkeypatch.setattr(loader.AutoTokenizer, "from_pretrained", lambda _: make_fake_tokenizer()) + monkeypatch.setattr( + loader.ORTModelForSequenceClassification, + "from_pretrained", + lambda *args, **kwargs: make_fake_session(logit=5.0), + ) + model = ModerationModel(model_path="mock-model", tokenizer_path="mock-tokenizer") + return ModerationService(model) + +@pytest.fixture +def clean_service(monkeypatch): + monkeypatch.setattr(loader.AutoTokenizer, "from_pretrained", lambda _: make_fake_tokenizer()) + monkeypatch.setattr( + loader.ORTModelForSequenceClassification, + "from_pretrained", + lambda *args, **kwargs: make_fake_session(logit=-5.0), + ) + model = ModerationModel(model_path="mock-model", tokenizer_path="mock-tokenizer") + return ModerationService(model) \ No newline at end of file diff --git a/tests/contract/test_grpc_contract.py b/tests/contract/test_grpc_contract.py new file mode 100644 index 0000000..8a903de --- /dev/null +++ b/tests/contract/test_grpc_contract.py @@ -0,0 +1,16 @@ +import pytest + +from aivalidatorservice.grpc import moderation_pb2 +from aivalidatorservice.handler.handler import ModerationHandler + +@pytest.mark.asyncio +async def test_grpc_contract_toxic_input_returns_expected_response(toxic_service): + request = moderation_pb2.ModerateObjectRequest( + id=123, + type=moderation_pb2.OBJECT_TYPE_COMMENT_TEXT, + text="Ihateyou", + ) + handler = ModerationHandler(toxic_service) + response = await handler.ModerateObject(request, context=None) + assert isinstance(response, moderation_pb2.ModerateObjectResponse) + assert response.success is True diff --git a/tests/tokenizer/test_tokenizer_shapes.py b/tests/tokenizer/test_tokenizer_shapes.py new file mode 100644 index 0000000..d48ec77 --- /dev/null +++ b/tests/tokenizer/test_tokenizer_shapes.py @@ -0,0 +1,4 @@ +def test_tokenizer_truncates_to_max_length_128(real_tokenizer): + out = real_tokenizer("token " * 400, return_tensors="pt", max_length=128, truncation=True) + assert out["input_ids"].shape == (1, 128) + assert out["attention_mask"].shape == (1, 128) diff --git a/tools/common.just b/tools/common.just index 052f282..ebb7076 100644 --- a/tools/common.just +++ b/tools/common.just @@ -39,9 +39,7 @@ lint: mypy --strict . test: - pytest \ - --cov=src/{{SOURCE}} \ - --cov-report=xml:coverage.xml + pdm run pytest tests/ prepare: format lint test From 72eb853a0853cdb5d0b894f4292203eb1978350c Mon Sep 17 00:00:00 2001 From: Samuel Date: Sun, 26 Apr 2026 13:47:25 +0200 Subject: [PATCH 2/8] scipy dependency fix for CI --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1f401be..3df29c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "ai-validator-service" version = "0.1.0" description = "Default template for PDM package" -dependencies = ["grpcio==1.75.1", "transformers>=4.57.6", "tokenizers>=0.21.4", "grpcio-reflection==1.75.1", "pydantic==2.11.9", "pydantic-settings==2.11.0", "psycopg2-binary==2.9.10", "watchfiles==1.1.0", "asyncpg>=0.30.0", "PyPika>=0.51.1", "grpcio-tools==1.75.1", "grpc-stubs==1.53.0.6", "protobuf==6.32.1", "pytest>=8.4.2", "pytest-asyncio==0.24.0", "pytest-cov==5.0.0", "pytest-mock==3.15.1", "pytest-faker==2.0.0", "Faker==37.11.0", "black==25.9.0", "isort==6.0.1", "flake8==7.3.0", "flake8-pyproject==1.2.3", "ruff==0.13.2", "mypy==1.18.2", "types-protobuf==6.32.1.20250918", "types-psycopg2==2.9.21.20250915", "scikit-learn>=1.6.1", "torch>=2.7.1", "optimum[onnxruntime]>=2.1.0", "tiktoken>=0.12.0", "sentencepiece>=0.2.1", "huggingface-hub>=0.36.2", "onnxruntime>=1.25.0"] +dependencies = ["scipy>=1.14.0", "grpcio==1.75.1", "transformers>=4.57.6", "tokenizers>=0.21.4", "grpcio-reflection==1.75.1", "pydantic==2.11.9", "pydantic-settings==2.11.0", "psycopg2-binary==2.9.10", "watchfiles==1.1.0", "asyncpg>=0.30.0", "PyPika>=0.51.1", "grpcio-tools==1.75.1", "grpc-stubs==1.53.0.6", "protobuf==6.32.1", "pytest>=8.4.2", "pytest-asyncio==0.24.0", "pytest-cov==5.0.0", "pytest-mock==3.15.1", "pytest-faker==2.0.0", "Faker==37.11.0", "black==25.9.0", "isort==6.0.1", "flake8==7.3.0", "flake8-pyproject==1.2.3", "ruff==0.13.2", "mypy==1.18.2", "types-protobuf==6.32.1.20250918", "types-psycopg2==2.9.21.20250915", "scikit-learn>=1.6.1", "torch>=2.7.1", "optimum[onnxruntime]>=2.1.0", "tiktoken>=0.12.0", "sentencepiece>=0.2.1", "huggingface-hub>=0.36.2", "onnxruntime>=1.25.0"] requires-python = ">=3.11" readme = "README.md" license = {text = "GNU GENERAL PUBLIC LICENSE"} From 6eaec6e62470ff727fdbbf08ba24056549b3749b Mon Sep 17 00:00:00 2001 From: Samuel Date: Fri, 1 May 2026 22:18:37 +0200 Subject: [PATCH 3/8] implemented logger (currently an import bug), formatted and linted (mypy complains not fixed yet) --- data/augment_dataset.py | 190 ++-- data/prepare_dataset.py | 194 ++-- logger/__init__.py | 1 + logger/custom_logger.py | 39 + pdm.lock | 170 ++-- pyproject.toml | 157 +++- src/aivalidatorservice/constants.py | 11 +- src/aivalidatorservice/grpc/moderation_pb2.py | 34 +- .../grpc/moderation_pb2.pyi | 15 +- .../grpc/moderation_pb2_grpc.py | 78 +- src/aivalidatorservice/handler/handler.py | 12 +- src/aivalidatorservice/handler/moderate.py | 22 +- src/aivalidatorservice/model/loader.py | 18 +- src/aivalidatorservice/server.py | 20 +- src/aivalidatorservice/service/moderate.py | 9 +- src/aivalidatorservice/service/service.py | 15 +- tests/behavioral/test_golden_path.py | 2 +- tests/conftest.py | 35 +- tests/contract/test_grpc_contract.py | 5 +- tests/tokenizer/test_tokenizer_shapes.py | 4 +- tools/common.just | 16 +- ...alidator-model-pipeline-google-colab.ipynb | 848 +++++++++--------- trainer/google_colab_train.py | 181 ++-- trainer/onnx_exporter.py | 35 +- trainer/quantizer.py | 15 +- trainer/train_qat.py | 196 ++-- 26 files changed, 1441 insertions(+), 881 deletions(-) create mode 100644 logger/__init__.py create mode 100644 logger/custom_logger.py diff --git a/data/augment_dataset.py b/data/augment_dataset.py index 5199653..721b3a0 100644 --- a/data/augment_dataset.py +++ b/data/augment_dataset.py @@ -21,22 +21,22 @@ from __future__ import annotations import random -import re from collections import defaultdict -from typing import Callable import numpy as np import pandas as pd from sklearn.feature_extraction.text import TfidfVectorizer -MAX_SYNTHETIC_RATIO = 0.40 +from logger.custom_logger import get_logger -P_TOXIC_BOUNDARY = 0.85 # space between toxic word and any neighbour -P_ADJACENT_BOUNDARY = 0.50 # space one hop away from a toxic word -P_RANDOM_BOUNDARY = 0.15 # all other word boundaries +MAX_SYNTHETIC_RATIO = 0.40 + +P_TOXIC_BOUNDARY = 0.85 # space between toxic word and any neighbour +P_ADJACENT_BOUNDARY = 0.50 # space one hop away from a toxic word +P_RANDOM_BOUNDARY = 0.15 # all other word boundaries TOXIC_RATIO_HIGH = 0.70 -TOXIC_RATIO_LOW = 0.30 +TOXIC_RATIO_LOW = 0.30 TOP_N_TOXIC_WORDS = 300 @@ -44,10 +44,12 @@ random.seed(SEED) np.random.seed(SEED) +log = get_logger(__name__) - -def build_toxic_vocabulary(df: pd.DataFrame, top_n: int = TOP_N_TOXIC_WORDS) -> set[str]: +def build_toxic_vocabulary( + df: pd.DataFrame, top_n: int = TOP_N_TOXIC_WORDS +) -> set[str]: """ Bootstrap a toxic-word vocabulary from the combined dataframe using TF-IDF discrimination between label=1 and label=0 texts. @@ -68,7 +70,7 @@ def build_toxic_vocabulary(df: pd.DataFrame, top_n: int = TOP_N_TOXIC_WORDS) -> ) all_texts = toxic_texts + clean_texts - labels = [1] * len(toxic_texts) + [0] * len(clean_texts) + labels = [1] * len(toxic_texts) + [0] * len(clean_texts) tfidf_matrix = vectorizer.fit_transform(all_texts) feature_names = np.array(vectorizer.get_feature_names_out()) @@ -80,31 +82,36 @@ def build_toxic_vocabulary(df: pd.DataFrame, top_n: int = TOP_N_TOXIC_WORDS) -> clean_mean = np.asarray(tfidf_matrix[clean_mask].mean(axis=0)).flatten() discrimination = toxic_mean - clean_mean - top_indices = np.argsort(discrimination)[::-1][:top_n] - toxic_vocab = set(feature_names[top_indices].tolist()) + top_indices = np.argsort(discrimination)[::-1][:top_n] + toxic_vocab = set(feature_names[top_indices].tolist()) - print(f" [vocab] Bootstrapped {len(toxic_vocab)} toxic indicator words via TF-IDF") + log.info( + f"[vocab] Bootstrapped {len(toxic_vocab)} toxic indicator words via TF-IDF" + ) return toxic_vocab + def _tokenize(text: str) -> list[str]: return text.split() + def _is_toxic_word(word: str, vocab: set[str]) -> bool: return word.lower().strip(".,!?;:\"'") in vocab + def _classify_boundaries(tokens: list[str], vocab: set[str]) -> list[str]: n = len(tokens) if n <= 1: return [] is_toxic = [_is_toxic_word(t, vocab) for t in tokens] - classes = [] + classes = [] for i in range(n - 1): - left_toxic = is_toxic[i] + left_toxic = is_toxic[i] right_toxic = is_toxic[i + 1] - left_adj = (i > 0 and is_toxic[i - 1]) - right_adj = (i + 2 < n and is_toxic[i + 2]) + left_adj = i > 0 and is_toxic[i - 1] + right_adj = i + 2 < n and is_toxic[i + 2] if left_toxic or right_toxic: classes.append("toxic") @@ -118,14 +125,13 @@ def _classify_boundaries(tokens: list[str], vocab: set[str]) -> list[str]: def _remove_space(boundary_class: str) -> bool: p = { - "toxic": P_TOXIC_BOUNDARY, + "toxic": P_TOXIC_BOUNDARY, "adjacent": P_ADJACENT_BOUNDARY, - "random": P_RANDOM_BOUNDARY, + "random": P_RANDOM_BOUNDARY, }[boundary_class] return random.random() < p - def aug_full_concat(text: str, **_) -> str: return "".join(_tokenize(text)) @@ -136,9 +142,9 @@ def aug_partial_concat(text: str, vocab: set[str], **_) -> str: return text classes = _classify_boundaries(tokens, vocab) - result = tokens[0] + result = tokens[0] for i, cls in enumerate(classes): - sep = "" if _remove_space(cls) else " " + sep = "" if _remove_space(cls) else " " result += sep + tokens[i + 1] return result @@ -149,8 +155,8 @@ def aug_context_filler( n_fillers: int = 1, **_, ) -> str: - fillers = random.sample(filler_pool, min(n_fillers, len(filler_pool))) - parts = fillers + [text] + fillers = random.sample(filler_pool, min(n_fillers, len(filler_pool))) + parts = [*fillers, text] random.shuffle(parts) return " ".join(p.strip().rstrip(".") + "." for p in parts) @@ -161,12 +167,16 @@ def aug_crosslang_concat( lang_pool: dict[str, list[str]], **_, ) -> str | None: - other_langs = [l for l in lang_pool if l != lang and lang_pool[l]] + other_langs = [ + lang_code + for lang_code in lang_pool + if lang_code != lang and lang_pool[lang_code] + ] if not other_langs: return None other_lang = random.choice(other_langs) other_text = random.choice(lang_pool[other_lang]) - parts = [text.strip(), other_text.strip()] + parts = [text.strip(), other_text.strip()] random.shuffle(parts) sep = "" if random.random() < 0.5 else " " return sep.join(parts) @@ -196,12 +206,11 @@ def _build_context_mix_row( parts = [toxic_text.strip(), clean_text.strip()] random.shuffle(parts) - sep = "" if random.random() < 0.5 else " " + sep = "" if random.random() < 0.5 else " " text = sep.join(parts) return text, label, category - def _target_counts( real_df: pd.DataFrame, buckets: list[tuple[str, int]], @@ -213,16 +222,18 @@ def _target_counts( real_counts: dict[tuple[str, int], int] = defaultdict(int) for cat, lbl in buckets: mask = (real_df["category"] == cat) & (real_df["label"] == lbl) - real_counts[(cat, lbl)] = int(mask.sum()) + real_counts[cat, lbl] = int(mask.sum()) - max_real = max(real_counts.values()) if real_counts else 0 - targets = {} + max_real = max(real_counts.values()) if real_counts else 0 + targets = {} for key, cnt in real_counts.items(): targets[key] = max(0, max_real - cnt) - total_real = len(real_df) + total_real = len(real_df) total_synthetic = sum(targets.values()) - max_allowed = int(total_real * MAX_SYNTHETIC_RATIO / (1 - MAX_SYNTHETIC_RATIO)) + max_allowed = int( + total_real * MAX_SYNTHETIC_RATIO / (1 - MAX_SYNTHETIC_RATIO) + ) if total_synthetic > max_allowed and total_synthetic > 0: scale = max_allowed / total_synthetic @@ -231,55 +242,57 @@ def _target_counts( return targets - def build_augmented_dataset(df: pd.DataFrame) -> pd.DataFrame: df = df.copy() - df["label"] = (pd.to_numeric(df["label"], errors="coerce").fillna(0.0) >= 0.5).astype("int8") + df["label"] = ( + pd.to_numeric(df["label"], errors="coerce").fillna(0.0) >= 0.5 + ).astype("int8") - print("\n[augment] Building toxic vocabulary...") + log.info("[augment] Building toxic vocabulary...") vocab = build_toxic_vocabulary(df) toxic_df = df[df["label"] == 1].copy() clean_df = df[df["label"] == 0].copy() - toxic_texts = toxic_df["text"].astype(str).tolist() clean_texts = clean_df["text"].astype(str).tolist() lang_pool: dict[str, list[str]] = defaultdict(list) for _, row in toxic_df.iterrows(): lang_pool[row["lang"]].append(row["text"]) - filler_pool = clean_texts + filler_pool = clean_texts buckets = [ - ("concatenation", 0), - ("concatenation", 1), - ("context", 0), - ("context", 1), - ("context_ambiguous",0), - ("context_ambiguous",1), - ("general", 0), - ("general", 1), + ("concatenation", 0), + ("concatenation", 1), + ("context", 0), + ("context", 1), + ("context_ambiguous", 0), + ("context_ambiguous", 1), + ("general", 0), + ("general", 1), ] targets = _target_counts(df, buckets) - print(f"[augment] Synthetic targets per bucket: {dict(targets)}") + log.debug(f"[augment] Synthetic targets per bucket: {dict(targets)}") rows: list[dict] = [] def add_row(text, label, lang, source, category): text = str(text).strip() if 3 <= len(text) <= 512: - rows.append({ - "text": text, - "label": np.int8(label), - "lang": lang, - "source": source, - "category": category, - }) + rows.append( + { + "text": text, + "label": np.int8(label), + "lang": lang, + "source": source, + "category": category, + } + ) cat_lbl = ("concatenation", 1) n = targets.get(cat_lbl, 0) - print(f"[augment] synthetic_concat: generating {n} rows") + log.info(f"[augment] synthetic_concat: generating {n} rows") sample = random.choices(toxic_df.to_dict("records"), k=n) for rec in sample: text = aug_full_concat(rec["text"]) @@ -287,53 +300,72 @@ def add_row(text, label, lang, source, category): n_toxic = targets.get(("concatenation", 1), 0) n_clean = targets.get(("concatenation", 0), 0) - print(f"[augment] synthetic_partial_concat: {n_toxic} toxic, {n_clean} clean") + log.info( + f"[augment] synthetic_partial_concat: {n_toxic} toxic, {n_clean} clean" + ) for rec in random.choices(toxic_df.to_dict("records"), k=n_toxic): text = aug_partial_concat(rec["text"], vocab=vocab) - add_row(text, 1, rec["lang"], "synthetic_partial_concat", "concatenation") + add_row( + text, 1, rec["lang"], "synthetic_partial_concat", "concatenation" + ) for rec in random.choices(clean_df.to_dict("records"), k=n_clean): text = aug_partial_concat(rec["text"], vocab=vocab) - add_row(text, 0, rec["lang"], "synthetic_partial_concat", "concatenation") + add_row( + text, 0, rec["lang"], "synthetic_partial_concat", "concatenation" + ) cat_lbl = ("context", 1) n = targets.get(cat_lbl, 0) - print(f"[augment] synthetic_context_filler: generating {n} rows") + log.info(f"[augment] synthetic_context_filler: generating {n} rows") for rec in random.choices(toxic_df.to_dict("records"), k=n): - text = aug_context_filler(rec["text"], filler_pool=filler_pool, n_fillers=random.randint(1, 2)) + text = aug_context_filler( + rec["text"], + filler_pool=filler_pool, + n_fillers=random.randint(1, 2), + ) add_row(text, 1, rec["lang"], "synthetic_context_filler", "context") - n_mix = targets.get(("context", 0), 0) + targets.get(("context_ambiguous", 0), 0) - print(f"[augment] synthetic_context_mix: generating ~{n_mix} rows") + n_mix = targets.get(("context", 0), 0) + targets.get( + ("context_ambiguous", 0), 0 + ) + log.info(f"[augment] synthetic_context_mix: generating ~{n_mix} rows") toxic_recs = random.choices(toxic_df.to_dict("records"), k=n_mix) clean_recs = random.choices(clean_df.to_dict("records"), k=n_mix) - for t_rec, c_rec in zip(toxic_recs, clean_recs): - text, label, category = _build_context_mix_row(t_rec["text"], c_rec["text"]) + for t_rec, c_rec in zip(toxic_recs, clean_recs, strict=False): + text, label, category = _build_context_mix_row( + t_rec["text"], c_rec["text"] + ) add_row(text, label, t_rec["lang"], "synthetic_context_mix", category) n = targets.get(("concatenation", 1), 0) // 2 # share budget with concat - print(f"[augment] synthetic_crosslang: generating {n} rows") + log.info(f"[augment] synthetic_crosslang: generating {n} rows") for rec in random.choices(toxic_df.to_dict("records"), k=n): - text = aug_crosslang_concat(rec["text"], lang=rec["lang"], lang_pool=lang_pool) + text = aug_crosslang_concat( + rec["text"], lang=rec["lang"], lang_pool=lang_pool + ) if text: add_row(text, 1, "mixed", "synthetic_crosslang", "concatenation") synth_df = pd.DataFrame(rows) if synth_df.empty: - print("[augment] Warning: no synthetic rows generated.") + log.error("[augment] no synthetic rows generated.") return synth_df synth_df["label"] = synth_df["label"].astype("int8") - print(f"\n[augment] Synthetic rows generated: {len(synth_df):,}") - print(synth_df.groupby(["source", "category", "label"]).size().to_string()) + log.info(f"[augment] Synthetic rows generated: {len(synth_df):,}") + log.debug( + synth_df.groupby(["source", "category", "label"]).size().to_string() + ) return synth_df - def augment_and_combine(real_df: pd.DataFrame) -> pd.DataFrame: real_df = real_df.copy() - real_df["label"] = (pd.to_numeric(real_df["label"], errors="coerce").fillna(0.0) >= 0.5).astype("int8") + real_df["label"] = ( + pd.to_numeric(real_df["label"], errors="coerce").fillna(0.0) >= 0.5 + ).astype("int8") synth_df = build_augmented_dataset(real_df) @@ -341,13 +373,17 @@ def augment_and_combine(real_df: pd.DataFrame) -> pd.DataFrame: combined["text"] = combined["text"].astype(str).str.strip() combined = combined[combined["text"].str.len().between(3, 512)] combined = combined.dropna(subset=["text", "label"]) - combined["label"] = (pd.to_numeric(combined["label"], errors="coerce").fillna(0.0) >= 0.5).astype("int8") + combined["label"] = ( + pd.to_numeric(combined["label"], errors="coerce").fillna(0.0) >= 0.5 + ).astype("int8") combined["_key"] = combined["text"].str.lower() combined = combined.drop_duplicates(subset="_key").drop(columns="_key") - print(f"\n[augment] Final combined dataset: {len(combined):,} rows") - print(f" Real: {len(real_df):,}") - print(f" Synthetic: {len(combined) - len(real_df):,}") - print(f" Synthetic ratio: {(len(combined) - len(real_df)) / len(combined):.1%}") - return combined \ No newline at end of file + log.info(f"[augment] Final combined dataset: {len(combined):,} rows") + log.info(f"[augment] Real: {len(real_df):,}") + log.info(f"[augment] Synthetic: {len(combined) - len(real_df):,}") + log.info( + f" Synthetic ratio: {(len(combined) - len(real_df)) / len(combined):.1%}" + ) + return combined diff --git a/data/prepare_dataset.py b/data/prepare_dataset.py index 58e5df4..8898b70 100644 --- a/data/prepare_dataset.py +++ b/data/prepare_dataset.py @@ -4,38 +4,41 @@ All datasets are pulled from HuggingFace: esclient/toxicity_multilanguage_dataset """ -from pyexpat import model + +from pathlib import Path import pandas as pd +from augment_dataset import augment_and_combine from datasets import load_dataset from huggingface_hub import hf_hub_download from sklearn.model_selection import train_test_split -from pathlib import Path -from augment_dataset import augment_and_combine + +from logger.custom_logger import get_logger OUT = Path("data/processed") OUT.mkdir(parents=True, exist_ok=True) +log = get_logger(__name__) -HF_REPO_ID = "esclient/toxicity_multilanguage_dataset" +HF_REPO_ID = "esclient/toxicity_multilanguage_dataset" HF_REPO_TYPE = "dataset" # Language code → ISO 639-1 tag for the parquet shards PARQUET_LANGS: dict[str, str] = { - "am": "am", # Amharic - "ar": "ar", # Arabic - "de": "de", # German - "en": "en", # English - "es": "es", # Spanish - "fr": "fr", # French - "he": "he", # Hebrew - "hi": "hi", # Hindi - "hin": "hi", # Hindi (alternate shard, same ISO code) - "it": "it", # Italian - "ja": "ja", # Japanese - "ru": "ru", # Russian - "tt": "tt", # Tatar - "uk": "uk", # Ukrainian - "zh": "zh", # Chinese + "am": "am", # Amharic + "ar": "ar", # Arabic + "de": "de", # German + "en": "en", # English + "es": "es", # Spanish + "fr": "fr", # French + "he": "he", # Hebrew + "hi": "hi", # Hindi + "hin": "hi", # Hindi (alternate shard, same ISO code) + "it": "it", # Italian + "ja": "ja", # Japanese + "ru": "ru", # Russian + "tt": "tt", # Tatar + "uk": "uk", # Ukrainian + "zh": "zh", # Chinese } @@ -56,21 +59,29 @@ def base_frame( ) -> pd.DataFrame: return pd.DataFrame( { - "text": pd.Series(text, dtype="object"), - "label": pd.Series(label, dtype="int8"), - "lang": lang, - "source": source, + "text": pd.Series(text, dtype="object"), + "label": pd.Series(label, dtype="int8"), + "lang": lang, + "source": source, "category": category, } ) + def _to_binary_label(raw: pd.Series) -> pd.Series: numeric = pd.to_numeric(raw, errors="coerce").fillna(0.0) return (numeric >= 0.5).astype("int8") def _label_from_parquet(df: pd.DataFrame) -> pd.Series: - for col in ("toxic", "toxicity", "label", "is_toxic", "target", "inappropriate"): + for col in ( + "toxic", + "toxicity", + "label", + "is_toxic", + "target", + "inappropriate", + ): if col in df.columns: raw = df[col] try: @@ -79,14 +90,22 @@ def _label_from_parquet(df: pd.DataFrame) -> pd.Series: raise KeyError( f"Column '{col}' found but could not be cast to float: {exc}. " f"Sample values: {raw.dropna().head(5).tolist()}" - ) + ) from exc raise KeyError( f"No known label column found. Available columns: {list(df.columns)}" ) def _text_col(df: pd.DataFrame) -> pd.Series: - for col in ("text", "comment", "comments", "comment_text", "content", "message", "sentence"): + for col in ( + "text", + "comment", + "comments", + "comment_text", + "content", + "message", + "sentence", + ): if col in df.columns: return df[col] raise KeyError( @@ -109,14 +128,14 @@ def load_parquet_shards() -> pd.DataFrame: filename = f"{prefix}-00000-of-00001.parquet" try: local_path = _hf_download(filename) - raw = pd.read_parquet(local_path) - text = _text_col(raw) + raw = pd.read_parquet(local_path) + text = _text_col(raw) label = _label_from_parquet(raw) - df = base_frame(text, label, lang_code, f"parquet_{prefix}") + df = base_frame(text, label, lang_code, f"parquet_{prefix}") frames.append(df) - print(f" {filename}: {len(df):,} rows (lang={lang_code})") + log.info(f"{filename}: {len(df):,} rows (lang={lang_code})") except Exception as exc: - print(f" [FAILED] {filename}: {exc}") + log.error(f" failed in loading {filename}: {exc}") if not frames: raise RuntimeError("No parquet shards loaded — check HF repo access.") @@ -127,29 +146,40 @@ def load_parquet_shards() -> pd.DataFrame: def load_inappropriate_messages() -> pd.DataFrame: local_path = _hf_download("Inappapropriate_messages.csv") raw = pd.read_csv(local_path) - print(f" CSV columns: {list(raw.columns)}") + log.debug(f"CSV columns: {list(raw.columns)}") text = _text_col(raw) try: label = _label_from_parquet(raw) except KeyError: - print(" No label column in CSV — treating all rows as toxic (1)") + log.info("No label column in CSV - treating all rows as toxic (1)") label = pd.Series([1] * len(raw), dtype="int8") - lang_col = next((c for c in raw.columns if c.lower() in ("lang", "language", "locale")), None) + lang_col = next( + ( + c + for c in raw.columns + if c.lower() in ("lang", "language", "locale") + ), + None, + ) lang = raw[lang_col].fillna("ru").astype(str) if lang_col else "ru" - return base_frame(text, label, lang, "inappropriate_messages", "inappropriate") + return base_frame( + text, label, lang, "inappropriate_messages", "inappropriate" + ) # ── 4. labled.csv (abusive True/False) ──────────────────────────────────────── def load_labled_csv() -> pd.DataFrame: local_path = _hf_download("labled.csv") raw = pd.read_csv(local_path) - print(f" CSV columns: {list(raw.columns)}") - text = _text_col(raw) - label = _to_binary_label(raw["abusive"].map({"True": 1, "False": 0, True: 1, False: 0})) + log.debug(f"CSV columns: {list(raw.columns)}") + text = _text_col(raw) + label = _to_binary_label( + raw["abusive"].map({"True": 1, "False": 0, True: 1, False: 0}) + ) return base_frame(text, label, "ru", "labled_csv") @@ -157,8 +187,8 @@ def load_labled_csv() -> pd.DataFrame: def load_russian_jsonl() -> pd.DataFrame: local_path = _hf_download("russian_dataset.jsonl") raw = pd.read_json(local_path, lines=True) - print(f" JSONL columns: {list(raw.columns)}") - text = _text_col(raw) + log.debug(f"JSONL columns: {list(raw.columns)}") + text = _text_col(raw) label = _label_from_parquet(raw) return base_frame(text, label, "ru", "russian_jsonl") @@ -167,14 +197,22 @@ def load_russian_jsonl() -> pd.DataFrame: def load_russian_tsv2() -> pd.DataFrame: local_path = _hf_download("russian_dataset_2.tsv") raw = pd.read_csv(local_path, sep="\t") - print(f" TSV columns: {list(raw.columns)}") - - toxic = base_frame(raw["ru_toxic_comment"], - pd.Series([1] * len(raw), dtype="int8"), - "ru", "russian_tsv2", "general") - clean = base_frame(raw["ru_neutral_comment"], - pd.Series([0] * len(raw), dtype="int8"), - "ru", "russian_tsv2", "general") + log.debug(f"TSV columns: {list(raw.columns)}") + + toxic = base_frame( + raw["ru_toxic_comment"], + pd.Series([1] * len(raw), dtype="int8"), + "ru", + "russian_tsv2", + "general", + ) + clean = base_frame( + raw["ru_neutral_comment"], + pd.Series([0] * len(raw), dtype="int8"), + "ru", + "russian_tsv2", + "general", + ) return pd.concat([toxic, clean], ignore_index=True) @@ -182,9 +220,9 @@ def load_russian_tsv2() -> pd.DataFrame: def load_russian_distorted() -> pd.DataFrame: local_path = _hf_download("russian_distorted_toxicity.tsv") raw = pd.read_csv(local_path, sep="\t") - print(f" TSV columns: {list(raw.columns)}") + log.debug(f"TSV columns: {list(raw.columns)}") - raw = raw.dropna(subset=["comments", "toxicity"]) + raw = raw.dropna(subset=["comments", "toxicity"]) label = _to_binary_label(raw["toxicity"]) return base_frame(raw["comments"], label, "ru", "russian_distorted") @@ -193,8 +231,8 @@ def load_russian_distorted() -> pd.DataFrame: def load_russian_comments_2ch_pikabu() -> pd.DataFrame: local_path = _hf_download("russian_comments_from_2ch_pikabu.csv") raw = pd.read_csv(local_path) - print(f" CSV columns: {list(raw.columns)}") - text = _text_col(raw) + log.debug(f"CSV columns: {list(raw.columns)}") + text = _text_col(raw) label = _label_from_parquet(raw) return base_frame(text, label, "ru", "russian_comments_2ch_pikabu") @@ -203,8 +241,8 @@ def load_russian_comments_2ch_pikabu() -> pd.DataFrame: def load_labeled_csv() -> pd.DataFrame: local_path = _hf_download("labeled.csv") raw = pd.read_csv(local_path) - print(f" CSV columns: {list(raw.columns)}") - text = _text_col(raw) + log.debug(f"CSV columns: {list(raw.columns)}") + text = _text_col(raw) label = _label_from_parquet(raw) return base_frame(text, label, "ru", "labeled_csv") @@ -214,7 +252,7 @@ def load_russian_dataset_3() -> pd.DataFrame: local_path = _hf_download("russian_dataset_3.txt") rows = [] - with open(local_path, "r", encoding="utf-8") as f: + with open(local_path, encoding="utf-8") as f: for line in f: line = line.strip() if not line: @@ -223,35 +261,41 @@ def load_russian_dataset_3() -> pd.DataFrame: if len(parts) < 2: continue labels_part, text = parts[0], parts[1] - labels = [p.strip() for p in labels_part.split(",") if p.strip()] + labels = [p.strip() for p in labels_part.split(",") if p.strip()] is_toxic = any(lbl != "__label__NORMAL" for lbl in labels) rows.append((text, 1 if is_toxic else 0)) raw = pd.DataFrame(rows, columns=["text", "label"]) - return base_frame(raw["text"], _to_binary_label(raw["label"]), "ru", "russian_dataset_3_txt") + return base_frame( + raw["text"], + _to_binary_label(raw["label"]), + "ru", + "russian_dataset_3_txt", + ) + def run() -> None: - print("Loading datasets...\n") + log.info("Loading datasets...") frames: list[pd.DataFrame] = [] for loader in [load_civil_comments]: try: df = loader() - print(f" {loader.__name__}: {len(df):,} rows") + log.info(f"{loader.__name__}: {len(df):,} rows") frames.append(df) except Exception as exc: - print(f" {loader.__name__} FAILED: {exc}") + log.error(f"{loader.__name__} FAILED: {exc}") - print("\n Loading parquet shards:") + log.info("Loading parquet shards") try: df = load_parquet_shards() - print(f" → parquet total: {len(df):,} rows") + log.info(f"Parquet total: {len(df):,} rows") frames.append(df) except Exception as exc: - print(f" Parquet shards FAILED: {exc}") + log.error(f"Parquet shards FAILED: {exc}") - print("\n Loading local files:") + log.info("Loading local files") local_loaders = [ load_inappropriate_messages, load_labled_csv, @@ -265,17 +309,17 @@ def run() -> None: for loader in local_loaders: try: df = loader() - print(f" {loader.__name__}: {len(df):,} rows") + log.info(f"{loader.__name__}: {len(df):,} rows") frames.append(df) except Exception as exc: - print(f" {loader.__name__} FAILED: {exc}") + log.error(f"{loader.__name__} FAILED: {exc}") if not frames: raise RuntimeError("All loaders failed — nothing to process.") df = pd.concat(frames, ignore_index=True) df = augment_and_combine(df) - print(f"\nRaw combined: {len(df):,} rows") + log.info(f"Raw combined: {len(df):,} rows") # ── quality filter ──────────────────────────────────────────────────────── df["text"] = df["text"].astype(str).str.strip() @@ -285,12 +329,14 @@ def run() -> None: df["_key"] = df["text"].str.lower() df = df.drop_duplicates(subset="_key").drop(columns="_key") - print(f"After dedup + filter: {len(df):,} rows") + log.info(f"After dedup + filter: {len(df):,} rows") - print("\nLabel distribution:") - print(df["label"].value_counts().rename({0: "clean", 1: "toxic"})) - print("\nLanguage distribution:") - print(df["lang"].value_counts()) + log.info("Label distribution") + log.debug( + df["label"].value_counts().rename({0: "clean", 1: "toxic"}).to_string() + ) + log.info("Language distribution") + log.debug(df["lang"].value_counts().to_string()) # ── stratified 80 / 10 / 10 split ──────────────────────────────────────── train, tmp = train_test_split( @@ -303,8 +349,8 @@ def run() -> None: for split, name in [(train, "train"), (val, "val"), (test, "test")]: path = OUT / f"{name}.parquet" split.reset_index(drop=True).to_parquet(path, index=False) - print(f"Saved {name}: {len(split):,} rows → {path}") + log.info(f"Saved {name}: {len(split):,} rows -> {path}") if __name__ == "__main__": - run() \ No newline at end of file + run() diff --git a/logger/__init__.py b/logger/__init__.py new file mode 100644 index 0000000..6b9d48a --- /dev/null +++ b/logger/__init__.py @@ -0,0 +1 @@ +from .custom_logger import get_logger \ No newline at end of file diff --git a/logger/custom_logger.py b/logger/custom_logger.py new file mode 100644 index 0000000..0ad69ba --- /dev/null +++ b/logger/custom_logger.py @@ -0,0 +1,39 @@ +import datetime +import logging + + +class AbseilStyleFormatter(logging.Formatter): + def format(self, record): + level = record.levelname + location = f"{record.filename}:{record.lineno}" + timespan = ( + datetime.datetime.now(datetime.UTC).strftime( + "%Y-%m-%dT%H:%M:%S.%f" + )[:-3] + + "Z" + ) + + subsystem = getattr(record, "subsystem", "app") + code = getattr(record, "code", "") + explanation = getattr(record, "explanation", "") + + msg = f"[{level}] {location} timespan={timespan} subsystem={subsystem}" + if code: + msg += f" code={code}" + msg += f' message="{record.getMessage()}"' + if explanation: + msg += f' explanation="{explanation}"' + + return msg + + +def get_logger(name: str) -> logging.Logger: + logger = logging.getLogger(name) + logger.setLevel(logging.INFO) + + if not logger.handlers: + handler = logging.StreamHandler() + handler.setFormatter(AbseilStyleFormatter()) + logger.addHandler(handler) + + return logger diff --git a/pdm.lock b/pdm.lock index 6b8cb92..4f8deb3 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "quantize"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:091bbe537cc7f8d027999c2849e85334761b181bb6ec234533139f03b92e8945" +content_hash = "sha256:c1ae26977be3fe63cf83b44a00c672b4aff02908a424825fcb98bca623af4539" [[metadata.targets]] requires_python = ">=3.11" @@ -290,7 +290,7 @@ name = "black" version = "25.9.0" requires_python = ">=3.9" summary = "The uncompromising code formatter." -groups = ["default"] +groups = ["dev"] dependencies = [ "click>=8.0.0", "mypy-extensions>=0.4.3", @@ -433,7 +433,7 @@ name = "click" version = "8.1.8" requires_python = ">=3.7" summary = "Composable command line interface toolkit" -groups = ["default"] +groups = ["dev"] dependencies = [ "colorama; platform_system == \"Windows\"", "importlib-metadata; python_version < \"3.8\"", @@ -460,7 +460,7 @@ name = "coverage" version = "7.10.7" requires_python = ">=3.9" summary = "Code coverage measurement for Python" -groups = ["default"] +groups = ["dev"] files = [ {file = "coverage-7.10.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fc04cc7a3db33664e0c2d10eb8990ff6b3536f6842c9590ae8da4c614b9ed05a"}, {file = "coverage-7.10.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e201e015644e207139f7e2351980feb7040e6f4b2c2978892f3e3789d1c125e5"}, @@ -574,7 +574,7 @@ version = "7.10.7" extras = ["toml"] requires_python = ">=3.9" summary = "Code coverage measurement for Python" -groups = ["default"] +groups = ["dev"] dependencies = [ "coverage==7.10.7", "tomli; python_full_version <= \"3.11.0a6\"", @@ -728,7 +728,7 @@ name = "faker" version = "37.11.0" requires_python = ">=3.9" summary = "Faker is a Python package that generates fake data for you." -groups = ["default"] +groups = ["dev"] dependencies = [ "tzdata", ] @@ -753,7 +753,7 @@ name = "flake8" version = "7.3.0" requires_python = ">=3.9" summary = "the modular source code checker: pep8 pyflakes and co" -groups = ["default"] +groups = ["dev"] dependencies = [ "mccabe<0.8.0,>=0.7.0", "pycodestyle<2.15.0,>=2.14.0", @@ -766,17 +766,17 @@ files = [ [[package]] name = "flake8-pyproject" -version = "1.2.3" -requires_python = ">= 3.6" +version = "1.2.4" +requires_python = ">=3.6" summary = "Flake8 plug-in loading the configuration from pyproject.toml" -groups = ["default"] +groups = ["dev"] dependencies = [ "Flake8>=5", "TOMLi; python_version < \"3.11\"", "TOMLi<2; python_version < \"3.7\"", ] files = [ - {file = "flake8_pyproject-1.2.3-py3-none-any.whl", hash = "sha256:6249fe53545205af5e76837644dc80b4c10037e73a0e5db87ff562d75fb5bd4a"}, + {file = "flake8_pyproject-1.2.4-py3-none-any.whl", hash = "sha256:ea34c057f9a9329c76d98723bb2bb498cc6ba8ff9872c4d19932d48c91249a77"}, ] [[package]] @@ -959,7 +959,7 @@ name = "grpc-stubs" version = "1.53.0.6" requires_python = ">=3.6" summary = "Mypy stubs for gRPC" -groups = ["default"] +groups = ["dev"] dependencies = [ "grpcio", ] @@ -973,7 +973,7 @@ name = "grpcio" version = "1.75.1" requires_python = ">=3.9" summary = "HTTP/2-based RPC framework" -groups = ["default"] +groups = ["default", "dev"] dependencies = [ "typing-extensions~=4.12", ] @@ -1061,7 +1061,7 @@ name = "grpcio-tools" version = "1.75.1" requires_python = ">=3.9" summary = "Protobuf code generator for gRPC" -groups = ["default"] +groups = ["dev"] dependencies = [ "grpcio>=1.75.1", "protobuf<7.0.0,>=6.31.1", @@ -1203,7 +1203,7 @@ name = "iniconfig" version = "2.1.0" requires_python = ">=3.8" summary = "brain-dead simple config-ini parsing" -groups = ["default"] +groups = ["dev"] files = [ {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, @@ -1214,7 +1214,7 @@ name = "isort" version = "6.0.1" requires_python = ">=3.9.0" summary = "A Python utility / library to sort Python imports." -groups = ["default"] +groups = ["dev"] files = [ {file = "isort-6.0.1-py3-none-any.whl", hash = "sha256:2dc5d7f65c9678d94c88dfc29161a320eec67328bc97aad576874cb4be1e9615"}, {file = "isort-6.0.1.tar.gz", hash = "sha256:1cb5df28dfbc742e490c5e41bad6da41b805b0a8be7bc93cd0fb2a8a890ac450"}, @@ -1326,7 +1326,7 @@ name = "mccabe" version = "0.7.0" requires_python = ">=3.6" summary = "McCabe checker, plugin for flake8" -groups = ["default"] +groups = ["dev"] files = [ {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, @@ -1573,7 +1573,7 @@ name = "mypy" version = "1.18.2" requires_python = ">=3.9" summary = "Optional static typing for Python" -groups = ["default"] +groups = ["dev"] dependencies = [ "mypy-extensions>=1.0.0", "pathspec>=0.9.0", @@ -1626,7 +1626,7 @@ name = "mypy-extensions" version = "1.1.0" requires_python = ">=3.8" summary = "Type system extensions for programs checked with the mypy type checker." -groups = ["default"] +groups = ["dev"] files = [ {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, @@ -2176,7 +2176,7 @@ name = "pathspec" version = "1.0.4" requires_python = ">=3.9" summary = "Utility library for gitignore style pattern matching of file paths." -groups = ["default"] +groups = ["dev"] files = [ {file = "pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723"}, {file = "pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645"}, @@ -2187,7 +2187,7 @@ name = "platformdirs" version = "4.4.0" requires_python = ">=3.9" summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." -groups = ["default"] +groups = ["dev"] files = [ {file = "platformdirs-4.4.0-py3-none-any.whl", hash = "sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85"}, {file = "platformdirs-4.4.0.tar.gz", hash = "sha256:ca753cf4d81dc309bc67b0ea38fd15dc97bc30ce419a7f58d13eb3bf14c4febf"}, @@ -2198,7 +2198,7 @@ name = "pluggy" version = "1.6.0" requires_python = ">=3.9" summary = "plugin and hook calling mechanisms for python" -groups = ["default"] +groups = ["dev"] files = [ {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, @@ -2340,7 +2340,7 @@ name = "protobuf" version = "6.32.1" requires_python = ">=3.9" summary = "" -groups = ["default", "quantize"] +groups = ["default", "dev", "quantize"] files = [ {file = "protobuf-6.32.1-cp310-abi3-win32.whl", hash = "sha256:a8a32a84bc9f2aad712041b8b366190f71dde248926da517bde9e832e4412085"}, {file = "protobuf-6.32.1-cp310-abi3-win_amd64.whl", hash = "sha256:b00a7d8c25fa471f16bc8153d0e53d6c9e827f0953f3c09aaa4331c718cae5e1"}, @@ -2490,7 +2490,7 @@ name = "pycodestyle" version = "2.14.0" requires_python = ">=3.9" summary = "Python style guide checker" -groups = ["default"] +groups = ["dev"] files = [ {file = "pycodestyle-2.14.0-py2.py3-none-any.whl", hash = "sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d"}, {file = "pycodestyle-2.14.0.tar.gz", hash = "sha256:c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783"}, @@ -2645,7 +2645,7 @@ name = "pyflakes" version = "3.4.0" requires_python = ">=3.9" summary = "passive checker of Python programs" -groups = ["default"] +groups = ["dev"] files = [ {file = "pyflakes-3.4.0-py2.py3-none-any.whl", hash = "sha256:f742a7dbd0d9cb9ea41e9a24a918996e8170c799fa528688d40dd582c8265f4f"}, {file = "pyflakes-3.4.0.tar.gz", hash = "sha256:b24f96fafb7d2ab0ec5075b7350b3d2d2218eab42003821c06344973d3ea2f58"}, @@ -2656,7 +2656,7 @@ name = "pygments" version = "2.20.0" requires_python = ">=3.9" summary = "Pygments is a syntax highlighting package written in Python." -groups = ["default"] +groups = ["dev"] files = [ {file = "pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176"}, {file = "pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"}, @@ -2680,7 +2680,7 @@ name = "pytest" version = "8.4.2" requires_python = ">=3.9" summary = "pytest: simple powerful testing with Python" -groups = ["default"] +groups = ["dev"] dependencies = [ "colorama>=0.4; sys_platform == \"win32\"", "exceptiongroup>=1; python_version < \"3.11\"", @@ -2700,7 +2700,7 @@ name = "pytest-asyncio" version = "0.24.0" requires_python = ">=3.8" summary = "Pytest support for asyncio" -groups = ["default"] +groups = ["dev"] dependencies = [ "pytest<9,>=8.2", ] @@ -2714,7 +2714,7 @@ name = "pytest-cov" version = "5.0.0" requires_python = ">=3.8" summary = "Pytest plugin for measuring coverage." -groups = ["default"] +groups = ["dev"] dependencies = [ "coverage[toml]>=5.2.1", "pytest>=4.6", @@ -2728,7 +2728,7 @@ files = [ name = "pytest-faker" version = "2.0.0" summary = "Faker integration with the pytest framework." -groups = ["default"] +groups = ["dev"] dependencies = [ "Faker>=0.7.3", ] @@ -2741,7 +2741,7 @@ name = "pytest-mock" version = "3.15.1" requires_python = ">=3.9" summary = "Thin-wrapper around the mock package for easier use with pytest" -groups = ["default"] +groups = ["dev"] dependencies = [ "pytest>=6.2.5", ] @@ -2780,7 +2780,7 @@ name = "pytokens" version = "0.4.1" requires_python = ">=3.8" summary = "A Fast, spec compliant Python 3.14+ tokenizer that runs on older Pythons." -groups = ["default"] +groups = ["dev"] files = [ {file = "pytokens-0.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a44ed93ea23415c54f3face3b65ef2b844d96aeb3455b8a69b3df6beab6acc5"}, {file = "pytokens-0.4.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:add8bf86b71a5d9fb5b89f023a80b791e04fba57960aa790cc6125f7f1d39dfe"}, @@ -3016,7 +3016,7 @@ name = "ruff" version = "0.13.2" requires_python = ">=3.7" summary = "An extremely fast Python linter and code formatter, written in Rust." -groups = ["default"] +groups = ["dev"] files = [ {file = "ruff-0.13.2-py3-none-linux_armv6l.whl", hash = "sha256:3796345842b55f033a78285e4f1641078f902020d8450cade03aad01bffd81c3"}, {file = "ruff-0.13.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ff7e4dda12e683e9709ac89e2dd436abf31a4d8a8fc3d89656231ed808e231d2"}, @@ -3110,39 +3110,75 @@ files = [ [[package]] name = "scipy" -version = "1.13.1" -requires_python = ">=3.9" +version = "1.17.1" +requires_python = ">=3.11" summary = "Fundamental algorithms for scientific computing in Python" groups = ["default"] dependencies = [ - "numpy<2.3,>=1.22.4", -] -files = [ - {file = "scipy-1.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:20335853b85e9a49ff7572ab453794298bcf0354d8068c5f6775a0eabf350aca"}, - {file = "scipy-1.13.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d605e9c23906d1994f55ace80e0125c587f96c020037ea6aa98d01b4bd2e222f"}, - {file = "scipy-1.13.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfa31f1def5c819b19ecc3a8b52d28ffdcc7ed52bb20c9a7589669dd3c250989"}, - {file = "scipy-1.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26264b282b9da0952a024ae34710c2aff7d27480ee91a2e82b7b7073c24722f"}, - {file = "scipy-1.13.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:eccfa1906eacc02de42d70ef4aecea45415f5be17e72b61bafcfd329bdc52e94"}, - {file = "scipy-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:2831f0dc9c5ea9edd6e51e6e769b655f08ec6db6e2e10f86ef39bd32eb11da54"}, - {file = "scipy-1.13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:27e52b09c0d3a1d5b63e1105f24177e544a222b43611aaf5bc44d4a0979e32f9"}, - {file = "scipy-1.13.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:54f430b00f0133e2224c3ba42b805bfd0086fe488835effa33fa291561932326"}, - {file = "scipy-1.13.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e89369d27f9e7b0884ae559a3a956e77c02114cc60a6058b4e5011572eea9299"}, - {file = "scipy-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a78b4b3345f1b6f68a763c6e25c0c9a23a9fd0f39f5f3d200efe8feda560a5fa"}, - {file = "scipy-1.13.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:45484bee6d65633752c490404513b9ef02475b4284c4cfab0ef946def50b3f59"}, - {file = "scipy-1.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:5713f62f781eebd8d597eb3f88b8bf9274e79eeabf63afb4a737abc6c84ad37b"}, - {file = "scipy-1.13.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5d72782f39716b2b3509cd7c33cdc08c96f2f4d2b06d51e52fb45a19ca0c86a1"}, - {file = "scipy-1.13.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:017367484ce5498445aade74b1d5ab377acdc65e27095155e448c88497755a5d"}, - {file = "scipy-1.13.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:949ae67db5fa78a86e8fa644b9a6b07252f449dcf74247108c50e1d20d2b4627"}, - {file = "scipy-1.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de3ade0e53bc1f21358aa74ff4830235d716211d7d077e340c7349bc3542e884"}, - {file = "scipy-1.13.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2ac65fb503dad64218c228e2dc2d0a0193f7904747db43014645ae139c8fad16"}, - {file = "scipy-1.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:cdd7dacfb95fea358916410ec61bbc20440f7860333aee6d882bb8046264e949"}, - {file = "scipy-1.13.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:436bbb42a94a8aeef855d755ce5a465479c721e9d684de76bf61a62e7c2b81d5"}, - {file = "scipy-1.13.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:8335549ebbca860c52bf3d02f80784e91a004b71b059e3eea9678ba994796a24"}, - {file = "scipy-1.13.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d533654b7d221a6a97304ab63c41c96473ff04459e404b83275b60aa8f4b7004"}, - {file = "scipy-1.13.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:637e98dcf185ba7f8e663e122ebf908c4702420477ae52a04f9908707456ba4d"}, - {file = "scipy-1.13.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a014c2b3697bde71724244f63de2476925596c24285c7a637364761f8710891c"}, - {file = "scipy-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:392e4ec766654852c25ebad4f64e4e584cf19820b980bc04960bca0b0cd6eaa2"}, - {file = "scipy-1.13.1.tar.gz", hash = "sha256:095a87a0312b08dfd6a6155cbbd310a8c51800fc931b8c0b84003014b874ed3c"}, + "numpy<2.7,>=1.26.4", +] +files = [ + {file = "scipy-1.17.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec"}, + {file = "scipy-1.17.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696"}, + {file = "scipy-1.17.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee"}, + {file = "scipy-1.17.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd"}, + {file = "scipy-1.17.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c"}, + {file = "scipy-1.17.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4"}, + {file = "scipy-1.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444"}, + {file = "scipy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082"}, + {file = "scipy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff"}, + {file = "scipy-1.17.1-cp311-cp311-win_arm64.whl", hash = "sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d"}, + {file = "scipy-1.17.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8"}, + {file = "scipy-1.17.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76"}, + {file = "scipy-1.17.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086"}, + {file = "scipy-1.17.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b"}, + {file = "scipy-1.17.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21"}, + {file = "scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458"}, + {file = "scipy-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb"}, + {file = "scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea"}, + {file = "scipy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87"}, + {file = "scipy-1.17.1-cp312-cp312-win_arm64.whl", hash = "sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3"}, + {file = "scipy-1.17.1-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c"}, + {file = "scipy-1.17.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f"}, + {file = "scipy-1.17.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d"}, + {file = "scipy-1.17.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b"}, + {file = "scipy-1.17.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6"}, + {file = "scipy-1.17.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464"}, + {file = "scipy-1.17.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950"}, + {file = "scipy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369"}, + {file = "scipy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448"}, + {file = "scipy-1.17.1-cp313-cp313-win_arm64.whl", hash = "sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87"}, + {file = "scipy-1.17.1-cp313-cp313t-macosx_10_14_x86_64.whl", hash = "sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a"}, + {file = "scipy-1.17.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0"}, + {file = "scipy-1.17.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce"}, + {file = "scipy-1.17.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6"}, + {file = "scipy-1.17.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e"}, + {file = "scipy-1.17.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475"}, + {file = "scipy-1.17.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50"}, + {file = "scipy-1.17.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca"}, + {file = "scipy-1.17.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c"}, + {file = "scipy-1.17.1-cp313-cp313t-win_arm64.whl", hash = "sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49"}, + {file = "scipy-1.17.1-cp314-cp314-macosx_10_14_x86_64.whl", hash = "sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717"}, + {file = "scipy-1.17.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9"}, + {file = "scipy-1.17.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b"}, + {file = "scipy-1.17.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866"}, + {file = "scipy-1.17.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350"}, + {file = "scipy-1.17.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118"}, + {file = "scipy-1.17.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068"}, + {file = "scipy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118"}, + {file = "scipy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19"}, + {file = "scipy-1.17.1-cp314-cp314-win_arm64.whl", hash = "sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293"}, + {file = "scipy-1.17.1-cp314-cp314t-macosx_10_14_x86_64.whl", hash = "sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6"}, + {file = "scipy-1.17.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1"}, + {file = "scipy-1.17.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39"}, + {file = "scipy-1.17.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca"}, + {file = "scipy-1.17.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad"}, + {file = "scipy-1.17.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a"}, + {file = "scipy-1.17.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4"}, + {file = "scipy-1.17.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2"}, + {file = "scipy-1.17.1-cp314-cp314t-win_amd64.whl", hash = "sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484"}, + {file = "scipy-1.17.1-cp314-cp314t-win_arm64.whl", hash = "sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21"}, + {file = "scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0"}, ] [[package]] @@ -3208,7 +3244,7 @@ name = "setuptools" version = "82.0.1" requires_python = ">=3.9" summary = "Most extensible Python build backend with support for C/C++ extension modules" -groups = ["default", "quantize"] +groups = ["default", "dev", "quantize"] files = [ {file = "setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb"}, {file = "setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9"}, @@ -3442,7 +3478,7 @@ name = "types-protobuf" version = "6.32.1.20250918" requires_python = ">=3.9" summary = "Typing stubs for protobuf" -groups = ["default"] +groups = ["dev"] files = [ {file = "types_protobuf-6.32.1.20250918-py3-none-any.whl", hash = "sha256:22ba6133d142d11cc34d3788ad6dead2732368ebb0406eaa7790ea6ae46c8d0b"}, {file = "types_protobuf-6.32.1.20250918.tar.gz", hash = "sha256:44ce0ae98475909ca72379946ab61a4435eec2a41090821e713c17e8faf5b88f"}, @@ -3453,7 +3489,7 @@ name = "types-psycopg2" version = "2.9.21.20250915" requires_python = ">=3.9" summary = "Typing stubs for psycopg2" -groups = ["default"] +groups = ["dev"] files = [ {file = "types_psycopg2-2.9.21.20250915-py3-none-any.whl", hash = "sha256:eefe5ccdc693fc086146e84c9ba437bb278efe1ef330b299a0cb71169dc6c55f"}, {file = "types_psycopg2-2.9.21.20250915.tar.gz", hash = "sha256:bfeb8f54c32490e7b5edc46215ab4163693192bc90407b4a023822de9239f5c8"}, @@ -3489,7 +3525,7 @@ name = "tzdata" version = "2026.1" requires_python = ">=2" summary = "Provider of IANA time zone data" -groups = ["default", "dev"] +groups = ["dev"] files = [ {file = "tzdata-2026.1-py2.py3-none-any.whl", hash = "sha256:4b1d2be7ac37ceafd7327b961aa3a54e467efbdb563a23655fbfe0d39cfc42a9"}, {file = "tzdata-2026.1.tar.gz", hash = "sha256:67658a1903c75917309e753fdc349ac0efd8c27db7a0cb406a25be4840f87f98"}, diff --git a/pyproject.toml b/pyproject.toml index 3df29c5..157c381 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,147 @@ +[tool.black] +line-length = 79 +target-version = ["py313"] +skip-string-normalization = false +exclude = ''' +/( + \.git + | \.hg + | \.venv + | venv + | build + | dist + | src/aivalidatorservice/grpc + | trainer/google_colab_train.py + | trainer/ai-validator-model-pipeline-google-colab.ipynb +)/ +''' + +[tool.isort] +profile = "black" +line_length = 79 +known_first_party = ["app"] +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +skip = [".venv", "src/aivalidatorservice/grpc", "trainer/google_colab_train.py", "trainer/ai-validator-model-pipeline-google-colab.ipynb"] + +[tool.flake8] +max-line-length = 120 +extend-ignore = ["E203", "W503"] +per-file-ignores = [ + "__init__.py:F401", + "tests/*:T20,ARG001" +] +exclude = [ + ".venv", + "__pycache__", + "build", + "dist", + "src/aivalidatorservice/grpc", + "trainer/google_colab_train.py", + "trainer/ai-validator-model-pipeline-google-colab.ipynb" +] + +[tool.ruff] +line-length = 120 +target-version = "py313" +preview = true +exclude = [".venv", "build", "dist", "src/aivalidatorservice/grpc", "trainer/google_colab_train.py", "trainer/ai-validator-model-pipeline-google-colab.ipynb"] + +[tool.ruff.lint] +select = [ + "E", + "F", + "B", + "UP", + "SIM", + "T20", + "ARG", + "C4", + "RUF" +] +ignore = [ + "E203", + "B008", + "SIM105", + "RUF001", + "RUF100", + "RUF003", + "RUF002" +] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["T20", "ARG001"] + +[tool.mypy] +python_version = "3.13" +strict = true +exclude = [ + 'src(?:[/\\])aivalidatorservice(?:[/\\])grpc', + 'trainer(?:[/\\])google_colab_train\.py', + 'trainer(?:[/\\])ai-validator-model-pipeline-google-colab\.ipynb', +] +warn_unused_configs = true +warn_return_any = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +show_error_codes = true +explicit_package_bases = true +mypy_path = ["src"] +plugins = ["pydantic.mypy"] + +[tool.pytest.ini_options] +addopts = "--import-mode=importlib" +asyncio_mode = "strict" +testpaths = ["tests"] +python_files = ["test_*.py"] + +[tool.coverage.run] +source = ["src/aivalidatorservice"] +relative_files = true + +[tool.coverage.report] +include = [ + "src/aivalidatorservice/handler/*", + "src/aivalidatorservice/service/*", +] +omit = ["src/aivalidatorservice/server.py"] + + [project] name = "ai-validator-service" version = "0.1.0" description = "Default template for PDM package" -dependencies = ["scipy>=1.14.0", "grpcio==1.75.1", "transformers>=4.57.6", "tokenizers>=0.21.4", "grpcio-reflection==1.75.1", "pydantic==2.11.9", "pydantic-settings==2.11.0", "psycopg2-binary==2.9.10", "watchfiles==1.1.0", "asyncpg>=0.30.0", "PyPika>=0.51.1", "grpcio-tools==1.75.1", "grpc-stubs==1.53.0.6", "protobuf==6.32.1", "pytest>=8.4.2", "pytest-asyncio==0.24.0", "pytest-cov==5.0.0", "pytest-mock==3.15.1", "pytest-faker==2.0.0", "Faker==37.11.0", "black==25.9.0", "isort==6.0.1", "flake8==7.3.0", "flake8-pyproject==1.2.3", "ruff==0.13.2", "mypy==1.18.2", "types-protobuf==6.32.1.20250918", "types-psycopg2==2.9.21.20250915", "scikit-learn>=1.6.1", "torch>=2.7.1", "optimum[onnxruntime]>=2.1.0", "tiktoken>=0.12.0", "sentencepiece>=0.2.1", "huggingface-hub>=0.36.2", "onnxruntime>=1.25.0"] +dependencies = [ + "scipy>=1.14.0", + "grpcio==1.75.1", + "transformers>=4.57.6", + "tokenizers>=0.21.4", + "grpcio-reflection==1.75.1", + "pydantic==2.11.9", + "pydantic-settings==2.11.0", + "psycopg2-binary==2.9.10", + "watchfiles==1.1.0", + "asyncpg>=0.30.0", + "PyPika>=0.51.1", + "scikit-learn>=1.6.1", + "torch>=2.7.1", + "optimum[onnxruntime]>=2.1.0", + "tiktoken>=0.12.0", + "sentencepiece>=0.2.1", + "huggingface-hub>=0.36.2", + "onnxruntime>=1.25.0", +] requires-python = ">=3.11" readme = "README.md" license = {text = "GNU GENERAL PUBLIC LICENSE"} - [tool.pdm] distribution = true @@ -16,6 +150,23 @@ run-server = "aivalidatorservice.server:main" [dependency-groups] dev = [ + "grpcio-tools==1.75.1", + "grpc-stubs==1.53.0.6", + "protobuf==6.32.1", + "pytest>=8.4.2", + "pytest-asyncio==0.24.0", + "pytest-cov==5.0.0", + "pytest-mock==3.15.1", + "pytest-faker==2.0.0", + "Faker==37.11.0", + "black==25.9.0", + "isort==6.0.1", + "flake8==7.3.0", + "flake8-pyproject>=1.2.4", + "ruff==0.13.2", + "mypy==1.18.2", + "types-protobuf==6.32.1.20250918", + "types-psycopg2==2.9.21.20250915", "datasets==3.6.0", "pandas==2.3.0", "pyarrow==20.0.0", @@ -23,4 +174,4 @@ dev = [ quantize = [ "torch>=2.7.1", "optimum[onnxruntime-gpu]>=2.1.0", -] +] \ No newline at end of file diff --git a/src/aivalidatorservice/constants.py b/src/aivalidatorservice/constants.py index 09543ec..43cc297 100644 --- a/src/aivalidatorservice/constants.py +++ b/src/aivalidatorservice/constants.py @@ -1,6 +1,11 @@ import pandas as pd + +from custom_logger import get_logger + +log = get_logger(__name__) + for f in ["russian_dataset_2.tsv", "russian_distorted_toxicity.tsv"]: df = pd.read_csv(f"data/datasets/{f}", sep="\t", nrows=3) - print(f"\n{f}:") - print(df.columns.tolist()) - print(df.head(2)) + log.info(f"Previewing dataset file: {f}") + log.debug(f"Columns: {df.columns.tolist()}") + log.debug(f"Head preview:\n{df.head(2)}") diff --git a/src/aivalidatorservice/grpc/moderation_pb2.py b/src/aivalidatorservice/grpc/moderation_pb2.py index 8860587..0764e8c 100644 --- a/src/aivalidatorservice/grpc/moderation_pb2.py +++ b/src/aivalidatorservice/grpc/moderation_pb2.py @@ -9,34 +9,30 @@ from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder + _runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, - 6, - 31, - 1, - '', - 'moderation.proto' + _runtime_version.Domain.PUBLIC, 6, 31, 1, "", "moderation.proto" ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x10moderation.proto\x12\nmoderation\"W\n\x15ModerateObjectRequest\x12\n\n\x02id\x18\x01 \x01(\x03\x12$\n\x04type\x18\x02 \x01(\x0e\x32\x16.moderation.ObjectType\x12\x0c\n\x04text\x18\x03 \x01(\t\")\n\x16ModerateObjectResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08*\x83\x01\n\nObjectType\x12\x1b\n\x17OBJECT_TYPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bOBJECT_TYPE_MOD_DESCRIPTION\x10\x01\x12\x1c\n\x18OBJECT_TYPE_COMMENT_TEXT\x10\x02\x12\x19\n\x15OBJECT_TYPE_USER_NAME\x10\x03\x32l\n\x11ModerationService\x12W\n\x0eModerateObject\x12!.moderation.ModerateObjectRequest\x1a\".moderation.ModerateObjectResponseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x10moderation.proto\x12\nmoderation"W\n\x15ModerateObjectRequest\x12\n\n\x02id\x18\x01 \x01(\x03\x12$\n\x04type\x18\x02 \x01(\x0e\x32\x16.moderation.ObjectType\x12\x0c\n\x04text\x18\x03 \x01(\t")\n\x16ModerateObjectResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08*\x83\x01\n\nObjectType\x12\x1b\n\x17OBJECT_TYPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bOBJECT_TYPE_MOD_DESCRIPTION\x10\x01\x12\x1c\n\x18OBJECT_TYPE_COMMENT_TEXT\x10\x02\x12\x19\n\x15OBJECT_TYPE_USER_NAME\x10\x03\x32l\n\x11ModerationService\x12W\n\x0eModerateObject\x12!.moderation.ModerateObjectRequest\x1a".moderation.ModerateObjectResponseb\x06proto3' +) _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'moderation_pb2', _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "moderation_pb2", _globals) if not _descriptor._USE_C_DESCRIPTORS: - DESCRIPTOR._loaded_options = None - _globals['_OBJECTTYPE']._serialized_start=165 - _globals['_OBJECTTYPE']._serialized_end=296 - _globals['_MODERATEOBJECTREQUEST']._serialized_start=32 - _globals['_MODERATEOBJECTREQUEST']._serialized_end=119 - _globals['_MODERATEOBJECTRESPONSE']._serialized_start=121 - _globals['_MODERATEOBJECTRESPONSE']._serialized_end=162 - _globals['_MODERATIONSERVICE']._serialized_start=298 - _globals['_MODERATIONSERVICE']._serialized_end=406 + DESCRIPTOR._loaded_options = None + _globals["_OBJECTTYPE"]._serialized_start = 165 + _globals["_OBJECTTYPE"]._serialized_end = 296 + _globals["_MODERATEOBJECTREQUEST"]._serialized_start = 32 + _globals["_MODERATEOBJECTREQUEST"]._serialized_end = 119 + _globals["_MODERATEOBJECTRESPONSE"]._serialized_start = 121 + _globals["_MODERATEOBJECTRESPONSE"]._serialized_end = 162 + _globals["_MODERATIONSERVICE"]._serialized_start = 298 + _globals["_MODERATIONSERVICE"]._serialized_end = 406 # @@protoc_insertion_point(module_scope) diff --git a/src/aivalidatorservice/grpc/moderation_pb2.pyi b/src/aivalidatorservice/grpc/moderation_pb2.pyi index 107cea2..a4682f9 100644 --- a/src/aivalidatorservice/grpc/moderation_pb2.pyi +++ b/src/aivalidatorservice/grpc/moderation_pb2.pyi @@ -1,7 +1,10 @@ -from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from typing import ClassVar as _ClassVar +from typing import Optional as _Optional +from typing import Union as _Union + from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message -from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper DESCRIPTOR: _descriptor.FileDescriptor @@ -11,6 +14,7 @@ class ObjectType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): OBJECT_TYPE_MOD_DESCRIPTION: _ClassVar[ObjectType] OBJECT_TYPE_COMMENT_TEXT: _ClassVar[ObjectType] OBJECT_TYPE_USER_NAME: _ClassVar[ObjectType] + OBJECT_TYPE_UNSPECIFIED: ObjectType OBJECT_TYPE_MOD_DESCRIPTION: ObjectType OBJECT_TYPE_COMMENT_TEXT: ObjectType @@ -24,7 +28,12 @@ class ModerateObjectRequest(_message.Message): id: int type: ObjectType text: str - def __init__(self, id: _Optional[int] = ..., type: _Optional[_Union[ObjectType, str]] = ..., text: _Optional[str] = ...) -> None: ... + def __init__( + self, + id: _Optional[int] = ..., + type: _Optional[_Union[ObjectType, str]] = ..., + text: _Optional[str] = ..., + ) -> None: ... class ModerateObjectResponse(_message.Message): __slots__ = ("success",) diff --git a/src/aivalidatorservice/grpc/moderation_pb2_grpc.py b/src/aivalidatorservice/grpc/moderation_pb2_grpc.py index ed857d0..412950c 100644 --- a/src/aivalidatorservice/grpc/moderation_pb2_grpc.py +++ b/src/aivalidatorservice/grpc/moderation_pb2_grpc.py @@ -1,27 +1,30 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" + import grpc -import warnings from . import moderation_pb2 as moderation__pb2 -GRPC_GENERATED_VERSION = '1.75.1' +GRPC_GENERATED_VERSION = "1.75.1" GRPC_VERSION = grpc.__version__ _version_not_supported = False try: from grpc._utilities import first_version_is_lower - _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) + + _version_not_supported = first_version_is_lower( + GRPC_VERSION, GRPC_GENERATED_VERSION + ) except ImportError: _version_not_supported = True if _version_not_supported: raise RuntimeError( - f'The grpc package installed is at version {GRPC_VERSION},' - + f' but the generated code in moderation_pb2_grpc.py depends on' - + f' grpcio>={GRPC_GENERATED_VERSION}.' - + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' - + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + f"The grpc package installed is at version {GRPC_VERSION}," + + " but the generated code in moderation_pb2_grpc.py depends on" + + f" grpcio>={GRPC_GENERATED_VERSION}." + + f" Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}" + + f" or downgrade your generated code using grpcio-tools<={GRPC_VERSION}." ) @@ -35,10 +38,11 @@ def __init__(self, channel): channel: A grpc.Channel. """ self.ModerateObject = channel.unary_unary( - '/moderation.ModerationService/ModerateObject', - request_serializer=moderation__pb2.ModerateObjectRequest.SerializeToString, - response_deserializer=moderation__pb2.ModerateObjectResponse.FromString, - _registered_method=True) + "/moderation.ModerationService/ModerateObject", + request_serializer=moderation__pb2.ModerateObjectRequest.SerializeToString, + response_deserializer=moderation__pb2.ModerateObjectResponse.FromString, + _registered_method=True, + ) class ModerationServiceServicer(object): @@ -47,43 +51,48 @@ class ModerationServiceServicer(object): def ModerateObject(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") def add_ModerationServiceServicer_to_server(servicer, server): rpc_method_handlers = { - 'ModerateObject': grpc.unary_unary_rpc_method_handler( - servicer.ModerateObject, - request_deserializer=moderation__pb2.ModerateObjectRequest.FromString, - response_serializer=moderation__pb2.ModerateObjectResponse.SerializeToString, - ), + "ModerateObject": grpc.unary_unary_rpc_method_handler( + servicer.ModerateObject, + request_deserializer=moderation__pb2.ModerateObjectRequest.FromString, + response_serializer=moderation__pb2.ModerateObjectResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( - 'moderation.ModerationService', rpc_method_handlers) + "moderation.ModerationService", rpc_method_handlers + ) server.add_generic_rpc_handlers((generic_handler,)) - server.add_registered_method_handlers('moderation.ModerationService', rpc_method_handlers) + server.add_registered_method_handlers( + "moderation.ModerationService", rpc_method_handlers + ) - # This class is part of an EXPERIMENTAL API. +# This class is part of an EXPERIMENTAL API. class ModerationService(object): """Missing associated documentation comment in .proto file.""" @staticmethod - def ModerateObject(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): + def ModerateObject( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): return grpc.experimental.unary_unary( request, target, - '/moderation.ModerationService/ModerateObject', + "/moderation.ModerationService/ModerateObject", moderation__pb2.ModerateObjectRequest.SerializeToString, moderation__pb2.ModerateObjectResponse.FromString, options, @@ -94,4 +103,5 @@ def ModerateObject(request, wait_for_ready, timeout, metadata, - _registered_method=True) + _registered_method=True, + ) diff --git a/src/aivalidatorservice/handler/handler.py b/src/aivalidatorservice/handler/handler.py index 69fccd6..4a368e2 100644 --- a/src/aivalidatorservice/handler/handler.py +++ b/src/aivalidatorservice/handler/handler.py @@ -1,7 +1,12 @@ import grpc + from aivalidatorservice.grpc import moderation_pb2, moderation_pb2_grpc -from aivalidatorservice.handler.moderate import Moderate as _moderate +from aivalidatorservice.handler.moderate import moderate as _moderate from aivalidatorservice.service.service import ModerationService +from logger.custom_logger import get_logger + +log = get_logger(__name__) + class ModerationHandler(moderation_pb2_grpc.ModerationServiceServicer): def __init__(self, service: ModerationService): @@ -12,4 +17,7 @@ async def ModerateObject( request: moderation_pb2.ModerateObjectRequest, context: grpc.ServicerContext, ) -> moderation_pb2.ModerateObjectResponse: - return await _moderate(self._service, request, context) \ No newline at end of file + log.debug( + f"Received ModerateObject request id={request.id} type={request.type}" + ) + return await _moderate(self._service, request, context) diff --git a/src/aivalidatorservice/handler/moderate.py b/src/aivalidatorservice/handler/moderate.py index e1ed312..1a3b30c 100644 --- a/src/aivalidatorservice/handler/moderate.py +++ b/src/aivalidatorservice/handler/moderate.py @@ -1,11 +1,25 @@ import grpc + from aivalidatorservice.grpc import moderation_pb2 from aivalidatorservice.service.service import ModerationService +from logger.custom_logger import get_logger + +log = get_logger(__name__) -async def Moderate( + +async def moderate( service: ModerationService, request: moderation_pb2.ModerateObjectRequest, - context: grpc.ServicerContext, + _context: grpc.ServicerContext, ) -> moderation_pb2.ModerateObjectResponse: - is_toxic = await service.moderate(request.text) - return moderation_pb2.ModerateObjectResponse(success=is_toxic) \ No newline at end of file + try: + is_toxic = await service.moderate(request.text) + except Exception as exc: + log.error(f"Moderation request failed: {exc}") + raise + + response = moderation_pb2.ModerateObjectResponse(success=is_toxic) + log.debug( + f"Responding ModerateObject id={request.id} success={response.success}" + ) + return response diff --git a/src/aivalidatorservice/model/loader.py b/src/aivalidatorservice/model/loader.py index e41243a..ce73c9c 100644 --- a/src/aivalidatorservice/model/loader.py +++ b/src/aivalidatorservice/model/loader.py @@ -1,11 +1,19 @@ +import torch from optimum.onnxruntime import ORTModelForSequenceClassification from transformers import AutoTokenizer -import torch + +from logger.custom_logger import get_logger TOXICITY_THRESHOLD = 0.75 +log = get_logger(__name__) + + class ModerationModel: def __init__(self, model_path: str, tokenizer_path: str): + log.info( + f"Initializing ModerationModel model_path={model_path} tokenizer_path={tokenizer_path}" + ) self._tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) self._model = ORTModelForSequenceClassification.from_pretrained( model_path, @@ -22,10 +30,12 @@ def _run(self, text: str) -> float: ) outputs = self._model(**inputs) probabilities = torch.softmax(outputs.logits, dim=-1) - toxic_score = probabilities[0][1].item() + toxic_score = probabilities[0][1].item() return toxic_score def predict(self, text: str) -> bool: score = self._run(text) - print(f"[debug] toxic_score={score:.4f} threshold={TOXICITY_THRESHOLD}") - return score > TOXICITY_THRESHOLD \ No newline at end of file + log.debug( + f"Prediction toxic_score={score:.4f} threshold={TOXICITY_THRESHOLD}" + ) + return score > TOXICITY_THRESHOLD diff --git a/src/aivalidatorservice/server.py b/src/aivalidatorservice/server.py index 13b2b04..b76471d 100644 --- a/src/aivalidatorservice/server.py +++ b/src/aivalidatorservice/server.py @@ -1,5 +1,4 @@ import asyncio -import logging from concurrent import futures import grpc @@ -7,14 +6,17 @@ from aivalidatorservice.grpc import moderation_pb2, moderation_pb2_grpc from aivalidatorservice.handler.handler import ModerationHandler +from aivalidatorservice.model.loader import ModerationModel from aivalidatorservice.service.service import ModerationService from aivalidatorservice.settings import Settings -from aivalidatorservice.model.loader import ModerationModel +from custom_logger import get_logger + async def serve() -> None: settings = Settings() settings.configure_logging() - logger = logging.getLogger(__name__) + log = get_logger(__name__) + log.info("Initializing moderation model and gRPC server") model = ModerationModel( model_path="./models/deberta-qat-int8-v2", @@ -25,21 +27,25 @@ async def serve() -> None: handler = ModerationHandler(service) server = grpc.aio.server(futures.ThreadPoolExecutor(max_workers=5)) - moderation_pb2_grpc.add_ModerationServiceServicer_to_server(handler, server) # type: ignore[no-untyped-call] + moderation_pb2_grpc.add_ModerationServiceServicer_to_server(handler, server) # type: ignore[no-untyped-call] SERVICE_NAMES = ( - moderation_pb2.DESCRIPTOR.services_by_name["ModerationService"].full_name, + moderation_pb2.DESCRIPTOR.services_by_name[ + "ModerationService" + ].full_name, reflection.SERVICE_NAME, ) reflection.enable_server_reflection(SERVICE_NAMES, server) server.add_insecure_port(f"{settings.host}:{settings.port}") await server.start() - logger.info(f"gRPC server listening on {settings.host}:{settings.port}") + log.info(f"gRPC server listening on {settings.host}:{settings.port}") await server.wait_for_termination() + def main() -> None: asyncio.run(serve()) + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/src/aivalidatorservice/service/moderate.py b/src/aivalidatorservice/service/moderate.py index 9b038a9..bdbc3d4 100644 --- a/src/aivalidatorservice/service/moderate.py +++ b/src/aivalidatorservice/service/moderate.py @@ -1,4 +1,9 @@ from aivalidatorservice.model.loader import ModerationModel +from custom_logger import get_logger -async def moderate(model: ModerationModel, normalized_text: str) -> bool: - return model.predict(normalized_text) \ No newline at end of file +log = get_logger(__name__) + + +def moderate(model: ModerationModel, normalized_text: str) -> bool: + log.debug("Running model prediction in service.moderate") + return model.predict(normalized_text) diff --git a/src/aivalidatorservice/service/service.py b/src/aivalidatorservice/service/service.py index eb1413f..3575a03 100644 --- a/src/aivalidatorservice/service/service.py +++ b/src/aivalidatorservice/service/service.py @@ -1,9 +1,20 @@ -from aivalidatorservice.service.moderate import moderate as _moderate +import asyncio + from aivalidatorservice.model.loader import ModerationModel +from aivalidatorservice.service.moderate import moderate as _moderate +from custom_logger import get_logger + +log = get_logger(__name__) + class ModerationService: def __init__(self, model: ModerationModel): self._model = model async def moderate(self, normalized_text: str) -> bool: - return await _moderate(self._model, normalized_text) \ No newline at end of file + log.debug(f"Service moderating text length={len(normalized_text)}") + result = await asyncio.to_thread( + _moderate, self._model, normalized_text + ) + log.debug(f"Service moderation result={result}") + return result diff --git a/tests/behavioral/test_golden_path.py b/tests/behavioral/test_golden_path.py index 74a065b..a119476 100644 --- a/tests/behavioral/test_golden_path.py +++ b/tests/behavioral/test_golden_path.py @@ -1,5 +1,6 @@ import pytest + @pytest.mark.asyncio async def test_high_logit_returns_true(toxic_service): result = await toxic_service.moderate("arbitrary text") @@ -24,4 +25,3 @@ async def test_low_logit_returns_false(clean_service): async def test_crash_safety_inputs_return_boolean(clean_service, text): result = await clean_service.moderate(text) assert isinstance(result, bool) - diff --git a/tests/conftest.py b/tests/conftest.py index 09728a9..a4d8baf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,13 @@ +from unittest.mock import MagicMock + import pytest import torch -from unittest.mock import MagicMock + from aivalidatorservice.model import loader from aivalidatorservice.model.loader import ModerationModel from aivalidatorservice.service.service import ModerationService + def make_fake_session(logit: float) -> MagicMock: session = MagicMock() outputs = MagicMock() @@ -12,6 +15,7 @@ def make_fake_session(logit: float) -> MagicMock: session.return_value = outputs return session + def make_fake_tokenizer(max_length: int = 128) -> MagicMock: tokenizer = MagicMock() tokenizer.return_value = { @@ -20,28 +24,43 @@ def make_fake_tokenizer(max_length: int = 128) -> MagicMock: } return tokenizer + @pytest.fixture(scope="session") def real_tokenizer(): return loader.AutoTokenizer.from_pretrained("microsoft/deberta-v3-small") + @pytest.fixture def toxic_service(monkeypatch): - monkeypatch.setattr(loader.AutoTokenizer, "from_pretrained", lambda _: make_fake_tokenizer()) + monkeypatch.setattr( + loader.AutoTokenizer, + "from_pretrained", + lambda _: make_fake_tokenizer(), + ) monkeypatch.setattr( loader.ORTModelForSequenceClassification, "from_pretrained", - lambda *args, **kwargs: make_fake_session(logit=5.0), + lambda *_args, **_kwargs: make_fake_session(logit=5.0), + ) + model = ModerationModel( + model_path="mock-model", tokenizer_path="mock-tokenizer" ) - model = ModerationModel(model_path="mock-model", tokenizer_path="mock-tokenizer") return ModerationService(model) + @pytest.fixture def clean_service(monkeypatch): - monkeypatch.setattr(loader.AutoTokenizer, "from_pretrained", lambda _: make_fake_tokenizer()) + monkeypatch.setattr( + loader.AutoTokenizer, + "from_pretrained", + lambda _: make_fake_tokenizer(), + ) monkeypatch.setattr( loader.ORTModelForSequenceClassification, "from_pretrained", - lambda *args, **kwargs: make_fake_session(logit=-5.0), + lambda *_args, **_kwargs: make_fake_session(logit=-5.0), ) - model = ModerationModel(model_path="mock-model", tokenizer_path="mock-tokenizer") - return ModerationService(model) \ No newline at end of file + model = ModerationModel( + model_path="mock-model", tokenizer_path="mock-tokenizer" + ) + return ModerationService(model) diff --git a/tests/contract/test_grpc_contract.py b/tests/contract/test_grpc_contract.py index 8a903de..6fb14e0 100644 --- a/tests/contract/test_grpc_contract.py +++ b/tests/contract/test_grpc_contract.py @@ -3,8 +3,11 @@ from aivalidatorservice.grpc import moderation_pb2 from aivalidatorservice.handler.handler import ModerationHandler + @pytest.mark.asyncio -async def test_grpc_contract_toxic_input_returns_expected_response(toxic_service): +async def test_grpc_contract_toxic_input_returns_expected_response( + toxic_service, +): request = moderation_pb2.ModerateObjectRequest( id=123, type=moderation_pb2.OBJECT_TYPE_COMMENT_TEXT, diff --git a/tests/tokenizer/test_tokenizer_shapes.py b/tests/tokenizer/test_tokenizer_shapes.py index d48ec77..a6dab04 100644 --- a/tests/tokenizer/test_tokenizer_shapes.py +++ b/tests/tokenizer/test_tokenizer_shapes.py @@ -1,4 +1,6 @@ def test_tokenizer_truncates_to_max_length_128(real_tokenizer): - out = real_tokenizer("token " * 400, return_tensors="pt", max_length=128, truncation=True) + out = real_tokenizer( + "token " * 400, return_tensors="pt", max_length=128, truncation=True + ) assert out["input_ids"].shape == (1, 128) assert out["attention_mask"].shape == (1, 128) diff --git a/tools/common.just b/tools/common.just index ebb7076..26776fe 100644 --- a/tools/common.just +++ b/tools/common.just @@ -27,16 +27,16 @@ gen-stubs-moderation: fetch-proto update-moderation: gen-stubs-moderation clean format: - black . - isort . - ruff check . --fix + pdm run black . + pdm run isort . + pdm run ruff check . --fix lint: - black --check . - isort . --check --diff - flake8 . - ruff check . - mypy --strict . + pdm run black --check . + pdm run isort . --check --diff + pdm run flake8 . + pdm run ruff check . + pdm run mypy --strict . test: pdm run pytest tests/ diff --git a/trainer/ai-validator-model-pipeline-google-colab.ipynb b/trainer/ai-validator-model-pipeline-google-colab.ipynb index a89eb7b..3b9a4cb 100644 --- a/trainer/ai-validator-model-pipeline-google-colab.ipynb +++ b/trainer/ai-validator-model-pipeline-google-colab.ipynb @@ -1,421 +1,433 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "4fdsSGd7-5KU" - }, - "outputs": [], - "source": [ - "from google.colab import drive\n", - "import os\n", - "\n", - "drive.mount('/content/drive')\n", - "\n", - "# Fail loudly if mount didn't work\n", - "assert os.path.exists('/content/drive/MyDrive'), \"Drive mount failed — check permissions\"\n", - "print(\"Drive mounted OK\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Awg2Uu511shQ" - }, - "outputs": [], - "source": [ - "from huggingface_hub import login\n", - "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n", - "\n", - "login()\n", - "\n", - "drive_path = \"/content/drive/MyDrive/deberta-toxicity-best\"\n", - "\n", - "# Load from Drive\n", - "model = AutoModelForSequenceClassification.from_pretrained(drive_path)\n", - "tokenizer = AutoTokenizer.from_pretrained(drive_path)\n", - "\n", - "# Push to Hub\n", - "model.push_to_hub(\"esclient/deberta-toxicity-model\", private=False)\n", - "tokenizer.push_to_hub(\"esclient/deberta-toxicity-model\", private=False)\n", - "\n", - "print(\"Done!\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "-tN8WyQbixTV" - }, - "outputs": [], - "source": [ - "import os\n", - "os.makedirs(\"data/processed\", exist_ok=True)\n", - "os.makedirs(\"models/deberta-toxicity\", exist_ok=True)\n", - "\n", - "from google.colab import files\n", - "uploaded = files.upload()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "L0Pnxpe3jEzi" - }, - "outputs": [], - "source": [ - "import shutil\n", - "for fname in [\"train.parquet\", \"val.parquet\", \"test.parquet\"]:\n", - " shutil.move(fname, f\"data/processed/{fname}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "nc5ddBJjjFjp" - }, - "outputs": [], - "source": [ - "!pip install transformers accelerate scikit-learn pandas pyarrow -q" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "uQPKs1DMjLtb" - }, - "outputs": [], - "source": [ - "# ── This code is optimized for Google Colab ──────────────────────────────\n", - "import os, numpy as np, pandas as pd, torch\n", - "from pathlib import Path\n", - "from torch.utils.data import Dataset\n", - "from transformers import (\n", - " AutoTokenizer, AutoModelForSequenceClassification,\n", - " TrainingArguments, Trainer, EarlyStoppingCallback,\n", - " TrainerCallback,\n", - ")\n", - "from sklearn.metrics import f1_score, precision_score, recall_score, roc_auc_score\n", - "from tqdm import tqdm\n", - "import shutil\n", - "\n", - "print(\"CUDA:\", torch.cuda.is_available())\n", - "print(\"Device:\", torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"CPU\")\n", - "\n", - "DATA_DIR = Path(\"data/processed\")\n", - "OUT_DIR = Path(\"models/deberta-toxicity\")\n", - "OUT_DIR.mkdir(parents=True, exist_ok=True)\n", - "MODEL_ID = \"microsoft/deberta-v3-small\"\n", - "MAX_LEN = 128\n", - "\n", - "class SaveBestToDriveCallback(TrainerCallback):\n", - " def __init__(self, drive_path, tokenizer, f1_weight=0.5, auc_weight=0.5):\n", - " self.drive_path = drive_path\n", - " self.tokenizer = tokenizer\n", - " self.f1_weight = f1_weight\n", - " self.auc_weight = auc_weight\n", - " self.best_score = 0.0\n", - "\n", - " def on_evaluate(self, args, state, control, metrics=None, **kwargs):\n", - " current_f1 = metrics.get(\"eval_f1\", 0.0)\n", - " current_auc = metrics.get(\"eval_roc_auc\", 0.0)\n", - "\n", - " combined = (self.f1_weight * current_f1) + (self.auc_weight * current_auc)\n", - "\n", - " print(f\"F1: {current_f1:.4f} | AUC: {current_auc:.4f} | Combined: {combined:.4f} (best: {self.best_score:.4f})\")\n", - "\n", - " if combined > self.best_score:\n", - " self.best_score = combined\n", - " print(f\"✓ New best combined score: {combined:.4f} - saving to Drive...\")\n", - "\n", - " if os.path.exists(self.drive_path):\n", - " shutil.rmtree(self.drive_path)\n", - "\n", - " kwargs[\"model\"].save_pretrained(self.drive_path)\n", - " self.tokenizer.save_pretrained(self.drive_path)\n", - " print(f\"✓ Saved to {self.drive_path}\")\n", - "\n", - "class ToxicityDataset(Dataset):\n", - " def __init__(self, df, tokenizer, max_len, name=\"dataset\"):\n", - " print(f\"Tokenizing {name} ({len(df)} samples)...\")\n", - " enc = tokenizer(\n", - " df[\"text\"].tolist(),\n", - " truncation=True,\n", - " padding=\"max_length\",\n", - " max_length=max_len,\n", - " return_tensors=\"pt\",\n", - " verbose=False,\n", - " )\n", - " self.input_ids = enc[\"input_ids\"]\n", - " self.attention_mask = enc[\"attention_mask\"]\n", - " self.labels = torch.tensor(df[\"label\"].values, dtype=torch.long)\n", - " print(f\"Done tokenizing {name}!\")\n", - "\n", - " def __len__(self): return len(self.labels)\n", - "\n", - " def __getitem__(self, idx):\n", - " return {\n", - " \"input_ids\": self.input_ids[idx],\n", - " \"attention_mask\": self.attention_mask[idx],\n", - " \"labels\": self.labels[idx],\n", - " }\n", - "\n", - "def compute_metrics(eval_pred):\n", - " logits, labels = eval_pred\n", - " # logits shape is (n, 2) - use argmax for preds, softmax[:,1] for probs\n", - " preds = np.argmax(logits, axis=-1)\n", - " probs = torch.softmax(\n", - " torch.tensor(logits, dtype=torch.float32), dim=-1\n", - " )[:, 1].numpy()\n", - " return {\n", - " \"f1\": f1_score(labels, preds, zero_division=0),\n", - " \"precision\": precision_score(labels, preds, zero_division=0),\n", - " \"recall\": recall_score(labels, preds, zero_division=0),\n", - " \"roc_auc\": roc_auc_score(labels, probs),\n", - " }\n", - "\n", - "class WeightedTrainer(Trainer):\n", - " def __init__(self, class_weights, *args, **kwargs):\n", - " super().__init__(*args, **kwargs)\n", - " self.class_weights = class_weights # tensor([w_clean, w_toxic])\n", - "\n", - " def compute_loss(self, model, inputs, return_outputs=False, **kwargs):\n", - " labels = inputs.pop(\"labels\") # long, shape (B,)\n", - " outputs = model(**inputs)\n", - " logits = outputs.logits # shape (B, 2)\n", - "\n", - " loss = torch.nn.functional.cross_entropy(\n", - " logits,\n", - " labels,\n", - " weight=self.class_weights.to(logits.device),\n", - " )\n", - " return (loss, outputs) if return_outputs else loss\n", - "\n", - "\n", - "# ── Data ─────────────────────────────────────────────────────────────────────\n", - "train_df = pd.read_parquet(DATA_DIR / \"train.parquet\")\n", - "val_df = pd.read_parquet(DATA_DIR / \"val.parquet\")\n", - "test_df = pd.read_parquet(DATA_DIR / \"test.parquet\")\n", - "\n", - "def stratified_sample(df, n, label_col=\"label\", seed=42):\n", - " return (\n", - " df.groupby(label_col, group_keys=False)\n", - " .apply(lambda x: x.sample(min(len(x), int(n * len(x) / len(df))), random_state=seed))\n", - " .sample(frac=1, random_state=seed) # shuffle\n", - " .reset_index(drop=True)\n", - " )\n", - "\n", - "def language_balanced_sample(df, en_cap=300_000, ru_cap=300_000, other_per_lang=8_000, seed=42):\n", - " \"\"\"\n", - " Cap EN and RU, keep a small slice of each other language for context.\n", - " Preserves label balance within each language slice.\n", - " \"\"\"\n", - " parts = []\n", - "\n", - " for lang, group in df.groupby(\"lang\"):\n", - " if lang == \"en\":\n", - " cap = en_cap\n", - " elif lang == \"ru\":\n", - " cap = ru_cap\n", - " elif lang == \"mixed\":\n", - " cap = 20_000\n", - " else:\n", - " cap = other_per_lang\n", - "\n", - " cap_n = min(len(group), cap)\n", - " # stratify within the language group\n", - " sampled = (\n", - " group.groupby(\"label\", group_keys=False)\n", - " .apply(lambda x: x.sample(min(len(x), int(cap_n * len(x) / len(group))), random_state=seed))\n", - " .sample(frac=1, random_state=seed)\n", - " .reset_index(drop=True)\n", - " )\n", - " parts.append(sampled)\n", - "\n", - " return pd.concat(parts).sample(frac=1, random_state=seed).reset_index(drop=True)\n", - "\n", - "train_df = language_balanced_sample(train_df, en_cap=300_000, ru_cap=300_000, other_per_lang=8_000)\n", - "\n", - "train_df = stratified_sample(train_df, n=150_000)\n", - "val_df = stratified_sample(val_df, n=20_000)\n", - "\n", - "print(f\"Train: {len(train_df):,} rows\")\n", - "print(f\"Val: {len(val_df):,} rows\")\n", - "print(f\"Test: {len(test_df):,} rows\")\n", - "print(f\"Avg text length: {train_df['text'].str.len().mean():.0f} chars\")\n", - "\n", - "n_clean, n_toxic = (train_df[\"label\"]==0).sum(), (train_df[\"label\"]==1).sum()\n", - "class_weights = torch.tensor(\n", - " [np.sqrt(len(train_df)/(2*n_clean)), np.sqrt(len(train_df)/(2*n_toxic))],\n", - " dtype=torch.float32\n", - ")\n", - "print(f\"Class weights clean: {class_weights[0]:.3f}, toxic: {class_weights[1]:.3f}\")\n", - "\n", - "print(f\"n_clean: {n_clean:,}, n_toxic: {n_toxic:,}\")\n", - "\n", - "tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)\n", - "model = AutoModelForSequenceClassification.from_pretrained(MODEL_ID, num_labels=2)\n", - "\n", - "train_ds = ToxicityDataset(train_df, tokenizer, MAX_LEN, name=\"train\")\n", - "val_ds = ToxicityDataset(val_df, tokenizer, MAX_LEN, name=\"val\")\n", - "\n", - "# ── Training args — fp16/bf16 hardcoded, no cfg object ───────────────────────\n", - "args = TrainingArguments(\n", - " output_dir=str(OUT_DIR),\n", - " num_train_epochs=3,\n", - " per_device_train_batch_size=16,\n", - " per_device_eval_batch_size=64,\n", - " gradient_accumulation_steps=4,\n", - " learning_rate=8e-7,\n", - " warmup_ratio=0.06,\n", - " weight_decay=0.01,\n", - " max_grad_norm=1.0,\n", - " fp16=False,\n", - " bf16=True,\n", - " lr_scheduler_type=\"cosine\",\n", - " eval_strategy=\"steps\",\n", - " eval_steps=100,\n", - " save_strategy=\"steps\",\n", - " save_steps=100,\n", - " load_best_model_at_end=True,\n", - " metric_for_best_model=\"f1\",\n", - " greater_is_better=True,\n", - " dataloader_num_workers=2,\n", - " report_to=\"none\",\n", - " logging_steps=50,\n", - ")\n", - "\n", - "trainer = WeightedTrainer(\n", - " class_weights=class_weights,\n", - " model=model,\n", - " args=args,\n", - " train_dataset=train_ds,\n", - " eval_dataset=val_ds,\n", - " compute_metrics=compute_metrics,\n", - " callbacks=[\n", - " EarlyStoppingCallback(early_stopping_patience=5),\n", - " SaveBestToDriveCallback(\n", - " drive_path=\"/content/drive/MyDrive/deberta-toxicity-best\",\n", - " tokenizer=tokenizer,\n", - " ),\n", - " ],\n", - ")\n", - "\n", - "model.eval()\n", - "with torch.no_grad():\n", - " sample = {k: v[:8].to(model.device) for k, v in {\n", - " \"input_ids\": train_ds.input_ids,\n", - " \"attention_mask\": train_ds.attention_mask\n", - " }.items()}\n", - " out = model(**sample)\n", - " print(\"Logits:\", out.logits)\n", - " print(\"Preds:\", out.logits.argmax(-1))\n", - " print(\"Labels:\", train_ds.labels[:8])\n", - "\n", - "trainer.train()\n", - "\n", - "test_ds = ToxicityDataset(test_df, tokenizer, MAX_LEN, name=\"test\")\n", - "test_results = trainer.evaluate(test_ds)\n", - "print(test_results)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "B8_w05_w1Fnq" - }, - "outputs": [], - "source": [ - "import torch\n", - "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n", - "\n", - "MAX_LEN = 128 # must match what you used during training\n", - "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", - "\n", - "def predict(texts, model, tokenizer, threshold=0.5):\n", - " model.eval()\n", - " enc = tokenizer(texts, truncation=True, padding=True, max_length=MAX_LEN, return_tensors=\"pt\").to(device)\n", - " with torch.no_grad():\n", - " logits = model(**enc).logits\n", - " probs = torch.softmax(logits, dim=-1)[:, 1].cpu().numpy()\n", - " preds = (probs >= threshold).astype(int)\n", - " return probs, preds\n", - "\n", - "# Load the saved model and tokenizer from Drive\n", - "drive_path = \"/content/drive/MyDrive/deberta-toxicity-best\"\n", - "print(f\"Loading model from {drive_path}...\")\n", - "inference_tokenizer = AutoTokenizer.from_pretrained(drive_path)\n", - "inference_model = AutoModelForSequenceClassification.from_pretrained(drive_path).to(device)\n", - "print(f\"Running on {device}\")\n", - "\n", - "samples = [\n", - " # English — obvious clean\n", - " \"Thanks for the detailed explanation, really helped me understand.\",\n", - " \"Happy birthday! Hope you have a wonderful day.\",\n", - "\n", - " # English — obvious toxic\n", - " \"Kill yourself you worthless piece of garbage.\",\n", - " \"All people like you should be exterminated.\",\n", - "\n", - " # English — subtle toxic (no slurs, but threatening)\n", - " \"You better watch your back, I know where you live.\",\n", - " \"Keep talking and see what happens to you.\",\n", - "\n", - " # English — edgy but clean (criticism, dark humor)\n", - " \"This policy is absolutely brain-dead and whoever designed it should be fired.\",\n", - " \"The movie was torture to sit through, completely unwatchable.\",\n", - "\n", - " # Russian — obvious clean\n", - " \"Спасибо за помощь, очень ценю ваше время.\", # Thank you for the help\n", - " \"Отличная работа, продолжайте в том же духе!\", # Great work, keep it up\n", - "\n", - " # Russian — obvious toxic\n", - " \"Ты тупая мразь, убирайся отсюда.\", # You're a disgusting idiot, get out\n", - " \"Такие люди как ты должны сдохнуть.\", # People like you should die\n", - "\n", - " # Russian — subtle toxic\n", - " \"Ну и рожа у тебя, смотреть противно.\", # What an ugly face, disgusting to look at\n", - " \"Понаехали тут всякие, житья не дают.\", # All these migrants coming here, no peace\n", - "\n", - " # Russian — edgy but clean\n", - " \"Этот закон — полный идиотизм от начала до конца.\", # This law is complete idiocy\n", - " \"Фильм — редкостная чушь, зря потратил время.\", # The film is utter nonsense, wasted my time\n", - "\n", - " # Mixed/tricky — context-dependent\n", - " \"You're so bad at this it's actually impressive.\", # ambiguous — could be playful\n", - " \"I want to kill this level, it's so hard.\", # gaming context, clean\n", - " \"Он просто животное.\", # He's just an animal — ambiguous insult\n", - " \"This is the worst community I've ever seen.\", # harsh but not toxic\n", - "]\n", - "\n", - "probs, preds = predict(samples, inference_model, inference_tokenizer)\n", - "for text, prob, pred in zip(samples, probs, preds):\n", - " label = \"TOXIC\" if pred else \"CLEAN\"\n", - " print(f\"[{label} {prob:.2f}] {text[:60]}\")" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "T4", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4fdsSGd7-5KU" + }, + "outputs": [], + "source": [ + "from google.colab import drive\n", + "import os\n", + "\n", + "drive.mount('/content/drive')\n", + "\n", + "# Fail loudly if mount didn't work\n", + "assert os.path.exists('/content/drive/MyDrive'), \"Drive mount failed — check permissions\"\n", + "print(\"Drive mounted OK\")" + ] }, - "nbformat": 4, - "nbformat_minor": 0 + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Awg2Uu511shQ" + }, + "outputs": [], + "source": [ + "from huggingface_hub import login\n", + "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n", + "\n", + "login()\n", + "\n", + "drive_path = \"/content/drive/MyDrive/deberta-toxicity-best\"\n", + "\n", + "# Load from Drive\n", + "model = AutoModelForSequenceClassification.from_pretrained(drive_path)\n", + "tokenizer = AutoTokenizer.from_pretrained(drive_path)\n", + "\n", + "# Push to Hub\n", + "model.push_to_hub(\"esclient/deberta-toxicity-model\", private=False)\n", + "tokenizer.push_to_hub(\"esclient/deberta-toxicity-model\", private=False)\n", + "\n", + "print(\"Done!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-tN8WyQbixTV" + }, + "outputs": [], + "source": [ + "import os\n", + "from google.colab import files\n", + "\n", + "os.makedirs(\"data/processed\", exist_ok=True)\n", + "os.makedirs(\"models/deberta-toxicity\", exist_ok=True)\n", + "\n", + "uploaded = files.upload()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "L0Pnxpe3jEzi" + }, + "outputs": [], + "source": [ + "import shutil\n", + "for fname in [\"train.parquet\", \"val.parquet\", \"test.parquet\"]:\n", + " shutil.move(fname, f\"data/processed/{fname}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nc5ddBJjjFjp" + }, + "outputs": [], + "source": [ + "!pip install transformers accelerate scikit-learn pandas pyarrow -q" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "uQPKs1DMjLtb" + }, + "outputs": [], + "source": [ + "# ── This code is optimized for Google Colab ──────────────────────────────\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import torch\n", + "from pathlib import Path\n", + "from torch.utils.data import Dataset\n", + "from transformers import (\n", + " AutoTokenizer, AutoModelForSequenceClassification,\n", + " TrainingArguments, Trainer, EarlyStoppingCallback,\n", + " TrainerCallback,\n", + ")\n", + "from sklearn.metrics import f1_score, precision_score, recall_score, roc_auc_score\n", + "import shutil\n", + "\n", + "print(\"CUDA:\", torch.cuda.is_available())\n", + "print(\"Device:\", torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"CPU\")\n", + "\n", + "DATA_DIR = Path(\"data/processed\")\n", + "OUT_DIR = Path(\"models/deberta-toxicity\")\n", + "OUT_DIR.mkdir(parents=True, exist_ok=True)\n", + "MODEL_ID = \"microsoft/deberta-v3-small\"\n", + "MAX_LEN = 128\n", + "\n", + "\n", + "class SaveBestToDriveCallback(TrainerCallback):\n", + " def __init__(self, drive_path, tokenizer, f1_weight=0.5, auc_weight=0.5):\n", + " self.drive_path = drive_path\n", + " self.tokenizer = tokenizer\n", + " self.f1_weight = f1_weight\n", + " self.auc_weight = auc_weight\n", + " self.best_score = 0.0\n", + "\n", + " def on_evaluate(self, args, state, control, metrics=None, **kwargs):\n", + " current_f1 = metrics.get(\"eval_f1\", 0.0)\n", + " current_auc = metrics.get(\"eval_roc_auc\", 0.0)\n", + "\n", + " combined = (self.f1_weight * current_f1) + (self.auc_weight * current_auc)\n", + "\n", + " print(f\"F1: {current_f1:.4f} | AUC: {current_auc:.4f} | Combined: {combined:.4f} (best: {self.best_score:.4f})\")\n", + "\n", + " if combined > self.best_score:\n", + " self.best_score = combined\n", + " print(f\"✓ New best combined score: {combined:.4f} - saving to Drive...\")\n", + "\n", + " if os.path.exists(self.drive_path):\n", + " shutil.rmtree(self.drive_path)\n", + "\n", + " kwargs[\"model\"].save_pretrained(self.drive_path)\n", + " self.tokenizer.save_pretrained(self.drive_path)\n", + " print(f\"✓ Saved to {self.drive_path}\")\n", + "\n", + "\n", + "class ToxicityDataset(Dataset):\n", + " def __init__(self, df, tokenizer, max_len, name=\"dataset\"):\n", + " print(f\"Tokenizing {name} ({len(df)} samples)...\")\n", + " enc = tokenizer(\n", + " df[\"text\"].tolist(),\n", + " truncation=True,\n", + " padding=\"max_length\",\n", + " max_length=max_len,\n", + " return_tensors=\"pt\",\n", + " verbose=False,\n", + " )\n", + " self.input_ids = enc[\"input_ids\"]\n", + " self.attention_mask = enc[\"attention_mask\"]\n", + " self.labels = torch.tensor(df[\"label\"].values, dtype=torch.long)\n", + " print(f\"Done tokenizing {name}!\")\n", + "\n", + " def __len__(self): return len(self.labels)\n", + "\n", + " def __getitem__(self, idx):\n", + " return {\n", + " \"input_ids\": self.input_ids[idx],\n", + " \"attention_mask\": self.attention_mask[idx],\n", + " \"labels\": self.labels[idx],\n", + " }\n", + "\n", + "\n", + "def compute_metrics(eval_pred):\n", + " logits, labels = eval_pred\n", + " # logits shape is (n, 2) - use argmax for preds, softmax[:,1] for probs\n", + " preds = np.argmax(logits, axis=-1)\n", + " probs = torch.softmax(\n", + " torch.tensor(logits, dtype=torch.float32), dim=-1\n", + " )[:, 1].numpy()\n", + " return {\n", + " \"f1\": f1_score(labels, preds, zero_division=0),\n", + " \"precision\": precision_score(labels, preds, zero_division=0),\n", + " \"recall\": recall_score(labels, preds, zero_division=0),\n", + " \"roc_auc\": roc_auc_score(labels, probs),\n", + " }\n", + "\n", + "\n", + "class WeightedTrainer(Trainer):\n", + " def __init__(self, class_weights, *args, **kwargs):\n", + " super().__init__(*args, **kwargs)\n", + " self.class_weights = class_weights # tensor([w_clean, w_toxic])\n", + "\n", + " def compute_loss(self, model, inputs, return_outputs=False, **kwargs):\n", + " labels = inputs.pop(\"labels\") # long, shape (B,)\n", + " outputs = model(**inputs)\n", + " logits = outputs.logits # shape (B, 2)\n", + "\n", + " loss = torch.nn.functional.cross_entropy(\n", + " logits,\n", + " labels,\n", + " weight=self.class_weights.to(logits.device),\n", + " )\n", + " return (loss, outputs) if return_outputs else loss\n", + "\n", + "\n", + "# ── Data ─────────────────────────────────────────────────────────────────────\n", + "train_df = pd.read_parquet(DATA_DIR / \"train.parquet\")\n", + "val_df = pd.read_parquet(DATA_DIR / \"val.parquet\")\n", + "test_df = pd.read_parquet(DATA_DIR / \"test.parquet\")\n", + "\n", + "\n", + "def stratified_sample(df, n, label_col=\"label\", seed=42):\n", + " return (\n", + " df.groupby(label_col, group_keys=False)\n", + " .apply(lambda x: x.sample(min(len(x), int(n * len(x) / len(df))), random_state=seed))\n", + " .sample(frac=1, random_state=seed) # shuffle\n", + " .reset_index(drop=True)\n", + " )\n", + "\n", + "\n", + "def language_balanced_sample(df, en_cap=300_000, ru_cap=300_000, other_per_lang=8_000, seed=42):\n", + " \"\"\"\n", + " Cap EN and RU, keep a small slice of each other language for context.\n", + " Preserves label balance within each language slice.\n", + " \"\"\"\n", + " parts = []\n", + "\n", + " for lang, group in df.groupby(\"lang\"):\n", + " if lang == \"en\":\n", + " cap = en_cap\n", + " elif lang == \"ru\":\n", + " cap = ru_cap\n", + " elif lang == \"mixed\":\n", + " cap = 20_000\n", + " else:\n", + " cap = other_per_lang\n", + "\n", + " cap_n = min(len(group), cap)\n", + " # stratify within the language group\n", + " sampled = (\n", + " group.groupby(\"label\", group_keys=False)\n", + " .apply(lambda x: x.sample(min(len(x), int(cap_n * len(x) / len(group))), random_state=seed))\n", + " .sample(frac=1, random_state=seed)\n", + " .reset_index(drop=True)\n", + " )\n", + " parts.append(sampled)\n", + "\n", + " return pd.concat(parts).sample(frac=1, random_state=seed).reset_index(drop=True)\n", + "\n", + "\n", + "train_df = language_balanced_sample(train_df, en_cap=300_000, ru_cap=300_000, other_per_lang=8_000)\n", + "\n", + "train_df = stratified_sample(train_df, n=150_000)\n", + "val_df = stratified_sample(val_df, n=20_000)\n", + "\n", + "print(f\"Train: {len(train_df):,} rows\")\n", + "print(f\"Val: {len(val_df):,} rows\")\n", + "print(f\"Test: {len(test_df):,} rows\")\n", + "print(f\"Avg text length: {train_df['text'].str.len().mean():.0f} chars\")\n", + "\n", + "n_clean, n_toxic = (train_df[\"label\"] == 0).sum(), (train_df[\"label\"] == 1).sum()\n", + "class_weights = torch.tensor(\n", + " [np.sqrt(len(train_df) / (2 * n_clean)), np.sqrt(len(train_df) / (2 * n_toxic))],\n", + " dtype=torch.float32\n", + ")\n", + "print(f\"Class weights clean: {class_weights[0]:.3f}, toxic: {class_weights[1]:.3f}\")\n", + "\n", + "print(f\"n_clean: {n_clean:,}, n_toxic: {n_toxic:,}\")\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)\n", + "model = AutoModelForSequenceClassification.from_pretrained(MODEL_ID, num_labels=2)\n", + "\n", + "train_ds = ToxicityDataset(train_df, tokenizer, MAX_LEN, name=\"train\")\n", + "val_ds = ToxicityDataset(val_df, tokenizer, MAX_LEN, name=\"val\")\n", + "\n", + "# ── Training args — fp16/bf16 hardcoded, no cfg object ───────────────────────\n", + "args = TrainingArguments(\n", + " output_dir=str(OUT_DIR),\n", + " num_train_epochs=3,\n", + " per_device_train_batch_size=16,\n", + " per_device_eval_batch_size=64,\n", + " gradient_accumulation_steps=4,\n", + " learning_rate=8e-7,\n", + " warmup_ratio=0.06,\n", + " weight_decay=0.01,\n", + " max_grad_norm=1.0,\n", + " fp16=False,\n", + " bf16=True,\n", + " lr_scheduler_type=\"cosine\",\n", + " eval_strategy=\"steps\",\n", + " eval_steps=100,\n", + " save_strategy=\"steps\",\n", + " save_steps=100,\n", + " load_best_model_at_end=True,\n", + " metric_for_best_model=\"f1\",\n", + " greater_is_better=True,\n", + " dataloader_num_workers=2,\n", + " report_to=\"none\",\n", + " logging_steps=50,\n", + ")\n", + "\n", + "trainer = WeightedTrainer(\n", + " class_weights=class_weights,\n", + " model=model,\n", + " args=args,\n", + " train_dataset=train_ds,\n", + " eval_dataset=val_ds,\n", + " compute_metrics=compute_metrics,\n", + " callbacks=[\n", + " EarlyStoppingCallback(early_stopping_patience=5),\n", + " SaveBestToDriveCallback(\n", + " drive_path=\"/content/drive/MyDrive/deberta-toxicity-best\",\n", + " tokenizer=tokenizer,\n", + " ),\n", + " ],\n", + ")\n", + "\n", + "model.eval()\n", + "with torch.no_grad():\n", + " sample = {k: v[:8].to(model.device) for k, v in {\n", + " \"input_ids\": train_ds.input_ids,\n", + " \"attention_mask\": train_ds.attention_mask\n", + " }.items()}\n", + " out = model(**sample)\n", + " print(\"Logits:\", out.logits)\n", + " print(\"Preds:\", out.logits.argmax(-1))\n", + " print(\"Labels:\", train_ds.labels[:8])\n", + "\n", + "trainer.train()\n", + "\n", + "test_ds = ToxicityDataset(test_df, tokenizer, MAX_LEN, name=\"test\")\n", + "test_results = trainer.evaluate(test_ds)\n", + "print(test_results)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "B8_w05_w1Fnq" + }, + "outputs": [], + "source": [ + "import torch\n", + "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n", + "\n", + "MAX_LEN = 128 # must match what you used during training\n", + "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + "\n", + "\n", + "def predict(texts, model, tokenizer, threshold=0.5):\n", + " model.eval()\n", + " enc = tokenizer(texts, truncation=True, padding=True, max_length=MAX_LEN, return_tensors=\"pt\").to(device)\n", + " with torch.no_grad():\n", + " logits = model(**enc).logits\n", + " probs = torch.softmax(logits, dim=-1)[:, 1].cpu().numpy()\n", + " preds = (probs >= threshold).astype(int)\n", + " return probs, preds\n", + "\n", + "\n", + "# Load the saved model and tokenizer from Drive\n", + "drive_path = \"/content/drive/MyDrive/deberta-toxicity-best\"\n", + "print(f\"Loading model from {drive_path}...\")\n", + "inference_tokenizer = AutoTokenizer.from_pretrained(drive_path)\n", + "inference_model = AutoModelForSequenceClassification.from_pretrained(drive_path).to(device)\n", + "print(f\"Running on {device}\")\n", + "\n", + "samples = [\n", + " # English — obvious clean\n", + " \"Thanks for the detailed explanation, really helped me understand.\",\n", + " \"Happy birthday! Hope you have a wonderful day.\",\n", + "\n", + " # English — obvious toxic\n", + " \"Kill yourself you worthless piece of garbage.\",\n", + " \"All people like you should be exterminated.\",\n", + "\n", + " # English — subtle toxic (no slurs, but threatening)\n", + " \"You better watch your back, I know where you live.\",\n", + " \"Keep talking and see what happens to you.\",\n", + "\n", + " # English — edgy but clean (criticism, dark humor)\n", + " \"This policy is absolutely brain-dead and whoever designed it should be fired.\",\n", + " \"The movie was torture to sit through, completely unwatchable.\",\n", + "\n", + " # Russian — obvious clean\n", + " \"Спасибо за помощь, очень ценю ваше время.\", # Thank you for the help\n", + " \"Отличная работа, продолжайте в том же духе!\", # Great work, keep it up\n", + "\n", + " # Russian — obvious toxic\n", + " \"Ты тупая мразь, убирайся отсюда.\", # You're a disgusting idiot, get out\n", + " \"Такие люди как ты должны сдохнуть.\", # People like you should die\n", + "\n", + " # Russian — subtle toxic\n", + " \"Ну и рожа у тебя, смотреть противно.\", # What an ugly face, disgusting to look at\n", + " \"Понаехали тут всякие, житья не дают.\", # All these migrants coming here, no peace\n", + "\n", + " # Russian — edgy but clean\n", + " \"Этот закон — полный идиотизм от начала до конца.\", # This law is complete idiocy\n", + " \"Фильм — редкостная чушь, зря потратил время.\", # The film is utter nonsense, wasted my time\n", + "\n", + " # Mixed/tricky — context-dependent\n", + " \"You're so bad at this it's actually impressive.\", # ambiguous — could be playful\n", + " \"I want to kill this level, it's so hard.\", # gaming context, clean\n", + " \"Он просто животное.\", # He's just an animal — ambiguous insult\n", + " \"This is the worst community I've ever seen.\", # harsh but not toxic\n", + "]\n", + "\n", + "probs, preds = predict(samples, inference_model, inference_tokenizer)\n", + "for text, prob, pred in zip(samples, probs, preds, strict=False):\n", + " label = \"TOXIC\" if pred else \"CLEAN\"\n", + " print(f\"[{label} {prob:.2f}] {text[:60]}\")" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/trainer/google_colab_train.py b/trainer/google_colab_train.py index 2193b69..5170b83 100644 --- a/trainer/google_colab_train.py +++ b/trainer/google_colab_train.py @@ -1,24 +1,39 @@ # ── This code is optimized for Google Colab ────────────────────────────── -import os, numpy as np, pandas as pd, torch +import os +import shutil from pathlib import Path + +import numpy as np +import pandas as pd +import torch +from sklearn.metrics import ( + f1_score, + precision_score, + recall_score, + roc_auc_score, +) from torch.utils.data import Dataset from transformers import ( - AutoTokenizer, AutoModelForSequenceClassification, - TrainingArguments, Trainer, EarlyStoppingCallback, + AutoModelForSequenceClassification, + AutoTokenizer, + EarlyStoppingCallback, + Trainer, TrainerCallback, + TrainingArguments, ) -from sklearn.metrics import f1_score, precision_score, recall_score, roc_auc_score -from tqdm import tqdm -import shutil print("CUDA:", torch.cuda.is_available()) -print("Device:", torch.cuda.get_device_name(0) if torch.cuda.is_available() else "CPU") +print( + "Device:", + torch.cuda.get_device_name(0) if torch.cuda.is_available() else "CPU", +) DATA_DIR = Path("data/processed") -OUT_DIR = Path("models/deberta-toxicity") +OUT_DIR = Path("models/deberta-toxicity") OUT_DIR.mkdir(parents=True, exist_ok=True) MODEL_ID = "microsoft/deberta-v3-small" -MAX_LEN = 128 +MAX_LEN = 128 + class SaveBestToDriveCallback(TrainerCallback): def __init__(self, drive_path, tokenizer, f1_weight=0.5, auc_weight=0.5): @@ -29,24 +44,31 @@ def __init__(self, drive_path, tokenizer, f1_weight=0.5, auc_weight=0.5): self.best_score = 0.0 def on_evaluate(self, args, state, control, metrics=None, **kwargs): - current_f1 = metrics.get("eval_f1", 0.0) + current_f1 = metrics.get("eval_f1", 0.0) current_auc = metrics.get("eval_roc_auc", 0.0) - - combined = (self.f1_weight * current_f1) + (self.auc_weight * current_auc) - - print(f"F1: {current_f1:.4f} | AUC: {current_auc:.4f} | Combined: {combined:.4f} (best: {self.best_score:.4f})") - + + combined = (self.f1_weight * current_f1) + ( + self.auc_weight * current_auc + ) + + print( + f"F1: {current_f1:.4f} | AUC: {current_auc:.4f} | Combined: {combined:.4f} (best: {self.best_score:.4f})" + ) + if combined > self.best_score: self.best_score = combined - print(f"✓ New best combined score: {combined:.4f} - saving to Drive...") - + print( + f"✓ New best combined score: {combined:.4f} - saving to Drive..." + ) + if os.path.exists(self.drive_path): shutil.rmtree(self.drive_path) - + kwargs["model"].save_pretrained(self.drive_path) self.tokenizer.save_pretrained(self.drive_path) print(f"✓ Saved to {self.drive_path}") + class ToxicityDataset(Dataset): def __init__(self, df, tokenizer, max_len, name="dataset"): print(f"Tokenizing {name} ({len(df)} samples)...") @@ -58,43 +80,46 @@ def __init__(self, df, tokenizer, max_len, name="dataset"): return_tensors="pt", verbose=False, ) - self.input_ids = enc["input_ids"] + self.input_ids = enc["input_ids"] self.attention_mask = enc["attention_mask"] - self.labels = torch.tensor(df["label"].values, dtype=torch.long) + self.labels = torch.tensor(df["label"].values, dtype=torch.long) print(f"Done tokenizing {name}!") - def __len__(self): return len(self.labels) + def __len__(self): + return len(self.labels) def __getitem__(self, idx): return { - "input_ids": self.input_ids[idx], + "input_ids": self.input_ids[idx], "attention_mask": self.attention_mask[idx], - "labels": self.labels[idx], + "labels": self.labels[idx], } + def compute_metrics(eval_pred): logits, labels = eval_pred # logits shape is (n, 2) - use argmax for preds, softmax[:,1] for probs preds = np.argmax(logits, axis=-1) - probs = torch.softmax( - torch.tensor(logits, dtype=torch.float32), dim=-1 - )[:, 1].numpy() + probs = torch.softmax(torch.tensor(logits, dtype=torch.float32), dim=-1)[ + :, 1 + ].numpy() return { - "f1": f1_score(labels, preds, zero_division=0), + "f1": f1_score(labels, preds, zero_division=0), "precision": precision_score(labels, preds, zero_division=0), - "recall": recall_score(labels, preds, zero_division=0), - "roc_auc": roc_auc_score(labels, probs), + "recall": recall_score(labels, preds, zero_division=0), + "roc_auc": roc_auc_score(labels, probs), } + class WeightedTrainer(Trainer): def __init__(self, class_weights, *args, **kwargs): super().__init__(*args, **kwargs) self.class_weights = class_weights # tensor([w_clean, w_toxic]) def compute_loss(self, model, inputs, return_outputs=False, **kwargs): - labels = inputs.pop("labels") # long, shape (B,) + labels = inputs.pop("labels") # long, shape (B,) outputs = model(**inputs) - logits = outputs.logits # shape (B, 2) + logits = outputs.logits # shape (B, 2) loss = torch.nn.functional.cross_entropy( logits, @@ -106,70 +131,99 @@ def compute_loss(self, model, inputs, return_outputs=False, **kwargs): # ── Data ───────────────────────────────────────────────────────────────────── train_df = pd.read_parquet(DATA_DIR / "train.parquet") -val_df = pd.read_parquet(DATA_DIR / "val.parquet") -test_df = pd.read_parquet(DATA_DIR / "test.parquet") +val_df = pd.read_parquet(DATA_DIR / "val.parquet") +test_df = pd.read_parquet(DATA_DIR / "test.parquet") + def stratified_sample(df, n, label_col="label", seed=42): return ( df.groupby(label_col, group_keys=False) - .apply(lambda x: x.sample(min(len(x), int(n * len(x) / len(df))), random_state=seed)) - .sample(frac=1, random_state=seed) # shuffle - .reset_index(drop=True) + .apply( + lambda x: x.sample( + min(len(x), int(n * len(x) / len(df))), random_state=seed + ) + ) + .sample(frac=1, random_state=seed) # shuffle + .reset_index(drop=True) ) -def language_balanced_sample(df, en_cap=300_000, ru_cap=300_000, other_per_lang=8_000, seed=42): + +def language_balanced_sample( + df, en_cap=300_000, ru_cap=300_000, other_per_lang=8_000, seed=42 +): """ Cap EN and RU, keep a small slice of each other language for context. Preserves label balance within each language slice. """ parts = [] - + for lang, group in df.groupby("lang"): if lang == "en": cap = en_cap elif lang == "ru": cap = ru_cap elif lang == "mixed": - cap = 20_000 + cap = 20_000 else: cap = other_per_lang - + cap_n = min(len(group), cap) # stratify within the language group sampled = ( group.groupby("label", group_keys=False) - .apply(lambda x: x.sample(min(len(x), int(cap_n * len(x) / len(group))), random_state=seed)) - .sample(frac=1, random_state=seed) - .reset_index(drop=True) + .apply( + lambda x: x.sample( + min(len(x), int(cap_n * len(x) / len(group))), + random_state=seed, + ) + ) + .sample(frac=1, random_state=seed) + .reset_index(drop=True) ) parts.append(sampled) - - return pd.concat(parts).sample(frac=1, random_state=seed).reset_index(drop=True) -train_df = language_balanced_sample(train_df, en_cap=300_000, ru_cap=300_000, other_per_lang=8_000) + return ( + pd.concat(parts) + .sample(frac=1, random_state=seed) + .reset_index(drop=True) + ) + + +train_df = language_balanced_sample( + train_df, en_cap=300_000, ru_cap=300_000, other_per_lang=8_000 +) train_df = stratified_sample(train_df, n=150_000) -val_df = stratified_sample(val_df, n=20_000) +val_df = stratified_sample(val_df, n=20_000) print(f"Train: {len(train_df):,} rows") print(f"Val: {len(val_df):,} rows") print(f"Test: {len(test_df):,} rows") print(f"Avg text length: {train_df['text'].str.len().mean():.0f} chars") -n_clean, n_toxic = (train_df["label"]==0).sum(), (train_df["label"]==1).sum() +n_clean, n_toxic = (train_df["label"] == 0).sum(), ( + train_df["label"] == 1 +).sum() class_weights = torch.tensor( - [np.sqrt(len(train_df)/(2*n_clean)), np.sqrt(len(train_df)/(2*n_toxic))], - dtype=torch.float32 + [ + np.sqrt(len(train_df) / (2 * n_clean)), + np.sqrt(len(train_df) / (2 * n_toxic)), + ], + dtype=torch.float32, +) +print( + f"Class weights clean: {class_weights[0]:.3f}, toxic: {class_weights[1]:.3f}" ) -print(f"Class weights clean: {class_weights[0]:.3f}, toxic: {class_weights[1]:.3f}") print(f"n_clean: {n_clean:,}, n_toxic: {n_toxic:,}") tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) -model = AutoModelForSequenceClassification.from_pretrained(MODEL_ID, num_labels=2) +model = AutoModelForSequenceClassification.from_pretrained( + MODEL_ID, num_labels=2 +) train_ds = ToxicityDataset(train_df, tokenizer, MAX_LEN, name="train") -val_ds = ToxicityDataset(val_df, tokenizer, MAX_LEN, name="val") +val_ds = ToxicityDataset(val_df, tokenizer, MAX_LEN, name="val") # ── Training args — fp16/bf16 hardcoded, no cfg object ─────────────────────── args = TrainingArguments( @@ -178,15 +232,15 @@ def language_balanced_sample(df, en_cap=300_000, ru_cap=300_000, other_per_lang= per_device_train_batch_size=16, per_device_eval_batch_size=64, gradient_accumulation_steps=4, - learning_rate=8e-7, + learning_rate=8e-7, warmup_ratio=0.06, weight_decay=0.01, - max_grad_norm=1.0, + max_grad_norm=1.0, fp16=False, bf16=True, lr_scheduler_type="cosine", eval_strategy="steps", - eval_steps=100, + eval_steps=100, save_strategy="steps", save_steps=100, load_best_model_at_end=True, @@ -194,7 +248,7 @@ def language_balanced_sample(df, en_cap=300_000, ru_cap=300_000, other_per_lang= greater_is_better=True, dataloader_num_workers=2, report_to="none", - logging_steps=50, + logging_steps=50, ) trainer = WeightedTrainer( @@ -215,10 +269,13 @@ def language_balanced_sample(df, en_cap=300_000, ru_cap=300_000, other_per_lang= model.eval() with torch.no_grad(): - sample = {k: v[:8].to(model.device) for k, v in { - "input_ids": train_ds.input_ids, - "attention_mask": train_ds.attention_mask - }.items()} + sample = { + k: v[:8].to(model.device) + for k, v in { + "input_ids": train_ds.input_ids, + "attention_mask": train_ds.attention_mask, + }.items() + } out = model(**sample) print("Logits:", out.logits) print("Preds:", out.logits.argmax(-1)) @@ -228,4 +285,4 @@ def language_balanced_sample(df, en_cap=300_000, ru_cap=300_000, other_per_lang= test_ds = ToxicityDataset(test_df, tokenizer, MAX_LEN, name="test") test_results = trainer.evaluate(test_ds) -print(test_results) \ No newline at end of file +print(test_results) diff --git a/trainer/onnx_exporter.py b/trainer/onnx_exporter.py index 5428e87..728ffe9 100644 --- a/trainer/onnx_exporter.py +++ b/trainer/onnx_exporter.py @@ -1,23 +1,29 @@ -import torch from pathlib import Path -from transformers import AutoModelForSequenceClassification, AutoTokenizer -from optimum.onnxruntime import ORTModelForSequenceClassification + +import torch from optimum.exporters.onnx import main_export +from optimum.onnxruntime import ORTModelForSequenceClassification +from transformers import AutoModelForSequenceClassification, AutoTokenizer + +from logger.custom_logger import get_logger DATA_DIR = Path("data/processed") -OUT_DIR = Path("models/deberta-qat") +OUT_DIR = Path("models/deberta-qat") ONNX_DIR = Path("models/deberta-qat-onnx") -BEST_DIR = Path("models/deberta-qat-best") +BEST_DIR = Path("models/deberta-qat-best") for d in [OUT_DIR, ONNX_DIR, BEST_DIR]: d.mkdir(parents=True, exist_ok=True) -BASE_ID = "microsoft/deberta-v3-small" +BASE_ID = "microsoft/deberta-v3-small" MODEL_ID = "esclient/deberta-toxicity-model" +log = get_logger(__name__) -print(f"Loading {MODEL_ID}...") +log.info(f"Loading {MODEL_ID}...") tokenizer = AutoTokenizer.from_pretrained(BASE_ID) -model = AutoModelForSequenceClassification.from_pretrained(MODEL_ID, num_labels=2) +model = AutoModelForSequenceClassification.from_pretrained( + MODEL_ID, num_labels=2 +) main_export( model_name_or_path="esclient/deberta-toxicity-model-qat", @@ -26,7 +32,14 @@ opset=14, ) -model = ORTModelForSequenceClassification.from_pretrained("models/deberta-qat-onnx-v2") -inputs = tokenizer("Kill yourself you worthless piece of garbage.", return_tensors="pt", max_length=128, truncation=True) +model = ORTModelForSequenceClassification.from_pretrained( + "models/deberta-qat-onnx-v2" +) +inputs = tokenizer( + "Kill yourself you worthless piece of garbage.", + return_tensors="pt", + max_length=128, + truncation=True, +) outputs = model(**inputs) -print(torch.softmax(outputs.logits, dim=-1)) \ No newline at end of file +log.debug(f"ONNX logits softmax: {torch.softmax(outputs.logits, dim=-1)}") diff --git a/trainer/quantizer.py b/trainer/quantizer.py index 7d18ff8..33d4158 100644 --- a/trainer/quantizer.py +++ b/trainer/quantizer.py @@ -1,15 +1,20 @@ +from pathlib import Path + from optimum.onnxruntime import ORTQuantizer from optimum.onnxruntime.configuration import AutoQuantizationConfig -from pathlib import Path + +from logger.custom_logger import get_logger ONNX_DIR = Path("models/deberta-qat-onnx-v2") -INT8_DIR = Path("models/deberta-qat-int8-v2") +INT8_DIR = Path("models/deberta-qat-int8-v2") +log = get_logger(__name__) quantizer = ORTQuantizer.from_pretrained(str(ONNX_DIR)) # avx2 = Intel/AMD CPUs with AVX2 support; use avx512 for newer Intel, arm64 for Apple Silicon / ARM servers -# Be careful of what you are using, because the wrong format can cause significant errors in the quantized model's outputs. +# Be careful when selecting a format: the wrong target can cause +# significant errors in the quantized model's outputs. -qconfig = AutoQuantizationConfig.avx2(is_static=False, per_channel=False) +qconfig = AutoQuantizationConfig.avx2(is_static=False, per_channel=False) quantizer.quantize(save_dir=str(INT8_DIR), quantization_config=qconfig) -print("Done") \ No newline at end of file +log.info("Quantization complete") diff --git a/trainer/train_qat.py b/trainer/train_qat.py index a20b076..6649251 100644 --- a/trainer/train_qat.py +++ b/trainer/train_qat.py @@ -1,54 +1,82 @@ -import os, numpy as np, pandas as pd, torch, shutil, time +import shutil from pathlib import Path + +import numpy as np +import pandas as pd +import torch +from sklearn.metrics import ( + f1_score, + precision_score, + recall_score, + roc_auc_score, +) from torch.utils.data import Dataset from transformers import ( AutoModelForSequenceClassification, - TrainingArguments, Trainer, - EarlyStoppingCallback, TrainerCallback, AutoTokenizer, + EarlyStoppingCallback, + Trainer, + TrainerCallback, + TrainingArguments, ) -from sklearn.metrics import f1_score, precision_score, recall_score, roc_auc_score -print("PyTorch:", torch.__version__) -print("CUDA: ", torch.cuda.is_available()) -print("Device: ", torch.cuda.get_device_name(0) if torch.cuda.is_available() else "CPU") -print("VRAM: ", round(torch.cuda.get_device_properties(0).total_memory / 1024**3, 1), "GB") +from logger.custom_logger import get_logger + +log = get_logger(__name__) + +log.info(f"PyTorch: {torch.__version__}") +log.info(f"CUDA: {torch.cuda.is_available()}") +device_name = ( + torch.cuda.get_device_name(0) if torch.cuda.is_available() else "CPU" +) +log.info(f"Device: {device_name}") +if torch.cuda.is_available(): + vram_gb = round( + torch.cuda.get_device_properties(0).total_memory / 1024**3, 1 + ) + log.info(f"VRAM: {vram_gb} GB") DATA_DIR = Path("data/processed") -OUT_DIR = Path("models/deberta-qat") -BEST_DIR = Path("models/deberta-qat-best") +OUT_DIR = Path("models/deberta-qat") +BEST_DIR = Path("models/deberta-qat-best") for d in [OUT_DIR, BEST_DIR]: d.mkdir(parents=True, exist_ok=True) -BASE_ID = "microsoft/deberta-v3-small" +BASE_ID = "microsoft/deberta-v3-small" MODEL_ID = "esclient/deberta-toxicity-model" -MAX_LEN = 128 +MAX_LEN = 128 + class SaveBestCallback(TrainerCallback): def __init__(self, save_path, tokenizer, f1_weight=0.5, auc_weight=0.5): - self.save_path = save_path - self.tokenizer = tokenizer - self.f1_weight = f1_weight + self.save_path = save_path + self.tokenizer = tokenizer + self.f1_weight = f1_weight self.auc_weight = auc_weight self.best_score = 0.0 - def on_evaluate(self, args, state, control, metrics=None, **kwargs): - current_f1 = metrics.get("eval_f1", 0.0) + def on_evaluate(self, _args, _state, _control, metrics=None, **kwargs): + current_f1 = metrics.get("eval_f1", 0.0) current_auc = metrics.get("eval_roc_auc", 0.0) - combined = (self.f1_weight * current_f1) + (self.auc_weight * current_auc) - print(f"F1: {current_f1:.4f} | AUC: {current_auc:.4f} | Combined: {combined:.4f} (best: {self.best_score:.4f})") + combined = (self.f1_weight * current_f1) + ( + self.auc_weight * current_auc + ) + log.info( + f"F1: {current_f1:.4f} | AUC: {current_auc:.4f} | Combined: {combined:.4f} (best: {self.best_score:.4f})" + ) if combined > self.best_score: self.best_score = combined if self.save_path.exists(): shutil.rmtree(self.save_path) kwargs["model"].save_pretrained(self.save_path) self.tokenizer.save_pretrained(self.save_path) - print(f"New best {combined:.4f} saved → {self.save_path}") + log.info(f"New best {combined:.4f} saved -> {self.save_path}") + class ToxicityDataset(Dataset): def __init__(self, df, tokenizer, max_len, name="dataset"): - print(f"Tokenizing {name} ({len(df):,} samples)...") + log.info(f"Tokenizing {name} ({len(df):,} samples)...") enc = tokenizer( df["text"].tolist(), truncation=True, @@ -56,118 +84,154 @@ def __init__(self, df, tokenizer, max_len, name="dataset"): max_length=max_len, return_tensors="pt", ) - self.input_ids = enc["input_ids"] + self.input_ids = enc["input_ids"] self.attention_mask = enc["attention_mask"] - self.labels = torch.tensor(df["label"].values, dtype=torch.long) - print(f"Done tokenizing {name}!") + self.labels = torch.tensor(df["label"].values, dtype=torch.long) + log.info(f"Done tokenizing {name}!") - def __len__(self): return len(self.labels) + def __len__(self): + return len(self.labels) def __getitem__(self, idx): return { - "input_ids": self.input_ids[idx], + "input_ids": self.input_ids[idx], "attention_mask": self.attention_mask[idx], - "labels": self.labels[idx], + "labels": self.labels[idx], } + def compute_metrics(eval_pred): logits, labels = eval_pred preds = np.argmax(logits, axis=-1) - probs = torch.softmax(torch.tensor(logits, dtype=torch.float32), dim=-1)[:, 1].numpy() + probs = torch.softmax(torch.tensor(logits, dtype=torch.float32), dim=-1)[ + :, 1 + ].numpy() return { - "f1": f1_score(labels, preds, zero_division=0), + "f1": f1_score(labels, preds, zero_division=0), "precision": precision_score(labels, preds, zero_division=0), - "recall": recall_score(labels, preds, zero_division=0), - "roc_auc": roc_auc_score(labels, probs), + "recall": recall_score(labels, preds, zero_division=0), + "roc_auc": roc_auc_score(labels, probs), } + class WeightedTrainer(Trainer): def __init__(self, class_weights, *args, **kwargs): super().__init__(*args, **kwargs) self.class_weights = class_weights - def compute_loss(self, model, inputs, return_outputs=False, **kwargs): - labels = inputs.pop("labels") + def compute_loss(self, model, inputs, return_outputs=False, **_kwargs): + labels = inputs.pop("labels") outputs = model(**inputs) - loss = torch.nn.functional.cross_entropy( - outputs.logits, labels, + loss = torch.nn.functional.cross_entropy( + outputs.logits, + labels, weight=self.class_weights.to(outputs.logits.device), ) return (loss, outputs) if return_outputs else loss -print("Loading parquet files...") + +log.info("Loading parquet files...") train_df = pd.read_parquet(DATA_DIR / "train.parquet") -val_df = pd.read_parquet(DATA_DIR / "val.parquet") +val_df = pd.read_parquet(DATA_DIR / "val.parquet") + def stratified_sample(df, n, label_col="label", seed=42): return ( df.groupby(label_col, group_keys=False) - .apply(lambda x: x.sample(min(len(x), int(n * len(x) / len(df))), random_state=seed), include_groups=True) - .sample(frac=1, random_state=seed) - .reset_index(drop=True) + .apply( + lambda x: x.sample( + min(len(x), int(n * len(x) / len(df))), random_state=seed + ), + include_groups=True, + ) + .sample(frac=1, random_state=seed) + .reset_index(drop=True) ) -def language_balanced_sample(df, en_cap=300_000, ru_cap=300_000, other_per_lang=8_000, seed=42): + +def language_balanced_sample( + df, en_cap=300_000, ru_cap=300_000, other_per_lang=8_000, seed=42 +): parts = [] for lang, group in df.groupby("lang"): - cap = {"en": en_cap, "ru": ru_cap, "mixed": 20_000}.get(lang, other_per_lang) + cap = {"en": en_cap, "ru": ru_cap, "mixed": 20_000}.get( + lang, other_per_lang + ) cap_n = min(len(group), cap) parts.append( group.groupby("label", group_keys=False) - .apply(lambda x: x.sample(min(len(x), int(cap_n * len(x) / len(group))), random_state=seed), include_groups=True) - .sample(frac=1, random_state=seed) - .reset_index(drop=True) + .apply( + lambda x, cap_n=cap_n, group_len=len(group): x.sample( + min(len(x), int(cap_n * len(x) / group_len)), + random_state=seed, + ), + include_groups=True, + ) + .sample(frac=1, random_state=seed) + .reset_index(drop=True) ) - return pd.concat(parts).sample(frac=1, random_state=seed).reset_index(drop=True) + return ( + pd.concat(parts) + .sample(frac=1, random_state=seed) + .reset_index(drop=True) + ) + train_df = language_balanced_sample(train_df) train_df = stratified_sample(train_df, n=50_000) -val_df = stratified_sample(val_df, n=10_000) +val_df = stratified_sample(val_df, n=10_000) -print(f"Train: {len(train_df):,} | Val: {len(val_df):,}") +log.info(f"Train: {len(train_df):,} | Val: {len(val_df):,}") n_clean = (train_df["label"] == 0).sum() n_toxic = (train_df["label"] == 1).sum() class_weights = torch.tensor( - [np.sqrt(len(train_df) / (2 * n_clean)), np.sqrt(len(train_df) / (2 * n_toxic))], + [ + np.sqrt(len(train_df) / (2 * n_clean)), + np.sqrt(len(train_df) / (2 * n_toxic)), + ], dtype=torch.float32, ) -print(f"Weights — clean: {class_weights[0]:.3f}, toxic: {class_weights[1]:.3f}") +log.info( + f"Weights — clean: {class_weights[0]:.3f}, toxic: {class_weights[1]:.3f}" +) -print(f"Loading {MODEL_ID}...") +log.info(f"Loading {MODEL_ID}...") tokenizer = AutoTokenizer.from_pretrained(BASE_ID) -model = AutoModelForSequenceClassification.from_pretrained(MODEL_ID, num_labels=2) +model = AutoModelForSequenceClassification.from_pretrained( + MODEL_ID, num_labels=2 +) train_ds = ToxicityDataset(train_df, tokenizer, MAX_LEN, "train") -val_ds = ToxicityDataset(val_df, tokenizer, MAX_LEN, "val") +val_ds = ToxicityDataset(val_df, tokenizer, MAX_LEN, "val") model.eval() with torch.no_grad(): sample = { - "input_ids": train_ds.input_ids[:8], + "input_ids": train_ds.input_ids[:8], "attention_mask": train_ds.attention_mask[:8], } out = model(**sample) - print("Logits:", out.logits) - print("Preds: ", out.logits.argmax(-1)) + log.debug(f"Logits: {out.logits}") + log.debug(f"Preds: {out.logits.argmax(-1)}") # ── Training args ─────────────────────────────────── -# This snippet of code has to be adjusted based on your GPU's VRAM. -# The current settings are for a 4GB card, which is quite tight for DeBERTa training. +# This snippet of code has to be adjusted based on your GPU's VRAM. +# The current settings are for a 4GB card, which is quite tight for DeBERTa training. # If you have more VRAM, you can increase batch sizes and reduce gradient accumulation steps for faster training. # Also note that some GPUs support fp16 but not bf16, so adjust those settings accordingly. args = TrainingArguments( output_dir=str(OUT_DIR), num_train_epochs=2, - per_device_train_batch_size=8, - per_device_eval_batch_size=32, - gradient_accumulation_steps=8, + per_device_train_batch_size=8, + per_device_eval_batch_size=32, + gradient_accumulation_steps=8, learning_rate=2e-7, warmup_ratio=0.06, weight_decay=0.01, max_grad_norm=1.0, - fp16=True, + fp16=True, bf16=False, lr_scheduler_type="cosine", eval_strategy="steps", @@ -184,8 +248,10 @@ def language_balanced_sample(df, en_cap=300_000, ru_cap=300_000, other_per_lang= trainer = WeightedTrainer( class_weights=class_weights, - model=model, args=args, - train_dataset=train_ds, eval_dataset=val_ds, + model=model, + args=args, + train_dataset=train_ds, + eval_dataset=val_ds, compute_metrics=compute_metrics, callbacks=[ EarlyStoppingCallback(early_stopping_patience=3), @@ -194,4 +260,4 @@ def language_balanced_sample(df, en_cap=300_000, ru_cap=300_000, other_per_lang= ) trainer.train() -print("Training complete.") +log.info("Training complete.") From 615b39bcad05ae66e7ffdb3ea217497424308315 Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 5 May 2026 18:41:54 +0200 Subject: [PATCH 4/8] Fixed logging system. the logs are in abseil custom format. fixed a the most of linter errors, a few remain --- data/augment_dataset.py | 25 ++++--- data/prepare_dataset.py | 7 +- logger/__init__.py | 1 - pyproject.toml | 3 +- src/aivalidatorservice/constants.py | 2 +- src/aivalidatorservice/handler/handler.py | 6 +- src/aivalidatorservice/handler/moderate.py | 2 +- .../logger}/custom_logger.py | 24 ++++-- src/aivalidatorservice/model/loader.py | 9 ++- src/aivalidatorservice/server.py | 2 +- src/aivalidatorservice/service/moderate.py | 2 +- src/aivalidatorservice/service/service.py | 2 +- src/aivalidatorservice/settings.py | 10 +-- tests/behavioral/test_golden_path.py | 14 +++- tests/conftest.py | 11 ++- tests/contract/test_grpc_contract.py | 12 ++- tests/tokenizer/test_tokenizer_shapes.py | 7 +- tools/common.just | 4 +- trainer/onnx_exporter.py | 4 +- trainer/quantizer.py | 2 +- trainer/train_qat.py | 74 +++++++++++++++---- 21 files changed, 156 insertions(+), 67 deletions(-) delete mode 100644 logger/__init__.py rename {logger => src/aivalidatorservice/logger}/custom_logger.py (60%) diff --git a/data/augment_dataset.py b/data/augment_dataset.py index 721b3a0..0c9ea16 100644 --- a/data/augment_dataset.py +++ b/data/augment_dataset.py @@ -22,12 +22,13 @@ import random from collections import defaultdict +from typing import Any import numpy as np import pandas as pd from sklearn.feature_extraction.text import TfidfVectorizer -from logger.custom_logger import get_logger +from aivalidatorservice.logger.custom_logger import get_logger MAX_SYNTHETIC_RATIO = 0.40 @@ -132,11 +133,11 @@ def _remove_space(boundary_class: str) -> bool: return random.random() < p -def aug_full_concat(text: str, **_) -> str: +def aug_full_concat(text: str, **_: Any) -> str: return "".join(_tokenize(text)) -def aug_partial_concat(text: str, vocab: set[str], **_) -> str: +def aug_partial_concat(text: str, vocab: set[str], **_: Any) -> str: tokens = _tokenize(text) if len(tokens) <= 1: return text @@ -153,7 +154,7 @@ def aug_context_filler( text: str, filler_pool: list[str], n_fillers: int = 1, - **_, + **_: Any, ) -> str: fillers = random.sample(filler_pool, min(n_fillers, len(filler_pool))) parts = [*fillers, text] @@ -165,7 +166,7 @@ def aug_crosslang_concat( text: str, lang: str, lang_pool: dict[str, list[str]], - **_, + **_: Any, ) -> str | None: other_langs = [ lang_code @@ -275,9 +276,11 @@ def build_augmented_dataset(df: pd.DataFrame) -> pd.DataFrame: targets = _target_counts(df, buckets) log.debug(f"[augment] Synthetic targets per bucket: {dict(targets)}") - rows: list[dict] = [] + rows: list[dict[str, object]] = [] - def add_row(text, label, lang, source, category): + def add_row( + text: object, label: int, lang: str, source: str, category: str + ) -> None: text = str(text).strip() if 3 <= len(text) <= 512: rows.append( @@ -342,11 +345,13 @@ def add_row(text, label, lang, source, category): n = targets.get(("concatenation", 1), 0) // 2 # share budget with concat log.info(f"[augment] synthetic_crosslang: generating {n} rows") for rec in random.choices(toxic_df.to_dict("records"), k=n): - text = aug_crosslang_concat( + cross_text = aug_crosslang_concat( rec["text"], lang=rec["lang"], lang_pool=lang_pool ) - if text: - add_row(text, 1, "mixed", "synthetic_crosslang", "concatenation") + if cross_text: + add_row( + cross_text, 1, "mixed", "synthetic_crosslang", "concatenation" + ) synth_df = pd.DataFrame(rows) if synth_df.empty: diff --git a/data/prepare_dataset.py b/data/prepare_dataset.py index 8898b70..c255807 100644 --- a/data/prepare_dataset.py +++ b/data/prepare_dataset.py @@ -5,6 +5,7 @@ All datasets are pulled from HuggingFace: esclient/toxicity_multilanguage_dataset """ +from collections.abc import Sequence from pathlib import Path import pandas as pd @@ -13,7 +14,7 @@ from huggingface_hub import hf_hub_download from sklearn.model_selection import train_test_split -from logger.custom_logger import get_logger +from aivalidatorservice.logger.custom_logger import get_logger OUT = Path("data/processed") OUT.mkdir(parents=True, exist_ok=True) @@ -51,8 +52,8 @@ def _hf_download(filename: str) -> str: def base_frame( - text, - label, + text: Sequence[object] | pd.Series, + label: Sequence[object] | pd.Series, lang: str, source: str, category: str = "general", diff --git a/logger/__init__.py b/logger/__init__.py deleted file mode 100644 index 6b9d48a..0000000 --- a/logger/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .custom_logger import get_logger \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 157c381..107f60b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ ignore = [ [tool.mypy] python_version = "3.13" strict = true +ignore_missing_imports = true exclude = [ 'src(?:[/\\])aivalidatorservice(?:[/\\])grpc', 'trainer(?:[/\\])google_colab_train\.py', @@ -93,7 +94,7 @@ warn_redundant_casts = true warn_unused_ignores = true show_error_codes = true explicit_package_bases = true -mypy_path = ["src"] +mypy_path = ["src", "."] plugins = ["pydantic.mypy"] [tool.pytest.ini_options] diff --git a/src/aivalidatorservice/constants.py b/src/aivalidatorservice/constants.py index 43cc297..e418582 100644 --- a/src/aivalidatorservice/constants.py +++ b/src/aivalidatorservice/constants.py @@ -1,6 +1,6 @@ import pandas as pd -from custom_logger import get_logger +from aivalidatorservice.logger.custom_logger import get_logger log = get_logger(__name__) diff --git a/src/aivalidatorservice/handler/handler.py b/src/aivalidatorservice/handler/handler.py index 4a368e2..a70cc8a 100644 --- a/src/aivalidatorservice/handler/handler.py +++ b/src/aivalidatorservice/handler/handler.py @@ -1,9 +1,11 @@ +from typing import Any + import grpc from aivalidatorservice.grpc import moderation_pb2, moderation_pb2_grpc from aivalidatorservice.handler.moderate import moderate as _moderate +from aivalidatorservice.logger.custom_logger import get_logger from aivalidatorservice.service.service import ModerationService -from logger.custom_logger import get_logger log = get_logger(__name__) @@ -15,7 +17,7 @@ def __init__(self, service: ModerationService): async def ModerateObject( self, request: moderation_pb2.ModerateObjectRequest, - context: grpc.ServicerContext, + context: grpc.aio.ServicerContext[Any, Any], ) -> moderation_pb2.ModerateObjectResponse: log.debug( f"Received ModerateObject request id={request.id} type={request.type}" diff --git a/src/aivalidatorservice/handler/moderate.py b/src/aivalidatorservice/handler/moderate.py index 1a3b30c..1eaa829 100644 --- a/src/aivalidatorservice/handler/moderate.py +++ b/src/aivalidatorservice/handler/moderate.py @@ -1,8 +1,8 @@ import grpc from aivalidatorservice.grpc import moderation_pb2 +from aivalidatorservice.logger.custom_logger import get_logger from aivalidatorservice.service.service import ModerationService -from logger.custom_logger import get_logger log = get_logger(__name__) diff --git a/logger/custom_logger.py b/src/aivalidatorservice/logger/custom_logger.py similarity index 60% rename from logger/custom_logger.py rename to src/aivalidatorservice/logger/custom_logger.py index 0ad69ba..f4e48b0 100644 --- a/logger/custom_logger.py +++ b/src/aivalidatorservice/logger/custom_logger.py @@ -1,14 +1,20 @@ import datetime import logging +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from aivalidatorservice.settings import Settings + +_configured_level = logging.INFO class AbseilStyleFormatter(logging.Formatter): - def format(self, record): + def format(self, record: logging.LogRecord) -> str: level = record.levelname location = f"{record.filename}:{record.lineno}" timespan = ( datetime.datetime.now(datetime.UTC).strftime( - "%Y-%m-%dT%H:%M:%S.%f" + "%Y-%m-%d %H:%M:%S.%f" )[:-3] + "Z" ) @@ -20,16 +26,24 @@ def format(self, record): msg = f"[{level}] {location} timespan={timespan} subsystem={subsystem}" if code: msg += f" code={code}" - msg += f' message="{record.getMessage()}"' + msg += f" message={record.getMessage()}" if explanation: - msg += f' explanation="{explanation}"' + msg += f" explanation={explanation}" return msg +def configure_logger(settings: "Settings") -> None: + global _configured_level + _configured_level = getattr( + logging, settings.log_level.upper(), logging.INFO + ) + + def get_logger(name: str) -> logging.Logger: logger = logging.getLogger(name) - logger.setLevel(logging.INFO) + logger.setLevel(_configured_level) + logger.propagate = False if not logger.handlers: handler = logging.StreamHandler() diff --git a/src/aivalidatorservice/model/loader.py b/src/aivalidatorservice/model/loader.py index ce73c9c..f8215f4 100644 --- a/src/aivalidatorservice/model/loader.py +++ b/src/aivalidatorservice/model/loader.py @@ -2,11 +2,16 @@ from optimum.onnxruntime import ORTModelForSequenceClassification from transformers import AutoTokenizer -from logger.custom_logger import get_logger +from aivalidatorservice.logger.custom_logger import get_logger TOXICITY_THRESHOLD = 0.75 log = get_logger(__name__) +__all__ = [ + "AutoTokenizer", + "ModerationModel", + "ORTModelForSequenceClassification", +] class ModerationModel: @@ -14,7 +19,7 @@ def __init__(self, model_path: str, tokenizer_path: str): log.info( f"Initializing ModerationModel model_path={model_path} tokenizer_path={tokenizer_path}" ) - self._tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) + self._tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) # type: ignore[no-untyped-call] self._model = ORTModelForSequenceClassification.from_pretrained( model_path, file_name="model_quantized.onnx", diff --git a/src/aivalidatorservice/server.py b/src/aivalidatorservice/server.py index b76471d..93d3a3e 100644 --- a/src/aivalidatorservice/server.py +++ b/src/aivalidatorservice/server.py @@ -6,10 +6,10 @@ from aivalidatorservice.grpc import moderation_pb2, moderation_pb2_grpc from aivalidatorservice.handler.handler import ModerationHandler +from aivalidatorservice.logger.custom_logger import get_logger from aivalidatorservice.model.loader import ModerationModel from aivalidatorservice.service.service import ModerationService from aivalidatorservice.settings import Settings -from custom_logger import get_logger async def serve() -> None: diff --git a/src/aivalidatorservice/service/moderate.py b/src/aivalidatorservice/service/moderate.py index bdbc3d4..2034986 100644 --- a/src/aivalidatorservice/service/moderate.py +++ b/src/aivalidatorservice/service/moderate.py @@ -1,5 +1,5 @@ +from aivalidatorservice.logger.custom_logger import get_logger from aivalidatorservice.model.loader import ModerationModel -from custom_logger import get_logger log = get_logger(__name__) diff --git a/src/aivalidatorservice/service/service.py b/src/aivalidatorservice/service/service.py index 3575a03..7a4be39 100644 --- a/src/aivalidatorservice/service/service.py +++ b/src/aivalidatorservice/service/service.py @@ -1,8 +1,8 @@ import asyncio +from aivalidatorservice.logger.custom_logger import get_logger from aivalidatorservice.model.loader import ModerationModel from aivalidatorservice.service.moderate import moderate as _moderate -from custom_logger import get_logger log = get_logger(__name__) diff --git a/src/aivalidatorservice/settings.py b/src/aivalidatorservice/settings.py index 7d217dc..84921f3 100644 --- a/src/aivalidatorservice/settings.py +++ b/src/aivalidatorservice/settings.py @@ -1,8 +1,8 @@ -import logging - from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict +from aivalidatorservice.logger.custom_logger import configure_logger + class Settings(BaseSettings): model_config = SettingsConfigDict( @@ -18,8 +18,4 @@ class Settings(BaseSettings): log_datefmt: str = Field(validation_alias="LOG_DATEFMT") def configure_logging(self) -> None: - logging.basicConfig( - level=self.log_level, - format=self.log_format, - datefmt=self.log_datefmt, - ) + configure_logger(self) diff --git a/tests/behavioral/test_golden_path.py b/tests/behavioral/test_golden_path.py index a119476..c1b6474 100644 --- a/tests/behavioral/test_golden_path.py +++ b/tests/behavioral/test_golden_path.py @@ -1,14 +1,20 @@ import pytest +from aivalidatorservice.service.service import ModerationService + @pytest.mark.asyncio -async def test_high_logit_returns_true(toxic_service): +async def test_high_logit_returns_true( + toxic_service: ModerationService, +) -> None: result = await toxic_service.moderate("arbitrary text") assert result is True @pytest.mark.asyncio -async def test_low_logit_returns_false(clean_service): +async def test_low_logit_returns_false( + clean_service: ModerationService, +) -> None: result = await clean_service.moderate("arbitrary text") assert result is False @@ -22,6 +28,8 @@ async def test_low_logit_returns_false(clean_service): "word " * 300, ], ) -async def test_crash_safety_inputs_return_boolean(clean_service, text): +async def test_crash_safety_inputs_return_boolean( + clean_service: ModerationService, text: str +) -> None: result = await clean_service.moderate(text) assert isinstance(result, bool) diff --git a/tests/conftest.py b/tests/conftest.py index a4d8baf..b7037f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,8 @@ import pytest import torch +from _pytest.monkeypatch import MonkeyPatch +from transformers import PreTrainedTokenizerBase from aivalidatorservice.model import loader from aivalidatorservice.model.loader import ModerationModel @@ -26,12 +28,13 @@ def make_fake_tokenizer(max_length: int = 128) -> MagicMock: @pytest.fixture(scope="session") -def real_tokenizer(): - return loader.AutoTokenizer.from_pretrained("microsoft/deberta-v3-small") +def real_tokenizer() -> PreTrainedTokenizerBase: + model_name = "microsoft/deberta-v3-small" + return loader.AutoTokenizer.from_pretrained(model_name) # type: ignore[no-untyped-call, no-any-return] @pytest.fixture -def toxic_service(monkeypatch): +def toxic_service(monkeypatch: MonkeyPatch) -> ModerationService: monkeypatch.setattr( loader.AutoTokenizer, "from_pretrained", @@ -49,7 +52,7 @@ def toxic_service(monkeypatch): @pytest.fixture -def clean_service(monkeypatch): +def clean_service(monkeypatch: MonkeyPatch) -> ModerationService: monkeypatch.setattr( loader.AutoTokenizer, "from_pretrained", diff --git a/tests/contract/test_grpc_contract.py b/tests/contract/test_grpc_contract.py index 6fb14e0..7fbcf9f 100644 --- a/tests/contract/test_grpc_contract.py +++ b/tests/contract/test_grpc_contract.py @@ -1,19 +1,25 @@ +from typing import cast + import pytest +from grpc.aio import ServicerContext from aivalidatorservice.grpc import moderation_pb2 from aivalidatorservice.handler.handler import ModerationHandler +from aivalidatorservice.service.service import ModerationService @pytest.mark.asyncio async def test_grpc_contract_toxic_input_returns_expected_response( - toxic_service, -): + toxic_service: ModerationService, +) -> None: request = moderation_pb2.ModerateObjectRequest( id=123, type=moderation_pb2.OBJECT_TYPE_COMMENT_TEXT, text="Ihateyou", ) handler = ModerationHandler(toxic_service) - response = await handler.ModerateObject(request, context=None) + response = await handler.ModerateObject( + request, context=cast(ServicerContext, None) + ) assert isinstance(response, moderation_pb2.ModerateObjectResponse) assert response.success is True diff --git a/tests/tokenizer/test_tokenizer_shapes.py b/tests/tokenizer/test_tokenizer_shapes.py index a6dab04..d80d403 100644 --- a/tests/tokenizer/test_tokenizer_shapes.py +++ b/tests/tokenizer/test_tokenizer_shapes.py @@ -1,4 +1,9 @@ -def test_tokenizer_truncates_to_max_length_128(real_tokenizer): +from transformers import PreTrainedTokenizerBase + + +def test_tokenizer_truncates_to_max_length_128( + real_tokenizer: PreTrainedTokenizerBase, +) -> None: out = real_tokenizer( "token " * 400, return_tensors="pt", max_length=128, truncation=True ) diff --git a/tools/common.just b/tools/common.just index 26776fe..8d30ea5 100644 --- a/tools/common.just +++ b/tools/common.just @@ -39,7 +39,9 @@ lint: pdm run mypy --strict . test: - pdm run pytest tests/ + pdm run pytest \ + --cov=src/{{SOURCE}} \ + --cov-report=xml:coverage.xml prepare: format lint test diff --git a/trainer/onnx_exporter.py b/trainer/onnx_exporter.py index 728ffe9..b6369bd 100644 --- a/trainer/onnx_exporter.py +++ b/trainer/onnx_exporter.py @@ -5,7 +5,7 @@ from optimum.onnxruntime import ORTModelForSequenceClassification from transformers import AutoModelForSequenceClassification, AutoTokenizer -from logger.custom_logger import get_logger +from aivalidatorservice.logger.custom_logger import get_logger DATA_DIR = Path("data/processed") OUT_DIR = Path("models/deberta-qat") @@ -20,7 +20,7 @@ log = get_logger(__name__) log.info(f"Loading {MODEL_ID}...") -tokenizer = AutoTokenizer.from_pretrained(BASE_ID) +tokenizer = AutoTokenizer.from_pretrained(BASE_ID) # type: ignore[no-untyped-call] model = AutoModelForSequenceClassification.from_pretrained( MODEL_ID, num_labels=2 ) diff --git a/trainer/quantizer.py b/trainer/quantizer.py index 33d4158..9e6b6a3 100644 --- a/trainer/quantizer.py +++ b/trainer/quantizer.py @@ -3,7 +3,7 @@ from optimum.onnxruntime import ORTQuantizer from optimum.onnxruntime.configuration import AutoQuantizationConfig -from logger.custom_logger import get_logger +from aivalidatorservice.logger.custom_logger import get_logger ONNX_DIR = Path("models/deberta-qat-onnx-v2") INT8_DIR = Path("models/deberta-qat-int8-v2") diff --git a/trainer/train_qat.py b/trainer/train_qat.py index 6649251..30f560d 100644 --- a/trainer/train_qat.py +++ b/trainer/train_qat.py @@ -1,5 +1,6 @@ import shutil from pathlib import Path +from typing import Any import numpy as np import pandas as pd @@ -17,10 +18,12 @@ EarlyStoppingCallback, Trainer, TrainerCallback, + TrainerControl, + TrainerState, TrainingArguments, ) -from logger.custom_logger import get_logger +from aivalidatorservice.logger.custom_logger import get_logger log = get_logger(__name__) @@ -49,16 +52,30 @@ class SaveBestCallback(TrainerCallback): - def __init__(self, save_path, tokenizer, f1_weight=0.5, auc_weight=0.5): + def __init__( + self, + save_path: Path, + tokenizer: Any, + f1_weight: float = 0.5, + auc_weight: float = 0.5, + ) -> None: self.save_path = save_path self.tokenizer = tokenizer self.f1_weight = f1_weight self.auc_weight = auc_weight self.best_score = 0.0 - def on_evaluate(self, _args, _state, _control, metrics=None, **kwargs): - current_f1 = metrics.get("eval_f1", 0.0) - current_auc = metrics.get("eval_roc_auc", 0.0) + def on_evaluate( + self, + _args: TrainingArguments, + _state: TrainerState, + _control: TrainerControl, + metrics: dict[str, float] | None = None, + **kwargs: Any, + ) -> None: + eval_metrics = metrics or {} + current_f1 = eval_metrics.get("eval_f1", 0.0) + current_auc = eval_metrics.get("eval_roc_auc", 0.0) combined = (self.f1_weight * current_f1) + ( self.auc_weight * current_auc ) @@ -74,8 +91,14 @@ def on_evaluate(self, _args, _state, _control, metrics=None, **kwargs): log.info(f"New best {combined:.4f} saved -> {self.save_path}") -class ToxicityDataset(Dataset): - def __init__(self, df, tokenizer, max_len, name="dataset"): +class ToxicityDataset(Dataset[dict[str, torch.Tensor]]): + def __init__( + self, + df: pd.DataFrame, + tokenizer: Any, + max_len: int, + name: str = "dataset", + ) -> None: log.info(f"Tokenizing {name} ({len(df):,} samples)...") enc = tokenizer( df["text"].tolist(), @@ -89,10 +112,10 @@ def __init__(self, df, tokenizer, max_len, name="dataset"): self.labels = torch.tensor(df["label"].values, dtype=torch.long) log.info(f"Done tokenizing {name}!") - def __len__(self): + def __len__(self) -> int: return len(self.labels) - def __getitem__(self, idx): + def __getitem__(self, idx: int) -> dict[str, torch.Tensor]: return { "input_ids": self.input_ids[idx], "attention_mask": self.attention_mask[idx], @@ -100,7 +123,11 @@ def __getitem__(self, idx): } -def compute_metrics(eval_pred): +def compute_metrics( + eval_pred: tuple[ + np.ndarray[Any, np.dtype[Any]], np.ndarray[Any, np.dtype[Any]] + ], +) -> dict[str, float]: logits, labels = eval_pred preds = np.argmax(logits, axis=-1) probs = torch.softmax(torch.tensor(logits, dtype=torch.float32), dim=-1)[ @@ -115,11 +142,20 @@ def compute_metrics(eval_pred): class WeightedTrainer(Trainer): - def __init__(self, class_weights, *args, **kwargs): + def __init__( + self, class_weights: torch.Tensor, *args: Any, **kwargs: Any + ) -> None: super().__init__(*args, **kwargs) self.class_weights = class_weights - def compute_loss(self, model, inputs, return_outputs=False, **_kwargs): + def compute_loss( + self, + model: torch.nn.Module, + inputs: dict[str, torch.Tensor | Any], + return_outputs: bool = False, + num_items_in_batch: torch.Tensor | None = None, + ) -> Any: + del num_items_in_batch labels = inputs.pop("labels") outputs = model(**inputs) loss = torch.nn.functional.cross_entropy( @@ -135,7 +171,9 @@ def compute_loss(self, model, inputs, return_outputs=False, **_kwargs): val_df = pd.read_parquet(DATA_DIR / "val.parquet") -def stratified_sample(df, n, label_col="label", seed=42): +def stratified_sample( + df: pd.DataFrame, n: int, label_col: str = "label", seed: int = 42 +) -> pd.DataFrame: return ( df.groupby(label_col, group_keys=False) .apply( @@ -150,8 +188,12 @@ def stratified_sample(df, n, label_col="label", seed=42): def language_balanced_sample( - df, en_cap=300_000, ru_cap=300_000, other_per_lang=8_000, seed=42 -): + df: pd.DataFrame, + en_cap: int = 300_000, + ru_cap: int = 300_000, + other_per_lang: int = 8_000, + seed: int = 42, +) -> pd.DataFrame: parts = [] for lang, group in df.groupby("lang"): cap = {"en": en_cap, "ru": ru_cap, "mixed": 20_000}.get( @@ -197,7 +239,7 @@ def language_balanced_sample( ) log.info(f"Loading {MODEL_ID}...") -tokenizer = AutoTokenizer.from_pretrained(BASE_ID) +tokenizer = AutoTokenizer.from_pretrained(BASE_ID) # type: ignore[no-untyped-call] model = AutoModelForSequenceClassification.from_pretrained( MODEL_ID, num_labels=2 ) From 75c6bfc018516c5edf0dfd4c4eb20dc0ccc3db09 Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 5 May 2026 21:38:38 +0200 Subject: [PATCH 5/8] added faker text generation to the golden path testing --- src/aivalidatorservice/handler/moderate.py | 4 +++- tests/behavioral/test_golden_path.py | 15 ++++++++++----- tests/contract/test_grpc_contract.py | 4 ++-- trainer/train_qat.py | 2 +- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/aivalidatorservice/handler/moderate.py b/src/aivalidatorservice/handler/moderate.py index 1eaa829..ac91e77 100644 --- a/src/aivalidatorservice/handler/moderate.py +++ b/src/aivalidatorservice/handler/moderate.py @@ -1,3 +1,5 @@ +from typing import Any + import grpc from aivalidatorservice.grpc import moderation_pb2 @@ -10,7 +12,7 @@ async def moderate( service: ModerationService, request: moderation_pb2.ModerateObjectRequest, - _context: grpc.ServicerContext, + _context: grpc.aio.ServicerContext[Any, Any], ) -> moderation_pb2.ModerateObjectResponse: try: is_toxic = await service.moderate(request.text) diff --git a/tests/behavioral/test_golden_path.py b/tests/behavioral/test_golden_path.py index c1b6474..fa58ee1 100644 --- a/tests/behavioral/test_golden_path.py +++ b/tests/behavioral/test_golden_path.py @@ -1,13 +1,15 @@ import pytest +from faker import Faker from aivalidatorservice.service.service import ModerationService +fake = Faker() @pytest.mark.asyncio async def test_high_logit_returns_true( toxic_service: ModerationService, ) -> None: - result = await toxic_service.moderate("arbitrary text") + result = await toxic_service.moderate(fake.text()) assert result is True @@ -15,7 +17,7 @@ async def test_high_logit_returns_true( async def test_low_logit_returns_false( clean_service: ModerationService, ) -> None: - result = await clean_service.moderate("arbitrary text") + result = await clean_service.moderate(fake.text()) assert result is False @@ -23,9 +25,12 @@ async def test_low_logit_returns_false( @pytest.mark.parametrize( "text", [ - "", - "Привет こんにちは مرحبا", - "word " * 300, + "", + fake.sentence(), + fake.text(), + fake.sentence(locale="fr_FR"), + fake.sentence(locale="ru_RU"), + fake.sentence(locale="ja_JP"), ], ) async def test_crash_safety_inputs_return_boolean( diff --git a/tests/contract/test_grpc_contract.py b/tests/contract/test_grpc_contract.py index 7fbcf9f..7c9bb39 100644 --- a/tests/contract/test_grpc_contract.py +++ b/tests/contract/test_grpc_contract.py @@ -1,4 +1,4 @@ -from typing import cast +from typing import Any, cast import pytest from grpc.aio import ServicerContext @@ -19,7 +19,7 @@ async def test_grpc_contract_toxic_input_returns_expected_response( ) handler = ModerationHandler(toxic_service) response = await handler.ModerateObject( - request, context=cast(ServicerContext, None) + request, context=cast(ServicerContext[Any, Any], None) ) assert isinstance(response, moderation_pb2.ModerateObjectResponse) assert response.success is True diff --git a/trainer/train_qat.py b/trainer/train_qat.py index 30f560d..b5ae0e7 100644 --- a/trainer/train_qat.py +++ b/trainer/train_qat.py @@ -72,7 +72,7 @@ def on_evaluate( _control: TrainerControl, metrics: dict[str, float] | None = None, **kwargs: Any, - ) -> None: + ) -> None: # type: ignore[override] eval_metrics = metrics or {} current_f1 = eval_metrics.get("eval_f1", 0.0) current_auc = eval_metrics.get("eval_roc_auc", 0.0) From 1953b192ede1544ab55432b57be9f57450c2858f Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 5 May 2026 21:50:57 +0200 Subject: [PATCH 6/8] bug fix --- tests/behavioral/test_golden_path.py | 12 ++++++++---- trainer/train_qat.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/behavioral/test_golden_path.py b/tests/behavioral/test_golden_path.py index fa58ee1..306ef70 100644 --- a/tests/behavioral/test_golden_path.py +++ b/tests/behavioral/test_golden_path.py @@ -4,6 +4,10 @@ from aivalidatorservice.service.service import ModerationService fake = Faker() +fake_fr = Faker("fr_FR") +fake_ru = Faker("ru_RU") +fake_ja = Faker("ja_JP") + @pytest.mark.asyncio async def test_high_logit_returns_true( @@ -25,12 +29,12 @@ async def test_low_logit_returns_false( @pytest.mark.parametrize( "text", [ - "", + "", fake.sentence(), fake.text(), - fake.sentence(locale="fr_FR"), - fake.sentence(locale="ru_RU"), - fake.sentence(locale="ja_JP"), + fake_fr.sentence(), + fake_ru.sentence(), + fake_ja.sentence(), ], ) async def test_crash_safety_inputs_return_boolean( diff --git a/trainer/train_qat.py b/trainer/train_qat.py index b5ae0e7..3336d60 100644 --- a/trainer/train_qat.py +++ b/trainer/train_qat.py @@ -72,7 +72,7 @@ def on_evaluate( _control: TrainerControl, metrics: dict[str, float] | None = None, **kwargs: Any, - ) -> None: # type: ignore[override] + ) -> None: # type: ignore[override] eval_metrics = metrics or {} current_f1 = eval_metrics.get("eval_f1", 0.0) current_auc = eval_metrics.get("eval_roc_auc", 0.0) From f9910152d73484807ce25564ad72b6fc0008972c Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 5 May 2026 21:55:45 +0200 Subject: [PATCH 7/8] bug fix --- trainer/train_qat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trainer/train_qat.py b/trainer/train_qat.py index 3336d60..8c9fc30 100644 --- a/trainer/train_qat.py +++ b/trainer/train_qat.py @@ -65,14 +65,14 @@ def __init__( self.auc_weight = auc_weight self.best_score = 0.0 - def on_evaluate( + def on_evaluate( # type: ignore[override] self, _args: TrainingArguments, _state: TrainerState, _control: TrainerControl, metrics: dict[str, float] | None = None, **kwargs: Any, - ) -> None: # type: ignore[override] + ) -> None: eval_metrics = metrics or {} current_f1 = eval_metrics.get("eval_f1", 0.0) current_auc = eval_metrics.get("eval_roc_auc", 0.0) From b52b45c19088bb4b4427c07b1deac07c76b44c18 Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 5 May 2026 22:04:48 +0200 Subject: [PATCH 8/8] formatted --- trainer/train_qat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trainer/train_qat.py b/trainer/train_qat.py index 8c9fc30..4c73524 100644 --- a/trainer/train_qat.py +++ b/trainer/train_qat.py @@ -65,14 +65,14 @@ def __init__( self.auc_weight = auc_weight self.best_score = 0.0 - def on_evaluate( # type: ignore[override] + def on_evaluate( # type: ignore[override] self, _args: TrainingArguments, _state: TrainerState, _control: TrainerControl, metrics: dict[str, float] | None = None, **kwargs: Any, - ) -> None: + ) -> None: eval_metrics = metrics or {} current_f1 = eval_metrics.get("eval_f1", 0.0) current_auc = eval_metrics.get("eval_roc_auc", 0.0)