Added unit tests and workflow for linting, import checks and running unit tests

.github/workflows/codeql.yml

@@ -0,0 +1,34 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "25 14 * * 1"
+
+jobs:
+  analyze:
+    name: Analyze (Python)
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      packages: read
+      actions: read
+      contents: read
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: python
+          queries: security-and-quality
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:python"

.github/workflows/pr-checks.yml

@@ -0,0 +1,42 @@
+name: PR Checks
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+jobs:
+  checks:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install flake8 isort
+
+      - name: Lint with flake8
+        run: |
+          flake8 src/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics
+          flake8 src/ tests/ --count --max-line-length=120 --statistics --exit-zero
+
+      - name: Check import ordering with isort
+        run: |
+          isort --check-only --diff src/ tests/
+
+      - name: Run unit tests
+        run: |
+          python -m pytest tests/ -v --tb=short

pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py
+python_classes = Test*
+python_functions = test_*
+addopts = -v --tb=short

requirements.txt

@@ -1,16 +1,19 @@
 # Azure SDK dependencies
 azure-identity>=1.25.3
 azure-ai-projects>=2.2.0
-azure-ai-evaluation==1.16.9
+azure-ai-evaluation==1.17.0
 azure-ai-inference>=1.0.0b9
 # Core Python packages
 python-dotenv>=1.2.2
 pyyaml>=6.0.3
 pip-system-certs>=5.3
 azure-monitor-query>=2.0.0
 azure-monitor-opentelemetry>=1.8.8
-aiohttp>=3.13.5
-agent-framework==1.7.0
+aiohttp>=3.14.1
+agent-framework>=1.9.0
 streamlit>=1.58.0
 pandas==2.3.3
-plotly>=6.7.0
+plotly>=6.8.0
+# Test dependencies
+pytest>=9.0.0
+pytest-asyncio>=1.3.0

src/agent_evaluation/agentic_ops/base_evaluator.py

@@ -1,9 +1,9 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+import logging
 import os
 import re
-import logging
 from typing import Dict, Union
 
 from .client import LLMClient

src/agent_evaluation/agentic_ops/client.py

@@ -8,12 +8,13 @@
 
 import json
 import logging
-from openai import AzureOpenAI
-from azure.identity import DefaultAzureCredential, get_bearer_token_provider
 import os
-from dotenv import load_dotenv
 import time
 
+from azure.identity import DefaultAzureCredential, get_bearer_token_provider
+from dotenv import load_dotenv
+from openai import AzureOpenAI
+
 load_dotenv()
 
 # Configure logging

src/agent_evaluation/agentic_ops/run_eval.py

@@ -1,11 +1,13 @@
-import os
 import inspect
+import logging
+import os
 import uuid
-from dotenv import load_dotenv
+
 from azure.ai.evaluation import evaluate
 from azure.ai.projects import AIProjectClient
-import logging
 from azure.identity import DefaultAzureCredential
+from dotenv import load_dotenv
+
 
 def get_logger(name: str):
     level = os.environ.get("LOG_LEVEL", "INFO").upper()

src/agent_evaluation/agentic_ops/runner.py

@@ -1,12 +1,14 @@
 import argparse
 import importlib
+import logging
+import os
 import sys
 import time
 from pathlib import Path
-from typing import Optional, Any, Dict
+from typing import Any, Dict, Optional
+
 import yaml
-import os
-import logging
+
 
 def get_logger(name: str):
     level = os.environ.get("LOG_LEVEL", "INFO").upper()

src/agent_evaluation/cli.py

@@ -9,11 +9,10 @@
 import argparse
 import sys
 from pathlib import Path
-from typing import List, Dict, Optional
+from typing import Dict, List, Optional
 
 import yaml
 
-
 # Root of the project (two levels up from this file)
 ROOT_DIR = Path(__file__).resolve().parents[2]
 SAMPLES_DIR = ROOT_DIR / "src" / "evaluations" / "offline"
@@ -74,7 +73,8 @@ def print_samples_table(samples: List[Dict[str, str]]) -> None:
 
 def run_sample(sample: Dict[str, str], extra_args: Optional[List[str]] = None) -> int:
     """Run a selected evaluation sample."""
-    from src.agent_evaluation.agentic_ops.runner import run_pipeline, parse_args
+    from src.agent_evaluation.agentic_ops.runner import (parse_args,
+                                                         run_pipeline)
 
     config_path = sample["config_path"]
     print(f"\n{'='*70}")

src/evaluations/offline/agentic_evaluation/eval_factory.py

@@ -1,8 +1,12 @@
-from azure.ai.evaluation import RelevanceEvaluator,  TaskAdherenceEvaluator, ToolCallAccuracyEvaluator
-from .evaluator.evaluator_repo.evaluate_agent_invoked import EvaluateAgentsInvoked
-
-import os
 import logging
+import os
+
+from azure.ai.evaluation import (RelevanceEvaluator, TaskAdherenceEvaluator,
+                                 ToolCallAccuracyEvaluator)
+
+from .evaluator.evaluator_repo.evaluate_agent_invoked import \
+    EvaluateAgentsInvoked
+
 
 def get_logger(name: str):
     level = os.environ.get("LOG_LEVEL", "INFO").upper()

src/evaluations/offline/agentic_evaluation/evaluator/eval_main.py

@@ -1,9 +1,11 @@
+import logging
 import os
 from pathlib import Path
+
 from src.agent_evaluation.agentic_ops.run_eval import execute_eval
-import logging
 from src.evaluations.offline.utils.constants import EVAL_NAME
 from src.evaluations.offline.utils.file_operations import get_next_run_id
+
 from ..eval_factory import EvaluatorFactory
 
 

src/evaluations/offline/agentic_evaluation/evaluator/evaluator_repo/evaluate_agent_invoked.py

@@ -1,5 +1,6 @@
 from .eval_utils.evaluation_utils import agent_invoked_accuracy, compute_recall
 
+
 class EvaluateAgentsInvoked:
     def __init__(self):
         pass

src/evaluations/offline/ai_judge_evaluation_custom/eval_factory.py

@@ -1,11 +1,13 @@
-from azure.ai.evaluation import RelevanceEvaluator, CoherenceEvaluator
+import logging
+import os
+
+from azure.ai.evaluation import CoherenceEvaluator, RelevanceEvaluator
+
 from .evaluator.evaluator_repo.coherence import CoherenceEvaluatorCustom
-from .evaluator.evaluator_repo.relevance import RelevanceEvaluatorCustom
 from .evaluator.evaluator_repo.fluency import FluencyEvaluatorCustom
+from .evaluator.evaluator_repo.relevance import RelevanceEvaluatorCustom
 from .evaluator.evaluator_repo.similarity import SimilarityEvaluatorCustom
 
-import os
-import logging
 
 def get_logger(name: str):
     level = os.environ.get("LOG_LEVEL", "INFO").upper()

src/evaluations/offline/ai_judge_evaluation_custom/evaluator/eval_main.py

@@ -1,9 +1,11 @@
+import logging
 import os
 from pathlib import Path
+
 from src.agent_evaluation.agentic_ops.run_eval import execute_eval
-import logging
 from src.evaluations.offline.utils.constants import EVAL_NAME
 from src.evaluations.offline.utils.file_operations import get_next_run_id
+
 from ..eval_factory import EvaluatorFactory
 
 

src/evaluations/offline/ai_judge_evaluation_custom/evaluator/evaluator_repo/coherence.py

@@ -1,5 +1,7 @@
 from typing import Dict, Union
-from ......agent_evaluation.agentic_ops.base_evaluator import BaseCustomEvaluator
+
+from ......agent_evaluation.agentic_ops.base_evaluator import \
+    BaseCustomEvaluator
 
 
 class CoherenceEvaluatorCustom(BaseCustomEvaluator):

src/evaluations/offline/ai_judge_evaluation_custom/evaluator/evaluator_repo/fluency.py

@@ -2,7 +2,9 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 from typing import Dict, Union
-from ......agent_evaluation.agentic_ops.base_evaluator import BaseCustomEvaluator
+
+from ......agent_evaluation.agentic_ops.base_evaluator import \
+    BaseCustomEvaluator
 
 
 class FluencyEvaluatorCustom(BaseCustomEvaluator):

src/evaluations/offline/ai_judge_evaluation_custom/evaluator/evaluator_repo/relevance.py

@@ -1,5 +1,7 @@
 from typing import Dict, Union
-from ......agent_evaluation.agentic_ops.base_evaluator import BaseCustomEvaluator
+
+from ......agent_evaluation.agentic_ops.base_evaluator import \
+    BaseCustomEvaluator
 
 
 class RelevanceEvaluatorCustom(BaseCustomEvaluator):

src/evaluations/offline/ai_judge_evaluation_custom/evaluator/evaluator_repo/similarity.py

@@ -2,7 +2,9 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 from typing import Dict, Union
-from ......agent_evaluation.agentic_ops.base_evaluator import BaseCustomEvaluator
+
+from ......agent_evaluation.agentic_ops.base_evaluator import \
+    BaseCustomEvaluator
 
 
 class SimilarityEvaluatorCustom(BaseCustomEvaluator):

src/evaluations/offline/pipeline_experiment_evaluation/eval_factory.py

@@ -1,7 +1,9 @@
-from azure.ai.evaluation import RelevanceEvaluator, TaskAdherenceEvaluator, ToolCallAccuracyEvaluator
-
-import os
 import logging
+import os
+
+from azure.ai.evaluation import (RelevanceEvaluator, TaskAdherenceEvaluator,
+                                 ToolCallAccuracyEvaluator)
+
 
 def get_logger(name: str):
     level = os.environ.get("LOG_LEVEL", "INFO").upper()

src/evaluations/offline/pipeline_experiment_evaluation/evaluator/eval_main.py

@@ -1,9 +1,11 @@
+import logging
 import os
 from pathlib import Path
+
 from src.agent_evaluation.agentic_ops.run_eval import execute_eval
-import logging
 from src.evaluations.offline.utils.constants import EVAL_NAME
 from src.evaluations.offline.utils.file_operations import get_next_run_id
+
 from ..eval_factory import EvaluatorFactory
 
 

src/evaluations/offline/pipeline_experiment_evaluation/experiment/agent_inference.py

@@ -11,11 +11,14 @@
 FLOW:
     Load Queries → Run Inference → Save Responses
 """
-import os
 import logging
+import os
 import random
 from pathlib import Path
-from src.evaluations.offline.utils.file_operations import load_queries_from_jsonl
+
+from src.evaluations.offline.utils.file_operations import \
+    load_queries_from_jsonl
+
 from .experiment_utils import get_file_paths, prepare_output_file, save_result
 
 
@@ -101,7 +104,7 @@ def inference_main(config: dict, args=None) -> None:
 if __name__ == "__main__":
     # For standalone execution, load config from experiment.yaml
     import yaml
-    
+
     # Get project root (go up 5 levels from this file)
     current_file = Path(__file__)  # .../experiment/agent_inference.py
     project_root = current_file.parent.parent.parent.parent.parent.parent  # Go up to project root

src/evaluations/offline/pipeline_experiment_evaluation/experiment/experiment_utils/file_utils.py

@@ -3,11 +3,11 @@
 ==============
 Helper functions for file management in inference pipelines.
 """
-import os
 import logging
+import os
 from pathlib import Path
-from src.evaluations.offline.utils.file_operations import append_to_jsonl
 
+from src.evaluations.offline.utils.file_operations import append_to_jsonl
 
 logger = logging.getLogger(__name__)
 

src/evaluations/offline/pipeline_multi_agent_evaluation/agent_inference/agent_tools.py

@@ -1,9 +1,10 @@
 """Tool functions for the device agents in the Multi-Agent system."""
 
+from random import choice, randint
 from typing import Annotated
-from pydantic import Field
-from random import randint, choice
+
 from agent_framework import tool
+from pydantic import Field
 
 
 # =============================================================================