EvaluationManager.py

import json
import os
from LabelProcessor import LabelProcessor

class EvaluationManager:
    def __init__(self):
        self.label_processor = LabelProcessor()
        self.gt_cache = {}

    def get_gt(self, file_path):
        """Extract a simple ground-truth label from a file path.

        The function strips directories and extensions, then removes
        non-alphabetic characters to produce a compact class name.

        Args:
            file_path (str): Path to the file whose ground-truth label is embedded in its name.

        Returns:
            str: Ground-truth label extracted from the filename.
        """
        gt = os.path.basename(file_path).split('.')[0]
        gt = ''.join([c for c in gt if c.isalpha()])
        return gt

    def get_gt_embedding(self, gt):
        """Return (and cache) the embedding for a ground-truth label.

        Args:
            gt (str): Ground-truth label string.

        Returns:
            numpy.ndarray: Embedding vector for the label.
        """
        if gt not in self.gt_cache:
            self.gt_cache[gt] = self.label_processor.compute_embedding(gt)
        return self.gt_cache[gt]

    def evaluate(self, predicted_embedding, ground_truth_embedding, similarity_threshold=0.8):
        """Compare two embeddings and decide if they are similar enough.

        Args:
            predicted_embedding (array-like): Embedding produced by the classifier.
            ground_truth_embedding (array-like): Embedding representing the ground truth label.
            similarity_threshold (float): Threshold above which the prediction is considered correct.

        Returns:
            bool: True if similarity >= threshold, else False.
        """
        similarity = self.label_processor.compute_similarity(predicted_embedding, ground_truth_embedding)
        return similarity >= similarity_threshold

    def accuracy(self, predictions_file, similarity_threshold=0.8):
        """Compute overall accuracy from a saved predictions JSON file.

        The predictions file is expected to contain objects with keys
        `file_path` and `embedding`.

        Args:
            predictions_file (str): Path to JSON file with classification results.
            similarity_threshold (float): Threshold to consider a prediction correct.

        Returns:
            float: Fraction of correct predictions (0.0-1.0).
        """
        with open(predictions_file, 'r') as f:
            data = json.load(f)

        correct = 0
        total = len(data)

        for item in data:
            file_path = item['file_path']
            predicted_embedding = item['embedding']
            gt = self.get_gt(file_path)
            ground_truth_embedding = self.get_gt_embedding(gt)

            if self.evaluate(predicted_embedding, ground_truth_embedding, similarity_threshold):
                correct += 1

        accuracy = correct / total if total > 0 else 0
        print(f"Accuracy: {accuracy:.2f}")
        return accuracy

    def class_accuracy(self, predictions_file, similarity_threshold=0.8):
        """Compute per-class accuracies and write them to a JSON file.

        Args:
            predictions_file (str): Path to JSON file with classification results.
            similarity_threshold (float): Threshold to consider a prediction correct.

        Returns:
            None
        """
        with open(predictions_file, 'r') as f:
            data = json.load(f)

        class_correct = {}
        class_total = {}

        for item in data:
            file_path = item['file_path']
            predicted_embedding = item['embedding']
            gt = self.get_gt(file_path)
            ground_truth_embedding = self.get_gt_embedding(gt)

            if gt not in class_total:
                class_total[gt] = 0
                class_correct[gt] = 0

            class_total[gt] += 1

            if self.evaluate(predicted_embedding, ground_truth_embedding, similarity_threshold):
                class_correct[gt] += 1

        class_accuracies = {}

        for gt in class_total:
            accuracy = class_correct[gt] / class_total[gt] if class_total[gt] > 0 else 0
            class_accuracies[gt] = accuracy

        with open(f"{predictions_file.split('.')[0]}_{similarity_threshold}_class_accuracies.json", 'w') as f:
            json.dump(class_accuracies, f, indent=4)