diff --git a/.gitignore b/.gitignore index ff83fbb..5fdeafa 100644 --- a/.gitignore +++ b/.gitignore @@ -240,3 +240,6 @@ tiktoken-docs.md config.yaml config.groq.yaml config.*.yaml + +# Generated eval report artifacts +reports/ diff --git a/openagent_eval/datasets/csv_loader.py b/openagent_eval/datasets/csv_loader.py index 4b1a8e4..b77d275 100644 --- a/openagent_eval/datasets/csv_loader.py +++ b/openagent_eval/datasets/csv_loader.py @@ -65,7 +65,7 @@ def load(self, path: Path) -> Dataset: raise InvalidDatasetError( message=f"Invalid CSV format: {e}", dataset_path=str(path), - format="csv", + data_format="csv", ) from e return self._parse_rows(rows, path) diff --git a/openagent_eval/datasets/factory.py b/openagent_eval/datasets/factory.py index b646409..b34872c 100644 --- a/openagent_eval/datasets/factory.py +++ b/openagent_eval/datasets/factory.py @@ -80,7 +80,7 @@ def _get_loader(config: DatasetConfig) -> BaseDatasetLoader: raise InvalidDatasetError( message=f"Unsupported dataset format: '{config.format}'. " f"Supported formats: {list(_FORMAT_MAP.keys())}", - format=config.format, + data_format=config.format, ) # Guard against directory paths, which have no usable extension and diff --git a/openagent_eval/datasets/hf_loader.py b/openagent_eval/datasets/hf_loader.py index 10fd58c..037560b 100644 --- a/openagent_eval/datasets/hf_loader.py +++ b/openagent_eval/datasets/hf_loader.py @@ -69,7 +69,7 @@ def load(self, path: Path) -> Dataset: "Install it with: pip install openagent-eval[datasets]" ), dataset_path=str(path), - format="hf", + data_format="hf", ) from e try: @@ -78,7 +78,7 @@ def load(self, path: Path) -> Dataset: raise InvalidDatasetError( message=f"Failed to load HuggingFace dataset: {e}", dataset_path=str(path), - format="hf", + data_format="hf", ) from e return self._parse_hf_dataset(hf_dataset, path) @@ -153,7 +153,7 @@ def load_from_hub( "The 'datasets' package is required for HuggingFace dataset loading. " "Install it with: pip install openagent-eval[datasets]" ), - format="hf", + data_format="hf", ) from e try: @@ -161,7 +161,7 @@ def load_from_hub( except Exception as e: raise InvalidDatasetError( message=f"Failed to load dataset '{dataset_name}': {e}", - format="hf", + data_format="hf", ) from e # Convert to list and apply limit diff --git a/openagent_eval/datasets/json_loader.py b/openagent_eval/datasets/json_loader.py index 74f90ae..91de253 100644 --- a/openagent_eval/datasets/json_loader.py +++ b/openagent_eval/datasets/json_loader.py @@ -63,7 +63,7 @@ def load(self, path: Path) -> Dataset: raise InvalidDatasetError( message=f"Invalid JSON in file: {e}", dataset_path=str(path), - format="json", + data_format="json", line_number=e.lineno, ) from e diff --git a/openagent_eval/datasets/jsonl_loader.py b/openagent_eval/datasets/jsonl_loader.py index 05f016a..368ef50 100644 --- a/openagent_eval/datasets/jsonl_loader.py +++ b/openagent_eval/datasets/jsonl_loader.py @@ -120,7 +120,7 @@ def _parse_lines(self, lines: list[str], path: Path) -> Dataset: raise InvalidDatasetError( message=f"Invalid JSON on line {line_num}: {e}", dataset_path=str(path), - format="jsonl", + data_format="jsonl", line_number=line_num, ) from e diff --git a/openagent_eval/datasets/pdf_loader.py b/openagent_eval/datasets/pdf_loader.py index 08cfa5d..124d47c 100644 --- a/openagent_eval/datasets/pdf_loader.py +++ b/openagent_eval/datasets/pdf_loader.py @@ -88,7 +88,7 @@ def load(self, path: Path) -> Dataset: "Install it with: pip install openagent-eval[pdf]" ), dataset_path=str(path), - format="pdf", + data_format="pdf", ) from e try: @@ -97,7 +97,7 @@ def load(self, path: Path) -> Dataset: raise InvalidDatasetError( message=f"Failed to read PDF: {e}", dataset_path=str(path), - format="pdf", + data_format="pdf", ) from e raw_items = self._extract_raw(reader, path) diff --git a/openagent_eval/exceptions/dataset.py b/openagent_eval/exceptions/dataset.py index ecf4ad5..c7ac9d8 100644 --- a/openagent_eval/exceptions/dataset.py +++ b/openagent_eval/exceptions/dataset.py @@ -57,7 +57,7 @@ class InvalidDatasetError(DatasetError): """Raised when a dataset has invalid format or structure. Attributes: - format: The detected or expected format of the dataset. + data_format: The detected or expected format of the dataset. line_number: The line number where the error occurred (if applicable). """ @@ -65,7 +65,7 @@ def __init__( self, message: str, dataset_path: str | None = None, - format: str | None = None, + data_format: str | None = None, line_number: int | None = None, details: dict | None = None, ) -> None: @@ -74,18 +74,18 @@ def __init__( Args: message: Human-readable error message. dataset_path: Path to the invalid dataset. - format: The detected or expected format. + data_format: The detected or expected format. line_number: The line number where the error occurred. details: Additional context about the error. """ error_details = details or {} - if format: - error_details["format"] = format + if data_format: + error_details["format"] = data_format if line_number: error_details["line_number"] = line_number super().__init__(message=message, dataset_path=dataset_path, details=error_details) - self.format = format + self.data_format = data_format self.line_number = line_number diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 387a625..9e0c8bc 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -85,8 +85,8 @@ def test_not_found_error(self) -> None: def test_invalid_dataset_error(self) -> None: """Test invalid dataset error.""" - error = InvalidDatasetError("Invalid format", format="json", line_number=10) - assert error.format == "json" + error = InvalidDatasetError("Invalid format", data_format="json", line_number=10) + assert error.data_format == "json" assert error.line_number == 10 def test_validation_error(self) -> None: