GH : 89 | Data validation script

.github/workflows/dataset-validator[tabular].yml

@@ -0,0 +1,44 @@
+name: Run Tabular Validator
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'data/**'
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: "Validation mode"
+        required: false
+        default: "tabular"
+
+jobs:
+  validate:
+    name: Validate Statistics Data [Tabular]
+    runs-on: ubuntu-latest
+
+    env:
+      DATASET_PATH: ${{ vars.DATASET_DIR_PATH_VAR || '../../data' }}
+      MODE: ${{ github.event.inputs.mode || 'tabular' }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: pip install -r scripts/validator/requirements.txt
+
+      - name: Run validator
+        working-directory: scripts/validator
+        run: |
+          echo "Running with:"
+          echo "Dataset Directory: $DATASET_PATH"
+          echo "Validation Mode: $MODE"
+
+          python main.py "$DATASET_PATH" "$MODE"

.gitignore

@@ -15,4 +15,7 @@ ingestion/.env
 # Generated zip files (not used by website)
 website/static/downloads/archive_Data.zip
 website/static/downloads/sources_Data.zip
-website/static/downloads/statistics_Data.zip
+website/static/downloads/statistics_Data.zip
+
+# Virtual environment
+venv/

scripts/validator/README.md

@@ -0,0 +1,73 @@
+# Data Validator Program
+
+A command-line tool for validating datasets using configurable validator types.
+
+---
+
+## Getting Started
+
+### Prerequisites
+
+- Python 3.x
+- `pip` and `venv`
+
+### Local Setup
+
+```bash
+# Navigate to the project directory
+cd scripts/validator
+
+# Create a virtual environment
+python3 -m venv venv
+
+# Activate the virtual environment
+source venv/bin/activate
+
+# Install dependencies
+pip install -r requirements.txt
+```
+
+### Running the Program
+
+```bash
+python main.py <path-to-dataset-directory> <validator-type>
+```
+
+**Example:**
+
+```bash
+python main.py ../../data/statistics tabular
+```
+
+---
+
+## Supported Validators
+
+| Validator | Description | Validations | Status |
+|-----------|-------------|-------------|--------|
+| `tabular` | Validates tabular data | `schema-validation` — Checks that the data conforms to a predefined schema (structures, types, constraints)<br><br>`duplicate-columns` — Detects columns that appear more than once<br><br>`row-column-mismatch` — Detects rows where the number of fields does not match the number of defined columns<br><br>`data-types-mismatch` — Identifies values that do not match the expected data type for their column (e.g. text in a numeric field)<br><br>`empty-values` — Flag cells that are null, blank, or contain only whitespace where a value is required<br><br>`value-overflow` — Catches values that exceed the maximum allowed length or numeric range for their column | ✅ Available |
+
+---
+
+## Project Structure
+
+```
+scripts/validator/
+│
+├── main.py               # Entry point
+├── requirements.txt      # Python dependencies
+├── README.md             # Project documentation
+│
+├── core/
+│   ├── baseRunner.py     # Base runner logic
+│   └── baseValidator.py  # Base validator interface
+│
+├── models/
+│   └── tabularSchema.json  # Schema definition for tabular validation
+│
+├── utils/
+│   └── utils.py          # Shared utility functions
+│
+└── validators/
+    └── tabular.py        # Tabular validator implementation
+```

scripts/validator/core/baseRunner.py

@@ -0,0 +1,34 @@
+import sys
+from pathlib import Path
+from utils.utils import Utils
+
+def run_validation(file_path, validator):
+    validator = validator()
+    all_errors = []
+    all_warnings = []
+
+    paths = list(Path(file_path).rglob("data.json"))
+
+    if not paths:
+        print("[INFO] No data.json files found")
+        sys.exit(0)
+
+    for path in paths:
+        errors, warnings = validator.validate_data(path)
+        all_errors.extend(errors)
+        all_warnings.extend(warnings)
+
+    if all_errors:
+        print(f" - {len(all_errors)} errors found")
+        for error in all_errors:
+            print(Utils.format_issue(error))
+
+    if all_warnings:
+        print(f" - {len(all_warnings)} warnings found")
+        for warning in all_warnings:
+            print(Utils.format_issue(warning))
+
+    if not all_errors and not all_warnings:
+        print("All data is valid ✅")
+
+    sys.exit(1 if all_errors else 0)

scripts/validator/core/baseValidator.py

@@ -0,0 +1,3 @@
+class BaseValidator:
+    def validate(self, file_path):
+        raise NotImplementedError

scripts/validator/main.py

@@ -0,0 +1,16 @@
+from core.baseRunner import run_validation
+from validators.tabular import TabularValidator
+import sys
+
+def main(file_path, validator):
+    if validator == "tabular":
+        run_validation(file_path, TabularValidator)
+    else:
+        print("Invalid validator")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3:
+        print("Usage: python main.py <directory> <validator>")
+        sys.exit(1)
+    main(sys.argv[1], sys.argv[2])

scripts/validator/models/tabularSchema.json

@@ -0,0 +1,22 @@
+{
+  "type": "object",
+  "required": ["columns", "rows"],
+  "properties": {
+    "columns": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "minItems": 1
+    },
+    "rows": {
+      "type": "array",
+      "items": {
+        "type": "array",
+        "items": {},
+        "minItems": 1
+      }
+    }
+  },
+  "additionalProperties": false
+}

scripts/validator/requirements.txt

@@ -0,0 +1,3 @@
+jsonschema>=4.25,<5
+pytest>=8.0.0
+

scripts/validator/tests/test_base_runner.py

@@ -0,0 +1,67 @@
+import pytest
+from unittest.mock import patch, MagicMock
+from core.baseRunner import run_validation
+
+class MockValidator:
+    def validate_data(self, path):
+        # We'll override this in specific tests
+        return [], []
+
+@patch("core.baseRunner.Path")
+def test_run_validation_no_files_found(mock_path_class, capsys):
+    # Setup mock Path.rglob to return an empty list
+    mock_path_instance = MagicMock()
+    mock_path_instance.rglob.return_value = []
+    mock_path_class.return_value = mock_path_instance
+
+    with pytest.raises(SystemExit) as excinfo:
+        run_validation("mock_dir", MockValidator)
+
+    assert excinfo.value.code == 0
+    captured = capsys.readouterr()
+    assert "[INFO] No data.json files found" in captured.out
+
+@patch("core.baseRunner.Path")
+def test_run_validation_all_valid(mock_path_class, capsys):
+    # Setup mock paths to return two file
+    mock_path_instance = MagicMock()
+    # Need to return an interable for rglob
+    mock_path_instance.rglob.return_value = ["mock_data_1.json", "mock_data_2.json"]
+    mock_path_class.return_value = mock_path_instance
+
+    class SuccessValidator(MockValidator):
+        def validate(self, path):
+            return [], []
+
+    with pytest.raises(SystemExit) as excinfo:
+        run_validation("mock_dir", SuccessValidator)
+
+    assert excinfo.value.code == 0
+    captured = capsys.readouterr()
+    assert "All data is valid ✅" in captured.out
+
+@patch("core.baseRunner.Path")
+def test_run_validation_with_errors_and_warnings(mock_path_class, capsys):
+    mock_path_instance = MagicMock()
+    mock_path_instance.rglob.return_value = ["mock_data.json"]
+    mock_path_class.return_value = mock_path_instance
+
+    class FailureValidator(MockValidator):
+        def validate_data(self, path):
+            errors = [{
+                "type": "error", "file": str(path), "row": 1, "column": "id", "message": "is wrong"
+            }]
+            warnings = [{
+                "type": "warning", "file": str(path), "row": 2, "column": "name", "message": "is empty"
+            }]
+            return errors, warnings
+
+    with pytest.raises(SystemExit) as excinfo:
+        run_validation("mock_dir", FailureValidator)
+
+    assert excinfo.value.code == 1
+    captured = capsys.readouterr()
+    assert "1 errors found" in captured.out
+    assert "1 warnings found" in captured.out
+    assert "[ERROR]" in captured.out
+    assert "[WARNING]" in captured.out

scripts/validator/tests/test_base_validator.py

@@ -0,0 +1,7 @@
+import pytest
+from core.baseValidator import BaseValidator
+
+def test_base_validator_validate_raises_not_implemented():
+    validator = BaseValidator()
+    with pytest.raises(NotImplementedError):
+        validator.validate("some_file.json")

scripts/validator/tests/test_main.py

@@ -0,0 +1,16 @@
+from unittest.mock import patch
+from main import main
+
+@patch("main.sys.exit")
+@patch("main.run_validation")
+def test_main_valid_validator(mock_run_validation, mock_sys_exit):
+    main("some_dir", "tabular")
+    mock_run_validation.assert_called_once()
+    assert mock_sys_exit.call_count == 0
+
+@patch("main.sys.exit")
+@patch("main.print")
+def test_main_invalid_validator(mock_print, mock_sys_exit):
+    main("some_dir", "unknown_validator")
+    mock_print.assert_called_with("Invalid validator")
+    mock_sys_exit.assert_called_with(1)