From 24a5d714356e0cd55d99ecfac6c542c43e494801 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Tue, 17 Mar 2026 12:47:59 +0530 Subject: [PATCH 01/16] feat : data validation script --- .gitignore | 5 +- scripts/validator/requirements.txt | 5 ++ scripts/validator/schema.json | 22 +++++++ scripts/validator/validator.py | 98 +++++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 scripts/validator/requirements.txt create mode 100644 scripts/validator/schema.json create mode 100644 scripts/validator/validator.py diff --git a/.gitignore b/.gitignore index 3deeccb3..3bef95e2 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,7 @@ ingestion/.env # Generated zip files (not used by website) website/static/downloads/archive_Data.zip website/static/downloads/sources_Data.zip -website/static/downloads/statistics_Data.zip \ No newline at end of file +website/static/downloads/statistics_Data.zip + +# Virtual environment +venv/ \ No newline at end of file diff --git a/scripts/validator/requirements.txt b/scripts/validator/requirements.txt new file mode 100644 index 00000000..79be454f --- /dev/null +++ b/scripts/validator/requirements.txt @@ -0,0 +1,5 @@ +jsonschema>=4.25,<5 + +# python -m venv .venv +# source .venv/bin/activate +# pip install -r requirements.txt diff --git a/scripts/validator/schema.json b/scripts/validator/schema.json new file mode 100644 index 00000000..d3bdb1e3 --- /dev/null +++ b/scripts/validator/schema.json @@ -0,0 +1,22 @@ +{ + "type": "object", + "required": ["columns", "rows"], + "properties": { + "columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": {}, + "minItems": 1 + } + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/scripts/validator/validator.py b/scripts/validator/validator.py new file mode 100644 index 00000000..ec9224ac --- /dev/null +++ b/scripts/validator/validator.py @@ -0,0 +1,98 @@ +import sys +import json +from pathlib import Path +from jsonschema import validate, ValidationError + +# Load schema once +with open("./schema.json") as f: + SCHEMA = json.load(f) + +def validate_file(file_path): + errors = [] + + # Load JSON + try: + with open(file_path) as f: + data = json.load(f) + except Exception as e: + return [f"[ERROR] {file_path}: Invalid JSON ({e})"] + + # 1. Schema validation + try: + validate(instance=data, schema=SCHEMA) + except ValidationError as e: + return [f"[ERROR] {file_path}: Schema error → {e.message}"] + + # 2. Custom validation -------------------------------------------------------------------------------------------------- + columns = data["columns"] + rows = data["rows"] + + num_cols = len(columns) + + # Check duplicate columns + if len(columns) != len(set(columns)): + errors.append(f"[ERROR] {file_path}: Duplicate column names found") + + # Check rows and columns mismatches + for i, row in enumerate(rows): + if len(row) != num_cols: + errors.append( + f"[ERROR] {file_path}: Row {i} has {len(row)} value(s), expected {num_cols} value(s)" + ) + + # Check data types + # if the column's first value starts from a sepcific data type, all the values down to the end on that column should be of the same data type + for i, row in enumerate(rows): + for j, value in enumerate(row): + expected_type = type(rows[0][j]) + if not isinstance(value, expected_type): + errors.append( + f"[ERROR] {file_path}: Row {i}, Column '{columns[j]}' has {value} ({type(value).__name__}), expected {expected_type.__name__}" + ) + + # Check for empty values (missing values) + for i, row in enumerate(rows): + for j, value in enumerate(row): + if value is None or value == "": + errors.append( + f"[WARNING] {file_path}: Row {i}, Column '{columns[j]}' has empty value" + ) + + # Floats should be in strings as a temporary mitigation + for i, row in enumerate(rows): + for j, value in enumerate(row): + if isinstance(value, float): + errors.append( + f"[WARNING] {file_path}: Row {i}, Column '{columns[j]}' has float value {value}. Convert to string. as a temporary mitigation" + ) + + return errors + + +def main(directory): + all_errors = [] + + paths = list(Path(directory).rglob("data.json")) + + if not paths: + print("[INFO] No data.json files found") + sys.exit(0) + + for path in paths: + all_errors.extend(validate_file(path)) + + if all_errors: + print("\n".join(all_errors)) + sys.exit(1) + else: + print("All data valid ✅") + sys.exit(0) + + +if __name__ == "__main__": + # python validator.py {data/statistics} + if len(sys.argv) < 2: + print("Usage: python validate.py ") + sys.exit(1) + + main(sys.argv[1]) \ No newline at end of file From 306b61eb80d18103db17da35d4cdd75cd617328e Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Wed, 18 Mar 2026 12:20:29 +0530 Subject: [PATCH 02/16] feat : package the script --- scripts/validator/core/baseRunner.py | 5 + scripts/validator/core/baseValidator.py | 3 + scripts/validator/main.py | 7 ++ .../tabularSchema.json} | 0 scripts/validator/validator.py | 98 ------------------- scripts/validator/validators/tabular.py | 60 ++++++++++++ 6 files changed, 75 insertions(+), 98 deletions(-) create mode 100644 scripts/validator/core/baseRunner.py create mode 100644 scripts/validator/core/baseValidator.py create mode 100644 scripts/validator/main.py rename scripts/validator/{schema.json => models/tabularSchema.json} (100%) delete mode 100644 scripts/validator/validator.py create mode 100644 scripts/validator/validators/tabular.py diff --git a/scripts/validator/core/baseRunner.py b/scripts/validator/core/baseRunner.py new file mode 100644 index 00000000..4f343723 --- /dev/null +++ b/scripts/validator/core/baseRunner.py @@ -0,0 +1,5 @@ +from validator.validators.tabular import TabularValidator + +def run_validation(file_path): + validator = TabularValidator() + return validator.validate(file_path) \ No newline at end of file diff --git a/scripts/validator/core/baseValidator.py b/scripts/validator/core/baseValidator.py new file mode 100644 index 00000000..c7d83a02 --- /dev/null +++ b/scripts/validator/core/baseValidator.py @@ -0,0 +1,3 @@ +class BaseValidator: + def validate(self, data, file_path): + raise NotImplementedError \ No newline at end of file diff --git a/scripts/validator/main.py b/scripts/validator/main.py new file mode 100644 index 00000000..0e4e96a0 --- /dev/null +++ b/scripts/validator/main.py @@ -0,0 +1,7 @@ +from validator.core.runner import run_validation + +def main(): + run_validation("/Users/yasandu/Documents/datasets/data/statistics") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/validator/schema.json b/scripts/validator/models/tabularSchema.json similarity index 100% rename from scripts/validator/schema.json rename to scripts/validator/models/tabularSchema.json diff --git a/scripts/validator/validator.py b/scripts/validator/validator.py deleted file mode 100644 index ec9224ac..00000000 --- a/scripts/validator/validator.py +++ /dev/null @@ -1,98 +0,0 @@ -import sys -import json -from pathlib import Path -from jsonschema import validate, ValidationError - -# Load schema once -with open("./schema.json") as f: - SCHEMA = json.load(f) - -def validate_file(file_path): - errors = [] - - # Load JSON - try: - with open(file_path) as f: - data = json.load(f) - except Exception as e: - return [f"[ERROR] {file_path}: Invalid JSON ({e})"] - - # 1. Schema validation - try: - validate(instance=data, schema=SCHEMA) - except ValidationError as e: - return [f"[ERROR] {file_path}: Schema error → {e.message}"] - - # 2. Custom validation -------------------------------------------------------------------------------------------------- - columns = data["columns"] - rows = data["rows"] - - num_cols = len(columns) - - # Check duplicate columns - if len(columns) != len(set(columns)): - errors.append(f"[ERROR] {file_path}: Duplicate column names found") - - # Check rows and columns mismatches - for i, row in enumerate(rows): - if len(row) != num_cols: - errors.append( - f"[ERROR] {file_path}: Row {i} has {len(row)} value(s), expected {num_cols} value(s)" - ) - - # Check data types - # if the column's first value starts from a sepcific data type, all the values down to the end on that column should be of the same data type - for i, row in enumerate(rows): - for j, value in enumerate(row): - expected_type = type(rows[0][j]) - if not isinstance(value, expected_type): - errors.append( - f"[ERROR] {file_path}: Row {i}, Column '{columns[j]}' has {value} ({type(value).__name__}), expected {expected_type.__name__}" - ) - - # Check for empty values (missing values) - for i, row in enumerate(rows): - for j, value in enumerate(row): - if value is None or value == "": - errors.append( - f"[WARNING] {file_path}: Row {i}, Column '{columns[j]}' has empty value" - ) - - # Floats should be in strings as a temporary mitigation - for i, row in enumerate(rows): - for j, value in enumerate(row): - if isinstance(value, float): - errors.append( - f"[WARNING] {file_path}: Row {i}, Column '{columns[j]}' has float value {value}. Convert to string. as a temporary mitigation" - ) - - return errors - - -def main(directory): - all_errors = [] - - paths = list(Path(directory).rglob("data.json")) - - if not paths: - print("[INFO] No data.json files found") - sys.exit(0) - - for path in paths: - all_errors.extend(validate_file(path)) - - if all_errors: - print("\n".join(all_errors)) - sys.exit(1) - else: - print("All data valid ✅") - sys.exit(0) - - -if __name__ == "__main__": - # python validator.py {data/statistics} - if len(sys.argv) < 2: - print("Usage: python validate.py ") - sys.exit(1) - - main(sys.argv[1]) \ No newline at end of file diff --git a/scripts/validator/validators/tabular.py b/scripts/validator/validators/tabular.py new file mode 100644 index 00000000..c73f546a --- /dev/null +++ b/scripts/validator/validators/tabular.py @@ -0,0 +1,60 @@ +from validator.core.baseValidator import BaseValidator +import json +from jsonschema import validate, ValidationError + +class TabularValidator(BaseValidator): + def __init__(self): + with open("./models/tabularSchema.json") as f: + self.schema = json.load(f) + + def validate(self, file_path): + errors = [] + + # Load JSON + try: + with open(file_path) as f: + data = json.load(f) + except Exception as e: + return [f"[ERROR] {file_path}: Invalid JSON ({e})"] + + # 1. Schema validation + try: + validate(instance=data, schema=self.schema) + except ValidationError as e: + return [f"[ERROR] {file_path}: Schema error → {e.message}"] + + # 2. Custom validation + columns = data["columns"] + rows = data["rows"] + + num_cols = len(columns) + + # Check duplicate columns + if len(columns) != len(set(columns)): + errors.append(f"[ERROR] {file_path}: Duplicate column names found") + + # Check rows and columns mismatches + for i, row in enumerate(rows): + if len(row) != num_cols: + errors.append( + f"[ERROR] {file_path}: Row {i} has {len(row)} value(s), expected {num_cols} value(s)" + ) + + # Check data types + # if the column's first value starts from a sepcific data type, all the values down to the end on that column should be of the same data type + for i, row in enumerate(rows): + for j, value in enumerate(row): + expected_type = type(rows[0][j]) + if not isinstance(value, expected_type): + errors.append( + f"[ERROR] {file_path}: Row {i}, Column '{columns[j]}' has {value} ({type(value).__name__}), expected {expected_type.__name__}" + ) + + # Check for empty values (missing values) + for i, row in enumerate(rows): + for j, value in enumerate(row): + if value is None or value == "": + errors.append( + f"[WARNING] {file_path}: Row {i}, Column '{columns[j]}' has empty value" + ) + return errors From 0e315422ee6317c725f2babb0a98af6d1cf94867 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 19 Mar 2026 15:09:53 +0530 Subject: [PATCH 03/16] feat : final refacotring and packaging the script --- scripts/validator/core/baseRunner.py | 37 ++++++++- scripts/validator/core/baseValidator.py | 2 +- scripts/validator/main.py | 17 +++- scripts/validator/utils/utils.py | 23 ++++++ scripts/validator/validators/tabular.py | 103 ++++++++++++++++++++---- 5 files changed, 158 insertions(+), 24 deletions(-) create mode 100644 scripts/validator/utils/utils.py diff --git a/scripts/validator/core/baseRunner.py b/scripts/validator/core/baseRunner.py index 4f343723..06db53c6 100644 --- a/scripts/validator/core/baseRunner.py +++ b/scripts/validator/core/baseRunner.py @@ -1,5 +1,34 @@ -from validator.validators.tabular import TabularValidator +import sys +from pathlib import Path +from utils.utils import Utils -def run_validation(file_path): - validator = TabularValidator() - return validator.validate(file_path) \ No newline at end of file +def run_validation(file_path, validator): + validator = validator() + all_errors = [] + all_warnings = [] + + paths = list(Path(file_path).rglob("data.json")) + + if not paths: + print("[INFO] No data.json files found") + sys.exit(0) + + for path in paths: + errors, warnings = validator.validate(path) + all_errors.extend(errors) + all_warnings.extend(warnings) + + if all_errors: + print(f" - {len(all_errors)} errors found") + for error in all_errors: + print(Utils.format_issue(error)) + + if all_warnings: + print(f" - {len(all_warnings)} warnings found") + for warning in all_warnings: + print(Utils.format_issue(warning)) + + if not all_errors and not all_warnings: + print("All data valid ✅") + + sys.exit(1 if all_errors else 0) diff --git a/scripts/validator/core/baseValidator.py b/scripts/validator/core/baseValidator.py index c7d83a02..998523fb 100644 --- a/scripts/validator/core/baseValidator.py +++ b/scripts/validator/core/baseValidator.py @@ -1,3 +1,3 @@ class BaseValidator: - def validate(self, data, file_path): + def validate(self, file_path): raise NotImplementedError \ No newline at end of file diff --git a/scripts/validator/main.py b/scripts/validator/main.py index 0e4e96a0..a9a303d9 100644 --- a/scripts/validator/main.py +++ b/scripts/validator/main.py @@ -1,7 +1,16 @@ -from validator.core.runner import run_validation +from core.baseRunner import run_validation +from validators.tabular import TabularValidator +import sys -def main(): - run_validation("/Users/yasandu/Documents/datasets/data/statistics") +def main(file_path, validator): + if validator == "tabular": + run_validation(file_path, TabularValidator) + else: + print("Invalid validator") + sys.exit(1) if __name__ == "__main__": - main() \ No newline at end of file + if len(sys.argv) < 3: + print("Usage: python main.py ") + sys.exit(1) + main(sys.argv[1], sys.argv[2]) \ No newline at end of file diff --git a/scripts/validator/utils/utils.py b/scripts/validator/utils/utils.py new file mode 100644 index 00000000..bdcac551 --- /dev/null +++ b/scripts/validator/utils/utils.py @@ -0,0 +1,23 @@ +class Utils: + @staticmethod + def format_issue(issue): + location = "" + + if issue.get("row") is not None: + location += f"Row {issue['row']}" + + if issue.get("column"): + # handle list of columns OR single column + if isinstance(issue["column"], list): + cols = ", ".join(issue["column"]) + location += f", Columns [{cols}]" + else: + location += f", Column '{issue['column']}'" + + return f"[{issue['type'].upper()}] {issue['file']}: {location} {issue['message']}" + + @staticmethod + def fits_in_int32(value: int) -> bool: + INT32_MIN = -2_147_483_648 + INT32_MAX = 2_147_483_647 + return INT32_MIN <= value <= INT32_MAX \ No newline at end of file diff --git a/scripts/validator/validators/tabular.py b/scripts/validator/validators/tabular.py index c73f546a..36a8400c 100644 --- a/scripts/validator/validators/tabular.py +++ b/scripts/validator/validators/tabular.py @@ -1,27 +1,30 @@ -from validator.core.baseValidator import BaseValidator +from core.baseValidator import BaseValidator import json from jsonschema import validate, ValidationError +from collections import Counter +from utils.utils import Utils class TabularValidator(BaseValidator): def __init__(self): with open("./models/tabularSchema.json") as f: self.schema = json.load(f) - + def validate(self, file_path): errors = [] + warnings = [] # Load JSON try: with open(file_path) as f: data = json.load(f) except Exception as e: - return [f"[ERROR] {file_path}: Invalid JSON ({e})"] + return [f"[ERROR] {file_path}: Invalid JSON ({e})"], [] # 1. Schema validation try: validate(instance=data, schema=self.schema) except ValidationError as e: - return [f"[ERROR] {file_path}: Schema error → {e.message}"] + return [f"[ERROR] {file_path}: Schema error → {e.message}"], [] # 2. Custom validation columns = data["columns"] @@ -31,30 +34,100 @@ def validate(self, file_path): # Check duplicate columns if len(columns) != len(set(columns)): - errors.append(f"[ERROR] {file_path}: Duplicate column names found") + column_counts = Counter(columns) + duplicates = [col for col, count in column_counts.items() if count > 1] + if duplicates: + errors.append( + { + "type": "error", + "file": file_path, + "row": None, + "column": duplicates, + "message": f"Duplicate column names found: {', '.join(duplicates)}", + } + ) # Check rows and columns mismatches for i, row in enumerate(rows): if len(row) != num_cols: errors.append( - f"[ERROR] {file_path}: Row {i} has {len(row)} value(s), expected {num_cols} value(s)" + { + "type": "error", + "file": file_path, + "row": i, + "column": None, + "message": f"has {len(row)} value(s), expected {num_cols} value(s)", + } ) - + # Check data types # if the column's first value starts from a sepcific data type, all the values down to the end on that column should be of the same data type for i, row in enumerate(rows): for j, value in enumerate(row): expected_type = type(rows[0][j]) - if not isinstance(value, expected_type): - errors.append( - f"[ERROR] {file_path}: Row {i}, Column '{columns[j]}' has {value} ({type(value).__name__}), expected {expected_type.__name__}" - ) - + if expected_type is str: + if not isinstance(value, str): + errors.append( + { + "type": "error", + "file": file_path, + "row": i, + "column": columns[j], + "message": f"has {value} ({type(value).__name__}), expected {expected_type.__name__}", + } + ) + elif expected_type is int: + if not isinstance(value, int): + errors.append( + { + "type": "error", + "file": file_path, + "row": i, + "column": columns[j], + "message": f"has {value} ({type(value).__name__}), expected {expected_type.__name__}", + } + ) + elif expected_type is float: + if not isinstance(value, (float, int)): + errors.append( + { + "type": "error", + "file": file_path, + "row": i, + "column": columns[j], + "message": f"has {value} ({type(value).__name__}), expected {expected_type.__name__}", + } + ) + # Check for empty values (missing values) for i, row in enumerate(rows): for j, value in enumerate(row): if value is None or value == "": - errors.append( - f"[WARNING] {file_path}: Row {i}, Column '{columns[j]}' has empty value" + warnings.append( + { + "type": "warning", + "file": file_path, + "row": i, + "column": columns[j], + "message": "has empty value", + } ) - return errors + + # Check for value overflow (temporary fix for opengin system) + for i, row in enumerate(rows): + for j, value in enumerate(row): + if isinstance(value, int): + if not Utils.fits_in_int32(value): + warnings.append( + { + "type": "warning", + "file": file_path, + "row": i, + "column": columns[j], + "message": f"has {value} ({type(value).__name__}), this is a big integer in postgres , consider when inserting into the database (postgres has 32 bit integer limit)", + } + ) + + return errors, warnings + + From 34415470bac370e9fc711039fca92902f7f971ee Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 19 Mar 2026 15:23:59 +0530 Subject: [PATCH 04/16] fix : fixing review comments --- scripts/validator/validators/tabular.py | 83 ++++++++++++++++--------- 1 file changed, 52 insertions(+), 31 deletions(-) diff --git a/scripts/validator/validators/tabular.py b/scripts/validator/validators/tabular.py index 36a8400c..63fe78b1 100644 --- a/scripts/validator/validators/tabular.py +++ b/scripts/validator/validators/tabular.py @@ -3,36 +3,15 @@ from jsonschema import validate, ValidationError from collections import Counter from utils.utils import Utils +from pathlib import Path class TabularValidator(BaseValidator): def __init__(self): - with open("./models/tabularSchema.json") as f: + with open(Path(__file__).parent / "../models/tabularSchema.json") as f: self.schema = json.load(f) - def validate(self, file_path): + def _check_duplicate_columns(self, file_path, columns): errors = [] - warnings = [] - - # Load JSON - try: - with open(file_path) as f: - data = json.load(f) - except Exception as e: - return [f"[ERROR] {file_path}: Invalid JSON ({e})"], [] - - # 1. Schema validation - try: - validate(instance=data, schema=self.schema) - except ValidationError as e: - return [f"[ERROR] {file_path}: Schema error → {e.message}"], [] - - # 2. Custom validation - columns = data["columns"] - rows = data["rows"] - - num_cols = len(columns) - - # Check duplicate columns if len(columns) != len(set(columns)): column_counts = Counter(columns) duplicates = [col for col, count in column_counts.items() if count > 1] @@ -45,9 +24,11 @@ def validate(self, file_path): "column": duplicates, "message": f"Duplicate column names found: {', '.join(duplicates)}", } - ) + ) + return errors - # Check rows and columns mismatches + def _check_row_column_mismatches(self, file_path, rows, num_cols): + errors = [] for i, row in enumerate(rows): if len(row) != num_cols: errors.append( @@ -59,9 +40,13 @@ def validate(self, file_path): "message": f"has {len(row)} value(s), expected {num_cols} value(s)", } ) + return errors - # Check data types - # if the column's first value starts from a sepcific data type, all the values down to the end on that column should be of the same data type + def _check_data_types(self, file_path, rows, columns): + errors = [] + if not rows: + return errors + for i, row in enumerate(rows): for j, value in enumerate(row): expected_type = type(rows[0][j]) @@ -98,8 +83,10 @@ def validate(self, file_path): "message": f"has {value} ({type(value).__name__}), expected {expected_type.__name__}", } ) + return errors - # Check for empty values (missing values) + def _check_empty_values(self, file_path, rows, columns): + warnings = [] for i, row in enumerate(rows): for j, value in enumerate(row): if value is None or value == "": @@ -112,8 +99,10 @@ def validate(self, file_path): "message": "has empty value", } ) - - # Check for value overflow (temporary fix for opengin system) + return warnings + + def _check_value_overflow(self, file_path, rows, columns): + warnings = [] for i, row in enumerate(rows): for j, value in enumerate(row): if isinstance(value, int): @@ -127,6 +116,38 @@ def validate(self, file_path): "message": f"has {value} ({type(value).__name__}), this is a big integer in postgres , consider when inserting into the database (postgres has 32 bit integer limit)", } ) + return warnings + + def validate(self, file_path): + errors = [] + warnings = [] + + # Load JSON + try: + with open(file_path) as f: + data = json.load(f) + except (json.JSONDecodeError, FileNotFoundError) as e: + return [f"[ERROR] {file_path}: Invalid JSON ({e})"], [] + + # 1. Schema validation + try: + validate(instance=data, schema=self.schema) + except ValidationError as e: + return [f"[ERROR] {file_path}: Schema error → {e.message}"], [] + + # 2. Custom validation + columns = data.get("columns", []) + rows = data.get("rows", []) + num_cols = len(columns) + + # errors -------- + errors.extend(self._check_duplicate_columns(file_path, columns)) + errors.extend(self._check_row_column_mismatches(file_path, rows, num_cols)) + errors.extend(self._check_data_types(file_path, rows, columns)) + + # warnings -------- + warnings.extend(self._check_empty_values(file_path, rows, columns)) + warnings.extend(self._check_value_overflow(file_path, rows, columns)) return errors, warnings From 9daa07cab00b14b900768bf4efbdadfddbf93c5b Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 19 Mar 2026 15:44:57 +0530 Subject: [PATCH 05/16] test : test workflow to validate datasets --- .../workflows/dataset-validator[tabular].yml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/dataset-validator[tabular].yml diff --git a/.github/workflows/dataset-validator[tabular].yml b/.github/workflows/dataset-validator[tabular].yml new file mode 100644 index 00000000..05b11ba9 --- /dev/null +++ b/.github/workflows/dataset-validator[tabular].yml @@ -0,0 +1,30 @@ +name: Run Tabular Validator + +on: + pull_request: + branches: + - workflow/validating-datasets + paths: + - 'data/statistics/**' + +jobs: + validate: + name: Validate Statistics Data + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + working-directory: scripts/validator + run: pip install -r requirements.txt + + - name: Run validator + working-directory: scripts/validator + run: python main.py ../../data/statistics tabular \ No newline at end of file From 9f917f7fab73a23ad5b2ed2e7d1751e3780db0a4 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Fri, 20 Mar 2026 11:36:51 +0530 Subject: [PATCH 06/16] feat : adding the ci-pipeline for data validation [tabular data] --- .../workflows/dataset-validator[tabular].yml | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dataset-validator[tabular].yml b/.github/workflows/dataset-validator[tabular].yml index 05b11ba9..9db45a2f 100644 --- a/.github/workflows/dataset-validator[tabular].yml +++ b/.github/workflows/dataset-validator[tabular].yml @@ -3,15 +3,25 @@ name: Run Tabular Validator on: pull_request: branches: - - workflow/validating-datasets + - main paths: - 'data/statistics/**' + workflow_dispatch: + inputs: + mode: + description: "Validation mode" + required: false + default: "tabular" jobs: validate: - name: Validate Statistics Data + name: Validate Statistics Data [Tabular] runs-on: ubuntu-latest + env: + DATASET_PATH: ${{ vars.DATASET_DIR_PATH_VAR || '../../data/statistics' }} + MODE: ${{ github.event.inputs.mode || 'tabular' }} + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -20,11 +30,15 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12' - + - name: Install dependencies - working-directory: scripts/validator - run: pip install -r requirements.txt + run: pip install -r scripts/validator/requirements.txt - name: Run validator working-directory: scripts/validator - run: python main.py ../../data/statistics tabular \ No newline at end of file + run: | + echo "Running with:" + echo "Dataset Directory: $DATASET_PATH" + echo "Validation Mode: $MODE" + + python main.py "$DATASET_PATH" "$MODE" \ No newline at end of file From e6400e33ea0ac0e219941ad43ef8d303ed4988a9 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Mon, 23 Mar 2026 13:48:06 +0530 Subject: [PATCH 07/16] fix: filename error fixing --- scripts/validator/{requirements.txt => requirements.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/validator/{requirements.txt => requirements.txt} (100%) diff --git a/scripts/validator/requirements.txt b/scripts/validator/requirements.txt similarity index 100% rename from scripts/validator/requirements.txt rename to scripts/validator/requirements.txt From ee6c34adc08cac9bf42c24c344243eeabfc8e565 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Mon, 23 Mar 2026 15:22:18 +0530 Subject: [PATCH 08/16] review: resolving review comments by chanuka and sehansi --- scripts/validator/requirements.txt | 3 - scripts/validator/validators/tabular.py | 153 ++++++++++++------------ 2 files changed, 78 insertions(+), 78 deletions(-) diff --git a/scripts/validator/requirements.txt b/scripts/validator/requirements.txt index 79be454f..d07debbe 100644 --- a/scripts/validator/requirements.txt +++ b/scripts/validator/requirements.txt @@ -1,5 +1,2 @@ jsonschema>=4.25,<5 -# python -m venv .venv -# source .venv/bin/activate -# pip install -r requirements.txt diff --git a/scripts/validator/validators/tabular.py b/scripts/validator/validators/tabular.py index 63fe78b1..365de33a 100644 --- a/scripts/validator/validators/tabular.py +++ b/scripts/validator/validators/tabular.py @@ -11,12 +11,11 @@ def __init__(self): self.schema = json.load(f) def _check_duplicate_columns(self, file_path, columns): - errors = [] if len(columns) != len(set(columns)): column_counts = Counter(columns) duplicates = [col for col, count in column_counts.items() if count > 1] if duplicates: - errors.append( + return [ { "type": "error", "file": file_path, @@ -24,57 +23,27 @@ def _check_duplicate_columns(self, file_path, columns): "column": duplicates, "message": f"Duplicate column names found: {', '.join(duplicates)}", } - ) - return errors + ] + return [] - def _check_row_column_mismatches(self, file_path, rows, num_cols): - errors = [] - for i, row in enumerate(rows): - if len(row) != num_cols: - errors.append( - { - "type": "error", - "file": file_path, - "row": i, - "column": None, - "message": f"has {len(row)} value(s), expected {num_cols} value(s)", - } - ) - return errors + def _check_row_column_mismatch(self, file_path, i, row, num_cols): + if len(row) != num_cols: + return [{ + "type": "error", + "file": file_path, + "row": i, + "column": None, + "message": f"has only {len(row)} value(s), expected {num_cols} value(s)", + }] + return [] - def _check_data_types(self, file_path, rows, columns): + def _check_data_types(self, file_path, i, row, first_row, columns): errors = [] - if not rows: - return errors - - for i, row in enumerate(rows): - for j, value in enumerate(row): - expected_type = type(rows[0][j]) - if expected_type is str: - if not isinstance(value, str): - errors.append( - { - "type": "error", - "file": file_path, - "row": i, - "column": columns[j], - "message": f"has {value} ({type(value).__name__}), expected {expected_type.__name__}", - } - ) - elif expected_type is int: - if not isinstance(value, int): - errors.append( - { - "type": "error", - "file": file_path, - "row": i, - "column": columns[j], - "message": f"has {value} ({type(value).__name__}), expected {expected_type.__name__}", - } - ) - elif expected_type is float: - if not isinstance(value, (float, int)): - errors.append( + for j, value in enumerate(row): + expected_type = type(first_row[j]) + if expected_type is str: + if not isinstance(value, expected_type): + errors.append( { "type": "error", "file": file_path, @@ -82,40 +51,57 @@ def _check_data_types(self, file_path, rows, columns): "column": columns[j], "message": f"has {value} ({type(value).__name__}), expected {expected_type.__name__}", } - ) + ) + elif expected_type is int: + if not isinstance(value, expected_type): + errors.append( + { + "type": "error", + "file": file_path, + "row": i, + "column": columns[j], + "message": f"has {value} ({type(value).__name__}), expected {expected_type.__name__}", + } + ) + elif expected_type is float: + if not isinstance(value, (expected_type, int)): + errors.append( + { + "type": "error", + "file": file_path, + "row": i, + "column": columns[j], + "message": f"has {value} ({type(value).__name__}), expected {expected_type.__name__} or whole number", + } + ) return errors - def _check_empty_values(self, file_path, rows, columns): + def _check_empty_values(self, file_path, i, row, columns): warnings = [] - for i, row in enumerate(rows): - for j, value in enumerate(row): - if value is None or value == "": - warnings.append( - { + for j, value in enumerate(row): + str_value = str(value).strip() if value is not None else "" + if str_value == "": + warnings.append({ "type": "warning", "file": file_path, "row": i, "column": columns[j], "message": "has empty value", - } - ) + }) return warnings - def _check_value_overflow(self, file_path, rows, columns): + def _check_value_overflow(self, file_path, i, row, columns): warnings = [] - for i, row in enumerate(rows): - for j, value in enumerate(row): - if isinstance(value, int): - if not Utils.fits_in_int32(value): - warnings.append( - { + for j, value in enumerate(row): + if isinstance(value, int): + if not Utils.fits_in_int32(value): + warnings.append({ "type": "warning", "file": file_path, "row": i, "column": columns[j], "message": f"has {value} ({type(value).__name__}), this is a big integer in postgres , consider when inserting into the database (postgres has 32 bit integer limit)", - } - ) + }) return warnings def validate(self, file_path): @@ -140,14 +126,31 @@ def validate(self, file_path): rows = data.get("rows", []) num_cols = len(columns) - # errors -------- + if not rows or not columns: + if not rows: + message = "No rows found" + elif not columns: + message = "No columns found" + else: + message = "No rows or columns found" + + errors.extend([{ + "type": "error", + "file": file_path, + "row": rows if rows else None, + "column": columns if columns else None, + "message": message + } + ]) + return errors, warnings + errors.extend(self._check_duplicate_columns(file_path, columns)) - errors.extend(self._check_row_column_mismatches(file_path, rows, num_cols)) - errors.extend(self._check_data_types(file_path, rows, columns)) - - # warnings -------- - warnings.extend(self._check_empty_values(file_path, rows, columns)) - warnings.extend(self._check_value_overflow(file_path, rows, columns)) + + for i, row in enumerate(rows): + errors.extend(self._check_row_column_mismatch(file_path, i, row, num_cols)) + warnings.extend(self._check_empty_values(file_path, i, row, columns)) + warnings.extend(self._check_value_overflow(file_path, i, row, columns)) + errors.extend(self._check_data_types(file_path, i, row, rows[0], columns)) return errors, warnings From a3ed6ac673b766fc7fb427e7bcc947a3adc2b343 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Mon, 23 Mar 2026 15:55:39 +0530 Subject: [PATCH 09/16] fix : fix a condition checking --- scripts/validator/validators/tabular.py | 48 ++++++++----------------- 1 file changed, 15 insertions(+), 33 deletions(-) diff --git a/scripts/validator/validators/tabular.py b/scripts/validator/validators/tabular.py index 365de33a..dd40071d 100644 --- a/scripts/validator/validators/tabular.py +++ b/scripts/validator/validators/tabular.py @@ -41,39 +41,21 @@ def _check_data_types(self, file_path, i, row, first_row, columns): errors = [] for j, value in enumerate(row): expected_type = type(first_row[j]) - if expected_type is str: - if not isinstance(value, expected_type): - errors.append( - { - "type": "error", - "file": file_path, - "row": i, - "column": columns[j], - "message": f"has {value} ({type(value).__name__}), expected {expected_type.__name__}", - } - ) - elif expected_type is int: - if not isinstance(value, expected_type): - errors.append( - { - "type": "error", - "file": file_path, - "row": i, - "column": columns[j], - "message": f"has {value} ({type(value).__name__}), expected {expected_type.__name__}", - } - ) - elif expected_type is float: - if not isinstance(value, (expected_type, int)): - errors.append( - { - "type": "error", - "file": file_path, - "row": i, - "column": columns[j], - "message": f"has {value} ({type(value).__name__}), expected {expected_type.__name__} or whole number", - } - ) + if expected_type is float: + allowed_types = (float, int) + expected_msg = "float or whole number" + else: + allowed_types = (expected_type) + expected_msg = expected_type.__name__ + + if not isinstance(value, allowed_types): + errors.append({ + "type": "error", + "file": file_path, + "row": i, + "column": columns[j], + "message": f"has {value} ({type(value).__name__}), expected {expected_msg}", + }) return errors def _check_empty_values(self, file_path, i, row, columns): From 5ef1cd2b4a4d1ded232c70fd48ebb4f521f9986a Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Mon, 23 Mar 2026 16:07:06 +0530 Subject: [PATCH 10/16] feat : adding the readme file --- scripts/validator/README.md | 75 +++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 scripts/validator/README.md diff --git a/scripts/validator/README.md b/scripts/validator/README.md new file mode 100644 index 00000000..f58ab10c --- /dev/null +++ b/scripts/validator/README.md @@ -0,0 +1,75 @@ +# Data Validator Program + +A command-line tool for validating datasets using configurable validator types. + +--- + +## Getting Started + +### Prerequisites + +- Python 3.x +- `pip` and `venv` + +### Local Setup + +```bash +# Navigate to the project directory +cd scripts/validator + +# Create a virtual environment +python3 -m venv venv + +# Activate the virtual environment +source venv/bin/activate + +# Install dependencies +pip install -r requirements.txt +``` + +### Running the Program + +```bash +python main.py +``` + +**Example:** + +```bash +python main.py ../../data/statistics tabular +``` + +--- + +## Supported Validators + +| Validator | Description | Status | +|-----------|--------------------------|---------| +| `tabular` | Validates tabular data | ✅ Available | + +> **Note:** Additional validators are currently under development. + +--- + +## Project Structure + +``` +scripts/validator/ +│ +├── main.py # Entry point +├── requirements.txt # Python dependencies +├── README.md # Project documentation +│ +├── core/ +│ ├── baseRunner.py # Base runner logic +│ └── baseValidator.py # Base validator interface +│ +├── models/ +│ └── tabularSchema.json # Schema definition for tabular validation +│ +├── utils/ +│ └── utils.py # Shared utility functions +│ +└── validators/ + └── tabular.py # Tabular validator implementation +``` \ No newline at end of file From 39b3fe89a9afc4e1623b85409c62441dfc7f9c36 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Tue, 24 Mar 2026 13:12:47 +0530 Subject: [PATCH 11/16] feat : adding test cases to the validation programme --- scripts/validator/requirements.txt | 1 + .../validator/tests/test_base_validator.py | 7 +++ scripts/validator/tests/test_main.py | 16 ++++++ scripts/validator/tests/test_utils.py | 57 +++++++++++++++++++ 4 files changed, 81 insertions(+) create mode 100644 scripts/validator/tests/test_base_validator.py create mode 100644 scripts/validator/tests/test_main.py create mode 100644 scripts/validator/tests/test_utils.py diff --git a/scripts/validator/requirements.txt b/scripts/validator/requirements.txt index d07debbe..6e221097 100644 --- a/scripts/validator/requirements.txt +++ b/scripts/validator/requirements.txt @@ -1,2 +1,3 @@ jsonschema>=4.25,<5 +pytest>=8.0.0 diff --git a/scripts/validator/tests/test_base_validator.py b/scripts/validator/tests/test_base_validator.py new file mode 100644 index 00000000..098f97ee --- /dev/null +++ b/scripts/validator/tests/test_base_validator.py @@ -0,0 +1,7 @@ +import pytest +from core.baseValidator import BaseValidator + +def test_base_validator_validate_raises_not_implemented(): + validator = BaseValidator() + with pytest.raises(NotImplementedError): + validator.validate("some_file.json") diff --git a/scripts/validator/tests/test_main.py b/scripts/validator/tests/test_main.py new file mode 100644 index 00000000..d610a66e --- /dev/null +++ b/scripts/validator/tests/test_main.py @@ -0,0 +1,16 @@ +from unittest.mock import patch +from main import main + +@patch("main.sys.exit") +@patch("main.run_validation") +def test_main_valid_validator(mock_run_validation, mock_sys_exit): + main("some_dir", "tabular") + mock_run_validation.assert_called_once() + assert mock_sys_exit.call_count == 0 + +@patch("main.sys.exit") +@patch("main.print") +def test_main_invalid_validator(mock_print, mock_sys_exit): + main("some_dir", "unknown_validator") + mock_print.assert_called_with("Invalid validator") + mock_sys_exit.assert_called_with(1) diff --git a/scripts/validator/tests/test_utils.py b/scripts/validator/tests/test_utils.py new file mode 100644 index 00000000..54e2b451 --- /dev/null +++ b/scripts/validator/tests/test_utils.py @@ -0,0 +1,57 @@ +from utils.utils import Utils + +def test_format_issue_with_row_and_single_column(): + issue = { + "type": "error", + "file": "data.json", + "row": 5, + "column": "name", + "message": "is invalid" + } + result = Utils.format_issue(issue) + assert result == "[ERROR] data.json: Row 5, Column 'name' is invalid" + +def test_format_issue_with_row_and_multiple_columns(): + issue = { + "type": "warning", + "file": "data.csv", + "row": 10, + "column": ["age", "dob"], + "message": "have conflicting values" + } + result = Utils.format_issue(issue) + assert result == "[WARNING] data.csv: Row 10, Columns [age, dob] have conflicting values" + +def test_format_issue_without_row_and_column(): + issue = { + "type": "error", + "file": "config.json", + "message": "File not found or unreadable" + } + result = Utils.format_issue(issue) + assert result == "[ERROR] config.json: File not found or unreadable" + +def test_format_issue_without_row_with_column(): + issue = { + "type": "error", + "file": "schema.json", + "column": ["header1", "header2"], + "message": "Duplicate columns" + } + result = Utils.format_issue(issue) + assert result == "[ERROR] schema.json: Columns [header1, header2] Duplicate columns" + +def test_fits_in_int32(): + # Boundary values + assert Utils.fits_in_int32(-2_147_483_648) is True + assert Utils.fits_in_int32(2_147_483_647) is True + + # Internal values + assert Utils.fits_in_int32(0) is True + assert Utils.fits_in_int32(1000) is True + assert Utils.fits_in_int32(-50000) is True + + # Out of bounds + assert Utils.fits_in_int32(-2_147_483_649) is False + assert Utils.fits_in_int32(2_147_483_648) is False + assert Utils.fits_in_int32(10_000_000_000) is False From f0a40e2a4a8d629a879540f1f040d11c871bc957 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Tue, 24 Mar 2026 13:40:42 +0530 Subject: [PATCH 12/16] fix : fixing util tests --- scripts/validator/tests/test_utils.py | 36 +++++++++++++++++++++------ 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/scripts/validator/tests/test_utils.py b/scripts/validator/tests/test_utils.py index 54e2b451..3cf9aa95 100644 --- a/scripts/validator/tests/test_utils.py +++ b/scripts/validator/tests/test_utils.py @@ -1,51 +1,73 @@ from utils.utils import Utils + def test_format_issue_with_row_and_single_column(): issue = { "type": "error", "file": "data.json", "row": 5, "column": "name", - "message": "is invalid" + "message": "is invalid", } result = Utils.format_issue(issue) assert result == "[ERROR] data.json: Row 5, Column 'name' is invalid" + def test_format_issue_with_row_and_multiple_columns(): issue = { "type": "warning", "file": "data.csv", "row": 10, "column": ["age", "dob"], - "message": "have conflicting values" + "message": "have conflicting values", } result = Utils.format_issue(issue) - assert result == "[WARNING] data.csv: Row 10, Columns [age, dob] have conflicting values" + assert ( + result + == "[WARNING] data.csv: Row 10, Columns [age, dob] have conflicting values" + ) + def test_format_issue_without_row_and_column(): issue = { "type": "error", "file": "config.json", - "message": "File not found or unreadable" + "message": "File not found or unreadable", } result = Utils.format_issue(issue) assert result == "[ERROR] config.json: File not found or unreadable" + +def test_format_issue_without_rosw_and_column(): + issue = { + "type": "error", + "file": "file.csv", + "row": None, + "column": ["header1", "header2"], + "message": "No rows found", + } + result = Utils.format_issue(issue) + assert result == "[ERROR] file.csv: , Columns [header1, header2] No rows found" + + def test_format_issue_without_row_with_column(): issue = { "type": "error", "file": "schema.json", "column": ["header1", "header2"], - "message": "Duplicate columns" + "message": "Duplicate columns", } result = Utils.format_issue(issue) - assert result == "[ERROR] schema.json: Columns [header1, header2] Duplicate columns" + assert ( + result == "[ERROR] schema.json: , Columns [header1, header2] Duplicate columns" + ) + def test_fits_in_int32(): # Boundary values assert Utils.fits_in_int32(-2_147_483_648) is True assert Utils.fits_in_int32(2_147_483_647) is True - + # Internal values assert Utils.fits_in_int32(0) is True assert Utils.fits_in_int32(1000) is True From bffa917ffda73bb05914afa0e49cf686fa973113 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Tue, 24 Mar 2026 13:57:36 +0530 Subject: [PATCH 13/16] feat : adding tests to the data validator --- scripts/validator/tests/test_base_runner.py | 67 +++++++++++ scripts/validator/tests/test_tabular.py | 116 ++++++++++++++++++++ scripts/validator/validators/tabular.py | 5 +- 3 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 scripts/validator/tests/test_base_runner.py create mode 100644 scripts/validator/tests/test_tabular.py diff --git a/scripts/validator/tests/test_base_runner.py b/scripts/validator/tests/test_base_runner.py new file mode 100644 index 00000000..0485f76e --- /dev/null +++ b/scripts/validator/tests/test_base_runner.py @@ -0,0 +1,67 @@ +import pytest +from unittest.mock import patch, MagicMock +from core.baseRunner import run_validation + +class MockValidator: + def validate(self, path): + # We'll override this in specific tests + return [], [] + +@patch("core.baseRunner.Path") +def test_run_validation_no_files_found(mock_path_class, capsys): + # Setup mock Path.rglob to return an empty list + mock_path_instance = MagicMock() + mock_path_instance.rglob.return_value = [] + mock_path_class.return_value = mock_path_instance + + with pytest.raises(SystemExit) as excinfo: + run_validation("mock_dir", MockValidator) + + assert excinfo.value.code == 0 + captured = capsys.readouterr() + assert "[INFO] No data.json files found" in captured.out + +@patch("core.baseRunner.Path") +def test_run_validation_all_valid(mock_path_class, capsys): + # Setup mock paths to return one file + mock_path_instance = MagicMock() + # Need to return an interable for rglob + mock_path_instance.rglob.return_value = ["mock_data_1.json", "mock_data_2.json"] + mock_path_class.return_value = mock_path_instance + + class SuccessValidator(MockValidator): + def validate(self, path): + return [], [] + + with pytest.raises(SystemExit) as excinfo: + run_validation("mock_dir", SuccessValidator) + + assert excinfo.value.code == 0 + captured = capsys.readouterr() + assert "All data valid ✅" in captured.out + +@patch("core.baseRunner.Path") +def test_run_validation_with_errors_and_warnings(mock_path_class, capsys): + mock_path_instance = MagicMock() + mock_path_instance.rglob.return_value = ["mock_data.json"] + mock_path_class.return_value = mock_path_instance + + class FailureValidator(MockValidator): + def validate(self, path): + errors = [{ + "type": "error", "file": str(path), "row": 1, "column": "id", "message": "is wrong" + }] + warnings = [{ + "type": "warning", "file": str(path), "row": 2, "column": "name", "message": "is empty" + }] + return errors, warnings + + with pytest.raises(SystemExit) as excinfo: + run_validation("mock_dir", FailureValidator) + + assert excinfo.value.code == 1 + captured = capsys.readouterr() + assert "1 errors found" in captured.out + assert "1 warnings found" in captured.out + assert "[ERROR]" in captured.out + assert "[WARNING]" in captured.out diff --git a/scripts/validator/tests/test_tabular.py b/scripts/validator/tests/test_tabular.py new file mode 100644 index 00000000..9c83c218 --- /dev/null +++ b/scripts/validator/tests/test_tabular.py @@ -0,0 +1,116 @@ +import pytest +import json +from unittest.mock import patch, mock_open +from validators.tabular import TabularValidator + +@pytest.fixture +def tabular_validator(): + return TabularValidator() + +def test_check_duplicate_columns(tabular_validator): + # No duplicates + errors = tabular_validator._check_duplicate_columns("mock.json", ["id", "name", "age"]) + assert len(errors) == 0 + + # Duplicates exist + errors = tabular_validator._check_duplicate_columns("mock.json", ["id", "name", "id", "age", "name"]) + assert len(errors) == 1 + assert errors[0]["type"] == "error" + assert "Duplicate column names found" in errors[0]["message"] + assert set(errors[0]["column"]) == {"id", "name"} + +def test_check_row_column_mismatch(tabular_validator): + # Match + errors = tabular_validator._check_row_column_mismatch("mock.json", 0, [1, "test"], 2) + assert len(errors) == 0 + + # Mismatch + errors = tabular_validator._check_row_column_mismatch("mock.json", 1, [1, "test", "extra"], 2) + assert len(errors) == 1 + assert errors[0]["type"] == "error" + assert errors[0]["row"] == 1 + assert "has only 3 value(s), expected 2 value(s)" in errors[0]["message"] + +def test_check_data_types(tabular_validator): + columns = ["id", "score", "name"] + first_row = [1, 5.5, "Alice"] # int, float, str + + # Valid row matching first row types + row_valid = [2, 10.0, "Bob"] + errors = tabular_validator._check_data_types("mock.json", 1, row_valid, first_row, columns) + assert len(errors) == 0 + + # Allow int for float columns + row_valid_int_for_float = [3, 10, "Charlie"] + errors = tabular_validator._check_data_types("mock.json", 2, row_valid_int_for_float, first_row, columns) + assert len(errors) == 0 + + # Invalid row + row_invalid = ["four", "five", 6] + errors = tabular_validator._check_data_types("mock.json", 3, row_invalid, first_row, columns) + assert len(errors) == 3 + assert "expected int" in errors[0]["message"] + assert "expected float or whole number" in errors[1]["message"] + assert "expected str" in errors[2]["message"] + +def test_check_empty_values(tabular_validator): + columns = ["id", "name", "notes"] + row_empty = [1, "", None] + warnings = tabular_validator._check_empty_values("mock.json", 0, row_empty, columns) + assert len(warnings) == 2 + assert warnings[0]["column"] == "name" + assert warnings[0]["message"] == "has empty value" + assert warnings[1]["column"] == "notes" + assert warnings[1]["message"] == "has empty value" + +def test_check_value_overflow(tabular_validator): + columns = ["id", "big_num"] + row = [1, 2_147_483_648] # Over max int32 + warnings = tabular_validator._check_value_overflow("mock.json", 0, row, columns) + assert len(warnings) == 1 + assert warnings[0]["column"] == "big_num" + assert "this is a big integer in postgres" in warnings[0]["message"] + +@patch("validators.tabular.open") +def test_validate_invalid_json(mock_file_open, tabular_validator): + mock_file_open.side_effect = FileNotFoundError() + errors, warnings = tabular_validator.validate("missing.json") + assert len(errors) == 1 + assert len(warnings) == 0 + assert "Invalid JSON" in errors[0] + +@patch("validators.tabular.open", new_callable=mock_open, read_data='{"invalid": "schema"}') +def test_validate_schema_error(mock_file_open, tabular_validator): + errors, warnings = tabular_validator.validate("schema_error.json") + assert len(errors) == 1 + assert len(warnings) == 0 + assert "Schema error" in errors[0] + +@patch("validators.tabular.open", new_callable=mock_open) +def test_validate_success(mock_file_open, tabular_validator): + valid_data = { + "columns": ["id", "name"], + "rows": [ + [1, "Alice"], + [2, "Bob"] + ] + } + mock_file_open.return_value.read.return_value = json.dumps(valid_data) + errors, warnings = tabular_validator.validate("valid.json") + assert len(errors) == 0 + assert len(warnings) == 0 + +@patch("validators.tabular.open", new_callable=mock_open) +def test_validate_with_warnings_and_errors(mock_file_open, tabular_validator): + invalid_data = { + "columns": ["id", "id", "name"], + "rows": [ + [1, 1, "Alice"], + [2, "not-int", "Bob"] + ] + } + mock_file_open.return_value.read.return_value = json.dumps(invalid_data) + errors, warnings = tabular_validator.validate("invalid.json") + + # Should have duplicate column error, and type mismatch error for 'not-int' + assert len(errors) >= 2 diff --git a/scripts/validator/validators/tabular.py b/scripts/validator/validators/tabular.py index dd40071d..78c5b693 100644 --- a/scripts/validator/validators/tabular.py +++ b/scripts/validator/validators/tabular.py @@ -116,14 +116,13 @@ def validate(self, file_path): else: message = "No rows or columns found" - errors.extend([{ + errors.append({ "type": "error", "file": file_path, "row": rows if rows else None, "column": columns if columns else None, "message": message - } - ]) + }) return errors, warnings errors.extend(self._check_duplicate_columns(file_path, columns)) From 4b0c88b7e852174f212be15b22cc326fae65068f Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:10:33 +0530 Subject: [PATCH 14/16] review: resolving review comments by zaeema --- scripts/validator/README.md | 8 ++-- scripts/validator/core/baseRunner.py | 4 +- scripts/validator/tests/test_base_runner.py | 8 ++-- scripts/validator/tests/test_tabular.py | 16 ++++---- scripts/validator/tests/test_utils.py | 2 +- scripts/validator/utils/utils.py | 1 - scripts/validator/validators/tabular.py | 43 +++++++++++++-------- 7 files changed, 44 insertions(+), 38 deletions(-) diff --git a/scripts/validator/README.md b/scripts/validator/README.md index f58ab10c..d429b63c 100644 --- a/scripts/validator/README.md +++ b/scripts/validator/README.md @@ -43,11 +43,9 @@ python main.py ../../data/statistics tabular ## Supported Validators -| Validator | Description | Status | -|-----------|--------------------------|---------| -| `tabular` | Validates tabular data | ✅ Available | - -> **Note:** Additional validators are currently under development. +| Validator | Description | Validations | Status | +|-----------|------------------------|-------------------|-------------| +| `tabular` | Validates tabular data | `schema-validation` `duplicate-columns` `row-column-mismatch` `data-types-mismatch` `empty-values` `value-overflow` | ✅ Available | --- diff --git a/scripts/validator/core/baseRunner.py b/scripts/validator/core/baseRunner.py index 06db53c6..6dc80e2d 100644 --- a/scripts/validator/core/baseRunner.py +++ b/scripts/validator/core/baseRunner.py @@ -14,7 +14,7 @@ def run_validation(file_path, validator): sys.exit(0) for path in paths: - errors, warnings = validator.validate(path) + errors, warnings = validator.validate_data(path) all_errors.extend(errors) all_warnings.extend(warnings) @@ -29,6 +29,6 @@ def run_validation(file_path, validator): print(Utils.format_issue(warning)) if not all_errors and not all_warnings: - print("All data valid ✅") + print("All data is valid ✅") sys.exit(1 if all_errors else 0) diff --git a/scripts/validator/tests/test_base_runner.py b/scripts/validator/tests/test_base_runner.py index 0485f76e..717c60a9 100644 --- a/scripts/validator/tests/test_base_runner.py +++ b/scripts/validator/tests/test_base_runner.py @@ -3,7 +3,7 @@ from core.baseRunner import run_validation class MockValidator: - def validate(self, path): + def validate_data(self, path): # We'll override this in specific tests return [], [] @@ -23,7 +23,7 @@ def test_run_validation_no_files_found(mock_path_class, capsys): @patch("core.baseRunner.Path") def test_run_validation_all_valid(mock_path_class, capsys): - # Setup mock paths to return one file + # Setup mock paths to return two file mock_path_instance = MagicMock() # Need to return an interable for rglob mock_path_instance.rglob.return_value = ["mock_data_1.json", "mock_data_2.json"] @@ -38,7 +38,7 @@ def validate(self, path): assert excinfo.value.code == 0 captured = capsys.readouterr() - assert "All data valid ✅" in captured.out + assert "All data is valid ✅" in captured.out @patch("core.baseRunner.Path") def test_run_validation_with_errors_and_warnings(mock_path_class, capsys): @@ -47,7 +47,7 @@ def test_run_validation_with_errors_and_warnings(mock_path_class, capsys): mock_path_class.return_value = mock_path_instance class FailureValidator(MockValidator): - def validate(self, path): + def validate_data(self, path): errors = [{ "type": "error", "file": str(path), "row": 1, "column": "id", "message": "is wrong" }] diff --git a/scripts/validator/tests/test_tabular.py b/scripts/validator/tests/test_tabular.py index 9c83c218..24e39a74 100644 --- a/scripts/validator/tests/test_tabular.py +++ b/scripts/validator/tests/test_tabular.py @@ -29,7 +29,7 @@ def test_check_row_column_mismatch(tabular_validator): assert len(errors) == 1 assert errors[0]["type"] == "error" assert errors[0]["row"] == 1 - assert "has only 3 value(s), expected 2 value(s)" in errors[0]["message"] + assert "has 3 value(s), expected 2 value(s)" in errors[0]["message"] def test_check_data_types(tabular_validator): columns = ["id", "score", "name"] @@ -69,22 +69,22 @@ def test_check_value_overflow(tabular_validator): warnings = tabular_validator._check_value_overflow("mock.json", 0, row, columns) assert len(warnings) == 1 assert warnings[0]["column"] == "big_num" - assert "this is a big integer in postgres" in warnings[0]["message"] + assert "which is a BIGINT as it exceeds PostgreSQL's 32-bit integer limit." in warnings[0]["message"] @patch("validators.tabular.open") def test_validate_invalid_json(mock_file_open, tabular_validator): mock_file_open.side_effect = FileNotFoundError() - errors, warnings = tabular_validator.validate("missing.json") + errors, warnings = tabular_validator.validate_data("missing.json") assert len(errors) == 1 assert len(warnings) == 0 assert "Invalid JSON" in errors[0] @patch("validators.tabular.open", new_callable=mock_open, read_data='{"invalid": "schema"}') -def test_validate_schema_error(mock_file_open, tabular_validator): - errors, warnings = tabular_validator.validate("schema_error.json") +def test_validate_schema_error(_mock_file_open, tabular_validator): + errors, warnings = tabular_validator.validate_data("schema_error.json") assert len(errors) == 1 assert len(warnings) == 0 - assert "Schema error" in errors[0] + assert "Schema error → 'columns' is a required property" in errors[0]["message"] @patch("validators.tabular.open", new_callable=mock_open) def test_validate_success(mock_file_open, tabular_validator): @@ -96,7 +96,7 @@ def test_validate_success(mock_file_open, tabular_validator): ] } mock_file_open.return_value.read.return_value = json.dumps(valid_data) - errors, warnings = tabular_validator.validate("valid.json") + errors, warnings = tabular_validator.validate_data("valid.json") assert len(errors) == 0 assert len(warnings) == 0 @@ -110,7 +110,7 @@ def test_validate_with_warnings_and_errors(mock_file_open, tabular_validator): ] } mock_file_open.return_value.read.return_value = json.dumps(invalid_data) - errors, warnings = tabular_validator.validate("invalid.json") + errors, warnings = tabular_validator.validate_data("invalid.json") # Should have duplicate column error, and type mismatch error for 'not-int' assert len(errors) >= 2 diff --git a/scripts/validator/tests/test_utils.py b/scripts/validator/tests/test_utils.py index 3cf9aa95..17c4acae 100644 --- a/scripts/validator/tests/test_utils.py +++ b/scripts/validator/tests/test_utils.py @@ -38,7 +38,7 @@ def test_format_issue_without_row_and_column(): assert result == "[ERROR] config.json: File not found or unreadable" -def test_format_issue_without_rosw_and_column(): +def test_format_issue_without_rows_and_columns(): issue = { "type": "error", "file": "file.csv", diff --git a/scripts/validator/utils/utils.py b/scripts/validator/utils/utils.py index bdcac551..cd1c9b41 100644 --- a/scripts/validator/utils/utils.py +++ b/scripts/validator/utils/utils.py @@ -7,7 +7,6 @@ def format_issue(issue): location += f"Row {issue['row']}" if issue.get("column"): - # handle list of columns OR single column if isinstance(issue["column"], list): cols = ", ".join(issue["column"]) location += f", Columns [{cols}]" diff --git a/scripts/validator/validators/tabular.py b/scripts/validator/validators/tabular.py index 78c5b693..3a233188 100644 --- a/scripts/validator/validators/tabular.py +++ b/scripts/validator/validators/tabular.py @@ -26,18 +26,18 @@ def _check_duplicate_columns(self, file_path, columns): ] return [] - def _check_row_column_mismatch(self, file_path, i, row, num_cols): + def _check_row_column_mismatch(self, file_path, index, row, num_cols): if len(row) != num_cols: return [{ "type": "error", "file": file_path, - "row": i, + "row": index, "column": None, - "message": f"has only {len(row)} value(s), expected {num_cols} value(s)", + "message": f"has {len(row)} value(s), expected {num_cols} value(s)", }] return [] - def _check_data_types(self, file_path, i, row, first_row, columns): + def _check_data_types(self, file_path, index, row, first_row, columns): errors = [] for j, value in enumerate(row): expected_type = type(first_row[j]) @@ -52,13 +52,13 @@ def _check_data_types(self, file_path, i, row, first_row, columns): errors.append({ "type": "error", "file": file_path, - "row": i, + "row": index, "column": columns[j], "message": f"has {value} ({type(value).__name__}), expected {expected_msg}", }) return errors - def _check_empty_values(self, file_path, i, row, columns): + def _check_empty_values(self, file_path, index, row, columns): warnings = [] for j, value in enumerate(row): str_value = str(value).strip() if value is not None else "" @@ -66,13 +66,13 @@ def _check_empty_values(self, file_path, i, row, columns): warnings.append({ "type": "warning", "file": file_path, - "row": i, + "row": index, "column": columns[j], "message": "has empty value", }) return warnings - def _check_value_overflow(self, file_path, i, row, columns): + def _check_value_overflow(self, file_path, index, row, columns): warnings = [] for j, value in enumerate(row): if isinstance(value, int): @@ -80,13 +80,13 @@ def _check_value_overflow(self, file_path, i, row, columns): warnings.append({ "type": "warning", "file": file_path, - "row": i, + "row": index, "column": columns[j], - "message": f"has {value} ({type(value).__name__}), this is a big integer in postgres , consider when inserting into the database (postgres has 32 bit integer limit)", + "message": f"has {value} ({type(value).__name__}), which is a BIGINT as it exceeds PostgreSQL's 32-bit integer limit.", }) return warnings - def validate(self, file_path): + def validate_data(self, file_path): errors = [] warnings = [] @@ -99,9 +99,17 @@ def validate(self, file_path): # 1. Schema validation try: + # imported validate function from the jsonschema library validate(instance=data, schema=self.schema) except ValidationError as e: - return [f"[ERROR] {file_path}: Schema error → {e.message}"], [] + errors.append({ + "type": "error", + "file": file_path, + "row": None, + "column": None, + "message": f"Schema error → {e.message}" + }) + return errors, warnings # 2. Custom validation columns = data.get("columns", []) @@ -123,15 +131,16 @@ def validate(self, file_path): "column": columns if columns else None, "message": message }) + print(errors) return errors, warnings errors.extend(self._check_duplicate_columns(file_path, columns)) - for i, row in enumerate(rows): - errors.extend(self._check_row_column_mismatch(file_path, i, row, num_cols)) - warnings.extend(self._check_empty_values(file_path, i, row, columns)) - warnings.extend(self._check_value_overflow(file_path, i, row, columns)) - errors.extend(self._check_data_types(file_path, i, row, rows[0], columns)) + for index, row in enumerate(rows, start=1): + errors.extend(self._check_row_column_mismatch(file_path, index, row, num_cols)) + warnings.extend(self._check_empty_values(file_path, index, row, columns)) + warnings.extend(self._check_value_overflow(file_path, index, row, columns)) + errors.extend(self._check_data_types(file_path, index, row, rows[0], columns)) return errors, warnings From c0eefab27c286f1762e11cd917e190a8e4f376dc Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Fri, 27 Mar 2026 16:39:00 +0530 Subject: [PATCH 15/16] fix : update the README.md --- scripts/validator/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/validator/README.md b/scripts/validator/README.md index d429b63c..879b4f40 100644 --- a/scripts/validator/README.md +++ b/scripts/validator/README.md @@ -43,9 +43,9 @@ python main.py ../../data/statistics tabular ## Supported Validators -| Validator | Description | Validations | Status | -|-----------|------------------------|-------------------|-------------| -| `tabular` | Validates tabular data | `schema-validation` `duplicate-columns` `row-column-mismatch` `data-types-mismatch` `empty-values` `value-overflow` | ✅ Available | +| Validator | Description | Validations | Status | +|-----------|-------------|-------------|--------| +| `tabular` | Validates tabular data | `schema-validation` — Checks that the data conforms to a predefined schema (structures, types, constraints)

`duplicate-columns` — Detects columns that appear more than once

`row-column-mismatch` — Detects rows where the number of fields does not match the number of defined columns

`data-types-mismatch` — Identifies values that do not match the expected data type for their column (e.g. text in a numeric field)

`empty-values` — Flag cells that are null, blank, or contain only whitespace where a value is required

`value-overflow` — Catches values that exceed the maximum allowed length or numeric range for their column | ✅ Available | --- From 3e7569adc6188e90763b8678706268a160d1bff2 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Fri, 27 Mar 2026 17:24:24 +0530 Subject: [PATCH 16/16] fixing review comments --- .../workflows/dataset-validator[tabular].yml | 4 +-- scripts/validator/validators/tabular.py | 27 +++++++++---------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/dataset-validator[tabular].yml b/.github/workflows/dataset-validator[tabular].yml index 9db45a2f..143b88cf 100644 --- a/.github/workflows/dataset-validator[tabular].yml +++ b/.github/workflows/dataset-validator[tabular].yml @@ -5,7 +5,7 @@ on: branches: - main paths: - - 'data/statistics/**' + - 'data/**' workflow_dispatch: inputs: mode: @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest env: - DATASET_PATH: ${{ vars.DATASET_DIR_PATH_VAR || '../../data/statistics' }} + DATASET_PATH: ${{ vars.DATASET_DIR_PATH_VAR || '../../data' }} MODE: ${{ github.event.inputs.mode || 'tabular' }} steps: diff --git a/scripts/validator/validators/tabular.py b/scripts/validator/validators/tabular.py index 3a233188..336d78dc 100644 --- a/scripts/validator/validators/tabular.py +++ b/scripts/validator/validators/tabular.py @@ -26,18 +26,18 @@ def _check_duplicate_columns(self, file_path, columns): ] return [] - def _check_row_column_mismatch(self, file_path, index, row, num_cols): + def _check_row_column_mismatch(self, file_path, row_index, row, num_cols): if len(row) != num_cols: return [{ "type": "error", "file": file_path, - "row": index, + "row": row_index, "column": None, "message": f"has {len(row)} value(s), expected {num_cols} value(s)", }] return [] - def _check_data_types(self, file_path, index, row, first_row, columns): + def _check_data_types(self, file_path, row_index, row, first_row, columns): errors = [] for j, value in enumerate(row): expected_type = type(first_row[j]) @@ -52,13 +52,13 @@ def _check_data_types(self, file_path, index, row, first_row, columns): errors.append({ "type": "error", "file": file_path, - "row": index, + "row": row_index, "column": columns[j], "message": f"has {value} ({type(value).__name__}), expected {expected_msg}", }) return errors - def _check_empty_values(self, file_path, index, row, columns): + def _check_empty_values(self, file_path, row_index, row, columns): warnings = [] for j, value in enumerate(row): str_value = str(value).strip() if value is not None else "" @@ -66,13 +66,13 @@ def _check_empty_values(self, file_path, index, row, columns): warnings.append({ "type": "warning", "file": file_path, - "row": index, + "row": row_index, "column": columns[j], "message": "has empty value", }) return warnings - def _check_value_overflow(self, file_path, index, row, columns): + def _check_value_overflow(self, file_path, row_index, row, columns): warnings = [] for j, value in enumerate(row): if isinstance(value, int): @@ -80,7 +80,7 @@ def _check_value_overflow(self, file_path, index, row, columns): warnings.append({ "type": "warning", "file": file_path, - "row": index, + "row": row_index, "column": columns[j], "message": f"has {value} ({type(value).__name__}), which is a BIGINT as it exceeds PostgreSQL's 32-bit integer limit.", }) @@ -131,16 +131,15 @@ def validate_data(self, file_path): "column": columns if columns else None, "message": message }) - print(errors) return errors, warnings errors.extend(self._check_duplicate_columns(file_path, columns)) - for index, row in enumerate(rows, start=1): - errors.extend(self._check_row_column_mismatch(file_path, index, row, num_cols)) - warnings.extend(self._check_empty_values(file_path, index, row, columns)) - warnings.extend(self._check_value_overflow(file_path, index, row, columns)) - errors.extend(self._check_data_types(file_path, index, row, rows[0], columns)) + for row_index, row in enumerate(rows, start=1): + errors.extend(self._check_row_column_mismatch(file_path, row_index, row, num_cols)) + warnings.extend(self._check_empty_values(file_path, row_index, row, columns)) + warnings.extend(self._check_value_overflow(file_path, row_index, row, columns)) + errors.extend(self._check_data_types(file_path, row_index, row, rows[0], columns)) return errors, warnings