diff --git a/converters/semantido/README.md b/converters/semantido/README.md new file mode 100644 index 00000000..fb11b97b --- /dev/null +++ b/converters/semantido/README.md @@ -0,0 +1,86 @@ + + +# semantido ⇄ Apache Ossie converter + +Converts between [semantido](https://github.com/hikarilabs/semantido) — a +code-native semantic layer authored as decorators on SQLAlchemy models and +Apache Ossie semantic model documents. + +Unlike file-to-file converters, semantido's source of truth is live Python: +the forward direction imports a module of decorated models, syncs the +semantic layer, and emits Ossie YAML through the typed `apache-ossie` +objects. The reverse direction is code generation: it produces a Python +module of `@semantic_table`-decorated models from an Ossie document. + +## Usage + +```bash +# decorated SQLAlchemy models -> Ossie YAML +ossie-semantido semantido-to-osi \ + -m models.[SQLAlchemy model] -p ./my_project \ + -n [model name] -o [model].osi.yaml + +# Ossie YAML -> generated semantido model code +ossie-semantido osi-to-semantido \ + -i [model].osi.yaml -o generated_model_[model name].py +``` + +## Mapping + +| semantido | Ossie | Notes | +|-------------------------------------------|-----------------------------------|------------------------------------------------------| +| decorated table | `datasets[]` | `source` from schema-qualified table name | +| column | `fields[]` | expression emitted as `ANSI_SQL` column reference | +| `description` / `_description` | `description` | | +| `business_context`, `application_context` | dataset `ai_context.instructions` | | +| `synonyms` / `_synonyms` | `ai_context.synonyms` | | +| `_sample_values` | field `ai_context.examples` | | +| `time_dimension=` + `_time_grain` | field `dimension.is_time` | grain preserved in extensions | +| FK relationships | `relationships[]` | join condition parsed to `from_columns`/`to_columns` | +| `sql_filters` | dataset `custom_extensions` | no Ossie core field (see below) | +| `_privacy_level` | field `custom_extensions` | no Ossie core field (see below) | + +### Metadata carried in `custom_extensions` + +semantido captures runtime-governance metadata that has no core-spec home +in Ossie today: per-field privacy classification, default SQL filters / +row-level security fragments, and time-dimension grain. These are preserved +losslessly under `vendor_name: SEMANTIDO` with `data` as a serialized JSON +string, so no information is lost in interchange — but tools without +semantido awareness will not interpret them. + +### Known lossy conversions + +Conversions report structured `ConverterIssue`s (mirroring `ossie_dbt`): +Ossie metrics and non-ANSI dialect expressions have no semantido +equivalent and are dropped with a recorded issue; generated code lists +them in a TODO block for review. + +## Tests + +The test fixture is an EMIR (European Market Infrastructure Regulation) +trade-reporting model — a regulated-industry schema whose semantics +(role-bridge fan-out, signed vs. unsigned amount conventions, +notional/valuation ambiguity) demonstrate why interchange must carry +business meaning, not just structure. + +```bash +uv run pytest tests/ +``` diff --git a/converters/semantido/pyproject.toml b/converters/semantido/pyproject.toml new file mode 100644 index 00000000..a2d27d39 --- /dev/null +++ b/converters/semantido/pyproject.toml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "apache-ossie-semantido" +version = "0.1.0.dev0" +description = "semantido (SQLAlchemy code-native semantic layer) <> Apache Ossie converter" +requires-python = ">=3.11" +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", +] +dependencies = [ + "apache-ossie>=0.2.0.dev0", + "semantido>=0.4.1", + "SQLAlchemy>=2.0", + "PyYAML>=6.0", +] + +[project.license] +text = "Apache-2.0" + +[project.scripts] +ossie-semantido = "ossie_semantido.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_semantido"] + +[tool.uv] +dev-dependencies = [ + "pytest>=8.0", + "ruff==0.15.22" +] + +[tool.uv.sources] +apache-ossie = { path = "../../python", editable = true } diff --git a/converters/semantido/src/ossie_semantido/__init__.py b/converters/semantido/src/ossie_semantido/__init__.py new file mode 100644 index 00000000..1014e6f9 --- /dev/null +++ b/converters/semantido/src/ossie_semantido/__init__.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""semantido <> Apache Ossie converter.""" + +from ossie_semantido.converter_issues import ( + ConverterIssue, + ConverterIssueType, + ConverterResult, +) +from ossie_semantido.loaders import load_from_module +from ossie_semantido.osi_to_semantido import osi_to_semantido_source +from ossie_semantido.semantido_to_osi import semantic_layer_to_osi + +__all__ = [ + "ConverterIssue", + "ConverterIssueType", + "ConverterResult", + "load_from_module", + "osi_to_semantido_source", + "semantic_layer_to_osi", +] diff --git a/converters/semantido/src/ossie_semantido/cli.py b/converters/semantido/src/ossie_semantido/cli.py new file mode 100644 index 00000000..138cbd26 --- /dev/null +++ b/converters/semantido/src/ossie_semantido/cli.py @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command-line interface for the semantido <> Apache Ossie converter.""" + +import argparse +import sys + +import yaml +from ossie import OSIDocument + +from ossie_semantido.loaders import load_from_module +from ossie_semantido.osi_to_semantido import osi_to_semantido_source +from ossie_semantido.semantido_to_osi import semantic_layer_to_osi + + +def _report_issues(issues) -> None: + for issue in issues: + print( + f"warning: {issue.issue_type.value}: {issue.element_name}", file=sys.stderr + ) + + +def _cmd_semantido_to_osi(args: argparse.Namespace) -> None: + layer = load_from_module(args.module, sys_path=args.path) + result = semantic_layer_to_osi(layer, model_name=args.name) + _report_issues(result.issues) + with open(args.output, "w", encoding="utf-8") as handle: + handle.write(result.output.to_osi_yaml()) + + +def _cmd_osi_to_semantido(args: argparse.Namespace) -> None: + with open(args.input, "r", encoding="utf-8") as handle: + document = OSIDocument.model_validate(yaml.safe_load(handle)) + result = osi_to_semantido_source(document) + _report_issues(result.issues) + with open(args.output, "w", encoding="utf-8") as handle: + handle.write(result.output) + + +def main() -> None: + parser = argparse.ArgumentParser( + prog="ossie-semantido", + description="Convert between semantido semantic layers and Apache Ossie documents", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + to_osi = subparsers.add_parser( + "semantido-to-osi", help="Convert decorated SQLAlchemy models → Ossie YAML" + ) + to_osi.add_argument( + "-m", + "--module", + required=True, + metavar="MODULE", + help="Dotted module path of the semantido models, e.g. models.emir_reporting", + ) + to_osi.add_argument( + "-p", + "--path", + default=None, + metavar="DIR", + help="Directory prepended to sys.path before import", + ) + to_osi.add_argument( + "-n", + "--name", + required=True, + metavar="NAME", + help="Name for the Ossie semantic_model", + ) + to_osi.add_argument( + "-o", + "--output", + required=True, + metavar="FILE", + help="Path for output Ossie YAML", + ) + to_osi.set_defaults(func=_cmd_semantido_to_osi) + + from_osi = subparsers.add_parser( + "osi-to-semantido", help="Convert Ossie YAML → generated semantido model code" + ) + from_osi.add_argument( + "-i", "--input", required=True, metavar="FILE", help="Path to Ossie YAML" + ) + from_osi.add_argument( + "-o", + "--output", + required=True, + metavar="FILE", + help="Path for generated Python module", + ) + from_osi.set_defaults(func=_cmd_osi_to_semantido) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/converters/semantido/src/ossie_semantido/constants.py b/converters/semantido/src/ossie_semantido/constants.py new file mode 100644 index 00000000..5e07cbb1 --- /dev/null +++ b/converters/semantido/src/ossie_semantido/constants.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared constants for the ossie-semantido converter.""" + +VENDOR_NAME = "SEMANTIDO" diff --git a/converters/semantido/src/ossie_semantido/converter_issues.py b/converters/semantido/src/ossie_semantido/converter_issues.py new file mode 100644 index 00000000..e35fbfb2 --- /dev/null +++ b/converters/semantido/src/ossie_semantido/converter_issues.py @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Issue reporting for lossy conversions, mirroring the ossie_dbt pattern.""" + +from dataclasses import dataclass +from enum import Enum +from typing import Generic, List, TypeVar + + +class ConverterIssueType(Enum): + """Identifies the kind of information loss that occurred during conversion.""" + + RELATIONSHIP_JOIN_UNPARSED = "RELATIONSHIP_JOIN_UNPARSED" + METRIC_DROPPED = "METRIC_DROPPED" + NON_ANSI_EXPRESSION_DROPPED = "NON_ANSI_EXPRESSION_DROPPED" + COMPOSITE_PRIMARY_KEY_TRUNCATED = "COMPOSITE_PRIMARY_KEY_TRUNCATED" + UNMAPPED_DATA_TYPE = "UNMAPPED_DATA_TYPE" + + +@dataclass(frozen=True) +class ConverterIssue: + """Records a single instance of information loss during conversion.""" + + issue_type: ConverterIssueType + element_name: str + + +T = TypeVar("T") + + +@dataclass(frozen=True) +class ConverterResult(Generic[T]): + """Return value of a converter's convert() method, pairing the output with any conversion issues.""" + + output: T + issues: List[ConverterIssue] diff --git a/converters/semantido/src/ossie_semantido/loaders.py b/converters/semantido/src/ossie_semantido/loaders.py new file mode 100644 index 00000000..eba7c85a --- /dev/null +++ b/converters/semantido/src/ossie_semantido/loaders.py @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Entry points for loading a semantido SemanticLayer. + +semantido's source of truth is live Python (decorated SQLAlchemy models), +so the primary loader imports a module and syncs the layer. A file-based +path is not provided in this converter: semantido's JSON export is a +rendering of the same layer, and consumers who have the JSON already have +a semantic model they can transform directly. +""" + +import importlib +import sys +from pathlib import Path + +from semantido import SemanticDeclarativeBase +from semantido.generators.semantic_layer import SemanticLayer + + +def load_from_module(module_path: str, sys_path: str | None = None) -> SemanticLayer: + """Import a module of decorated models and return the synced layer. + + Args: + module_path: Dotted module path, e.g. ``models.emir_reporting``. + sys_path: Optional directory prepended to ``sys.path`` first, so + model packages can be loaded without installation. + """ + if sys_path: + sys.path.insert(0, str(Path(sys_path).resolve())) + importlib.import_module(module_path) + return SemanticDeclarativeBase.sync_semantic_layer() diff --git a/converters/semantido/src/ossie_semantido/osi_to_semantido.py b/converters/semantido/src/ossie_semantido/osi_to_semantido.py new file mode 100644 index 00000000..8aa9eb0c --- /dev/null +++ b/converters/semantido/src/ossie_semantido/osi_to_semantido.py @@ -0,0 +1,164 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Apache Ossie document -> generated semantido model code. + +This direction is code generation: it emits a Python module of +``@semantic_table``-decorated SQLAlchemy models. Constructs with no +semantido equivalent (metrics, non-ANSI dialect expressions) are dropped +with a recorded ConverterIssue and listed in a TODO comment block at the +top of the generated file. +""" + +import json +from typing import List + +from ossie import OSIDataset, OSIDocument, OSIField + +from ossie_semantido.constants import VENDOR_NAME +from ossie_semantido.converter_issues import ( + ConverterIssue, + ConverterIssueType, + ConverterResult, +) + +_TYPE_MAP = { + "VARCHAR": "String", + "CHAR": "String", + "TEXT": "String", + "INTEGER": "Integer", + "BIGINT": "Integer", + "SMALLINT": "Integer", + "NUMERIC": "Numeric", + "DECIMAL": "Numeric", + "FLOAT": "Float", + "DATE": "Date", + "DATETIME": "DateTime", + "TIMESTAMP": "DateTime", + "BOOLEAN": "Boolean", +} + + +def _vendor_payload(entity) -> dict: + for ext in entity.custom_extensions or []: + if ext.vendor_name == VENDOR_NAME: + try: + return json.loads(ext.data) + except (TypeError, ValueError): + return {} + return {} + + +def _sqlalchemy_type(field: OSIField, issues: List[ConverterIssue]) -> str: + data_type = _vendor_payload(field).get("data_type", "") + base = data_type.split("(")[0].strip().upper() + if base in _TYPE_MAP: + return _TYPE_MAP[base] + if base: + issues.append( + ConverterIssue( + ConverterIssueType.UNMAPPED_DATA_TYPE, f"{field.name}:{base}" + ) + ) + return "String" + + +def _ai(entity): + ctx = entity.ai_context + if ctx is None or isinstance(ctx, str): + return None + return ctx + + +def _class_name(dataset_name: str) -> str: + return "".join(part.capitalize() for part in dataset_name.split("_")) + + +def _emit_dataset(dataset: OSIDataset, issues: List[ConverterIssue]) -> str: + payload = _vendor_payload(dataset) + ctx = _ai(dataset) + + decorator_kwargs = [] + if dataset.description: + decorator_kwargs.append(f" description={dataset.description!r},") + if ctx and ctx.instructions: + decorator_kwargs.append(f" business_context={ctx.instructions!r},") + if ctx and ctx.synonyms: + decorator_kwargs.append(f" synonyms={list(ctx.synonyms)!r},") + if payload.get("sql_filters"): + decorator_kwargs.append(f" sql_filters={payload['sql_filters']!r},") + if payload.get("time_dimension"): + decorator_kwargs.append(f" time_dimension={payload['time_dimension']!r},") + + lines = ["@semantic_table("] + decorator_kwargs + [")"] + lines.append(f"class {_class_name(dataset.name)}(SemanticDeclarativeBase):") + lines.append(f' __tablename__ = "{dataset.name}"') + lines.append("") + + primary_keys = set(dataset.primary_key or []) + for field in dataset.fields or []: + field_payload = _vendor_payload(field) + field_ctx = _ai(field) + sa_type = _sqlalchemy_type(field, issues) + pk = ", primary_key=True" if field.name in primary_keys else "" + lines.append(f" {field.name} = Column({sa_type}{pk})") + if field.description: + lines.append(f" {field.name}_description = {field.description!r}") + if field_payload.get("privacy_level"): + level = field_payload["privacy_level"].upper() + lines.append(f" {field.name}_privacy_level = PrivacyLevel.{level}") + if field_ctx and field_ctx.examples: + lines.append( + f" {field.name}_sample_values = {[str(e) for e in field_ctx.examples]!r}" + ) + if field_payload.get("time_grain"): + grain = field_payload["time_grain"].upper() + lines.append(f" {field.name}_time_grain = TimeGrain.{grain}") + return "\n".join(lines) + + +def osi_to_semantido_source(document: OSIDocument) -> ConverterResult[str]: + """Generate semantido model source code from an Ossie document.""" + issues: List[ConverterIssue] = [] + blocks = [] + + for model in document.semantic_model: + for metric in model.metrics or []: + issues.append( + ConverterIssue(ConverterIssueType.METRIC_DROPPED, metric.name) + ) + for dataset in model.datasets: + blocks.append(_emit_dataset(dataset, issues)) + + todo_block = "" + if issues: + todo_lines = "\n".join( + f"# {i.issue_type.value}: {i.element_name}" for i in issues + ) + todo_block = ( + "# TODO(ossie-semantido): the following Ossie constructs have no\n" + "# semantido equivalent yet and were not converted:\n" + todo_lines + "\n\n" + ) + + header = ( + '"""semantido models generated by ossie-semantido. Review before use."""\n\n' + "from sqlalchemy import Boolean, Column, Date, DateTime, Float, Integer, Numeric, String\n\n" + "from semantido import SemanticDeclarativeBase, semantic_table\n" + "from semantido.generators.semantic_layer import PrivacyLevel, TimeGrain\n\n\n" + ) + source = header + todo_block + "\n\n\n".join(blocks) + "\n" + return ConverterResult(output=source, issues=issues) diff --git a/converters/semantido/src/ossie_semantido/semantido_to_osi.py b/converters/semantido/src/ossie_semantido/semantido_to_osi.py new file mode 100644 index 00000000..9cf55689 --- /dev/null +++ b/converters/semantido/src/ossie_semantido/semantido_to_osi.py @@ -0,0 +1,218 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""semantido SemanticLayer -> Apache Ossie document conversion. + +Builds the Ossie model through the typed ``apache-ossie`` objects, so the +output is schema-conformant by construction. Governance metadata that has +no Ossie core field (privacy levels, SQL filters, time grain, the primary +time axis) is preserved losslessly in ``custom_extensions`` under the +``SEMANTIDO`` vendor name, with ``data`` as a serialized JSON string as +the specification requires. +""" + +import json +import re +from typing import List, Optional + +from ossie import ( + OSIAIContextObject, + OSICustomExtension, + OSIDataset, + OSIDialect, + OSIDialectExpression, + OSIDimension, + OSIDocument, + OSIExpression, + OSIField, + OSIRelationship, + OSISemanticModel, +) +from semantido.generators.semantic_layer import ( + Column, + Relationship, + SemanticLayer, + Table, +) + +from ossie_semantido.constants import VENDOR_NAME +from ossie_semantido.converter_issues import ( + ConverterIssue, + ConverterIssueType, + ConverterResult, +) + +_JOIN_RE = re.compile(r"^\s*(\w+)\.(\w+)\s*=\s*(\w+)\.(\w+)\s*$") + + +def _extension(payload: dict) -> OSICustomExtension: + """Wrap semantido-specific metadata as a spec-conformant custom extension.""" + return OSICustomExtension( + vendor_name=VENDOR_NAME, + data=json.dumps(payload, sort_keys=True, default=str), + ) + + +def _ansi(expression: str) -> OSIExpression: + return OSIExpression( + dialects=[ + OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression=expression) + ] + ) + + +def _column_to_field(column: Column) -> OSIField: + ai_kwargs = {} + if column.synonyms: + ai_kwargs["synonyms"] = tuple(column.synonyms) + if column.sample_values: + ai_kwargs["examples"] = tuple(str(v) for v in column.sample_values) + if column.application_rules: + ai_kwargs["instructions"] = " ".join(column.application_rules) + + extension_payload = {} + if column.privacy_level is not None: + extension_payload["privacy_level"] = column.privacy_level.value + if column.time_grain is not None: + extension_payload["time_grain"] = column.time_grain.value + if column.data_type: + extension_payload["data_type"] = column.data_type + if column.references: + extension_payload["references"] = column.references + + return OSIField( + name=column.name, + expression=_ansi(column.name), + dimension=OSIDimension(is_time=True) if column.is_time_dimension else None, + description=column.description or None, + ai_context=OSIAIContextObject(**ai_kwargs) if ai_kwargs else None, + custom_extensions=[_extension(extension_payload)] + if extension_payload + else None, + ) + + +def _table_to_dataset(table: Table) -> OSIDataset: + instructions_parts = [ + p for p in (table.business_context, table.application_context) if p + ] + + ai_kwargs = {} + if instructions_parts: + ai_kwargs["instructions"] = " ".join(instructions_parts) + if table.synonyms: + ai_kwargs["synonyms"] = tuple(table.synonyms) + + extension_payload = {} + if table.sql_filters: + extension_payload["sql_filters"] = table.sql_filters + if table.time_dimension: + extension_payload["time_dimension"] = table.time_dimension + + return OSIDataset( + name=table.name, + source=f"{table.schema}.{table.name}" if table.schema else table.name, + primary_key=table.primary_key if table.primary_key else None, + description=table.description or None, + ai_context=OSIAIContextObject(**ai_kwargs) if ai_kwargs else None, + fields=[_column_to_field(c) for c in table.columns] or None, + custom_extensions=[_extension(extension_payload)] + if extension_payload + else None, + ) + + +def _relationship_to_osi( + rel: Relationship, issues: List[ConverterIssue] +) -> Optional[OSIRelationship]: + match = _JOIN_RE.match(rel.join_condition or "") + if not match: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.RELATIONSHIP_JOIN_UNPARSED, + element_name=f"{rel.from_table}->{rel.to_table}", + ) + ) + return None + + left_table, left_col, right_table, right_col = match.groups() + # Normalize so from/to match the declared direction regardless of + # which side of the equality each table appeared on. + if left_table == rel.from_table: + from_columns, to_columns = [left_col], [right_col] + elif right_table == rel.from_table: + from_columns, to_columns = [right_col], [left_col] + else: + # Neither side of the join references rel.from_table (e.g., aliases) + # Treat this as unparseable rather than silently swapping columns. + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.RELATIONSHIP_JOIN_UNPARSED, + element_name=f"{rel.from_table}->{rel.to_table}", + ) + ) + return None + + ai_kwargs = {} + if rel.description: + ai_kwargs["instructions"] = rel.description + + return OSIRelationship.model_validate( + { + "name": f"{rel.from_table}_to_{rel.to_table}", + "from": rel.from_table, + "to": rel.to_table, + "from_columns": from_columns, + "to_columns": to_columns, + "ai_context": ai_kwargs or None, + "custom_extensions": [ + _extension( + {"relationship_type": rel.relationship_type.value} + ).model_dump() + ], + } + ) + + +def semantic_layer_to_osi( + layer: SemanticLayer, model_name: str, description: Optional[str] = None +) -> ConverterResult[OSIDocument]: + """Convert a synced semantido SemanticLayer into an Ossie document.""" + issues: List[ConverterIssue] = [] + + datasets = [_table_to_dataset(t) for t in layer.tables.values()] + + relationships = [] + for rel in layer.relationships: + converted = _relationship_to_osi(rel, issues) + if converted is not None: + relationships.append(converted) + + model_extensions = None + if layer.application_glossary: + model_extensions = [ + _extension({"application_glossary": layer.application_glossary}) + ] + + model = OSISemanticModel( + name=model_name, + description=description, + datasets=datasets, + relationships=relationships or None, + custom_extensions=model_extensions, + ) + return ConverterResult(output=OSIDocument(semantic_model=[model]), issues=issues) diff --git a/converters/semantido/tests/__init__.py b/converters/semantido/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/semantido/tests/fixtures/__init__.py b/converters/semantido/tests/fixtures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/semantido/tests/fixtures/emir_reporting/__init__.py b/converters/semantido/tests/fixtures/emir_reporting/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/semantido/tests/fixtures/emir_reporting/models/__init__.py b/converters/semantido/tests/fixtures/emir_reporting/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/semantido/tests/fixtures/emir_reporting/models/emir_reporting.py b/converters/semantido/tests/fixtures/emir_reporting/models/emir_reporting.py new file mode 100644 index 00000000..2485e3af --- /dev/null +++ b/converters/semantido/tests/fixtures/emir_reporting/models/emir_reporting.py @@ -0,0 +1,226 @@ +"""EMIR trade-reporting semantic model (regulated-industry fixture). + +Simplified but structurally honest EMIR REFIT reporting schema. It +deliberately encodes three semantics-dependent failure modes that plain +DDL cannot express: bridge fan-out via the counterparty role table, sign +conventions (signed valuations vs. unsigned posted/received collateral), +and amount ambiguity (notional vs. valuation vs. collateral). +""" + +from sqlalchemy import Column, Date, DateTime, ForeignKey, Integer, Numeric, String +from sqlalchemy.orm import relationship + +from semantido import SemanticDeclarativeBase, semantic_table +from semantido.generators.semantic_layer import PrivacyLevel, TimeGrain + + +@semantic_table( + description="Legal entity master for derivative counterparties, one row per LEI.", + business_context=( + "EMIR counterparty classification drives clearing and reporting " + "obligations (FC / NFC+ / NFC- under Art. 2 and Art. 10)." + ), + synonyms=["legal entities", "counterparties"], +) +class Counterparty(SemanticDeclarativeBase): + __tablename__ = "counterparty" + + lei = Column(String(20), primary_key=True) + lei_description = ( + "Legal Entity Identifier (ISO 17442). The sole join key for " + "counterparty identity everywhere in this schema." + ) + lei_sample_values = ["529900T8BM49AURSDO55"] + + legal_name = Column(String(200), nullable=False) + legal_name_description = "Registered legal name as per GLEIF record." + legal_name_privacy_level = PrivacyLevel.CONFIDENTIAL + + classification = Column(String(4), nullable=False) + classification_description = ( + "EMIR counterparty classification: 'FC' (financial counterparty, Art. 2(8)), " + "'NFC+' (non-financial above clearing threshold, Art. 10), 'NFC-' (below). " + "Filter on this before any obligation question." + ) + classification_sample_values = ["FC", "NFC+"] + + country = Column(String(2), nullable=False) + country_description = "ISO 3166-1 alpha-2 country of incorporation." + + roles = relationship("CounterpartyTradeRole", back_populates="counterparty") + roles_relationship_description = "Roles this entity plays on trades." + + +@semantic_table( + description=( + "Reportable OTC derivative trades, one row per UTI. Immutable birth " + "record of the trade; evolving economics live in trade_state." + ), + business_context=( + "AMBIGUITY GUARD: notional_amount is contract size, NOT current value " + "and NOT exposure. 'How big' -> notional; 'worth today' -> " + "trade_state.valuation_amount; 'at risk' -> collateral_report." + ), + synonyms=["trades", "derivatives", "positions"], + time_dimension="execution_timestamp", +) +class Trade(SemanticDeclarativeBase): + __tablename__ = "trade" + + uti = Column(String(52), primary_key=True) + uti_description = "Unique Trade Identifier (Art. 9 REFIT). Canonical trade key." + + asset_class = Column(String(4), nullable=False) + asset_class_description = ( + "EMIR asset class: 'IR' rates, 'CR' credit, 'EQ' equity, 'FX' FX, 'CO' commodity." + ) + asset_class_sample_values = ["IR", "FX"] + + notional_amount = Column(Numeric(20, 2), nullable=False) + notional_amount_description = ( + "Trade notional in notional_currency. Contract size, not value. Never " + "sum across currencies without conversion." + ) + notional_amount_privacy_level = PrivacyLevel.RESTRICTED + + notional_currency = Column(String(3), nullable=False) + notional_currency_description = "ISO 4217 currency of notional_amount." + notional_currency_sample_values = ["EUR", "USD"] + + execution_timestamp = Column(DateTime, nullable=False) + execution_timestamp_description = "UTC execution time (Art. 9 REFIT). Primary time axis." + execution_timestamp_time_grain = TimeGrain.SECOND + + states = relationship("TradeState", back_populates="trade") + states_relationship_description = "Daily state reports for this trade." + roles = relationship("CounterpartyTradeRole", back_populates="trade") + roles_relationship_description = "Counterparty roles attached to this trade." + + +@semantic_table( + description=( + "Daily trade-state reports (EMIR REFIT is state-based), one row per " + "UTI per reporting_date. The latest state per UTI is the current view." + ), + business_context=( + "SIGN CONVENTION: valuation_amount is signed from the REPORTING " + "counterparty's perspective. Aggregating across counterparties without " + "flipping signs by role is meaningless." + ), + sql_filters=[ + "reporting_date = (SELECT MAX(ts2.reporting_date) FROM trade_state ts2 " + "WHERE ts2.uti = trade_state.uti)" + ], + time_dimension="reporting_date", +) +class TradeState(SemanticDeclarativeBase): + __tablename__ = "trade_state" + + id = Column(Integer, primary_key=True) + uti = Column(String(52), ForeignKey("trade.uti"), nullable=False) + reporting_date = Column(Date, nullable=False) + reporting_date_description = "State date; one state per trade per day." + reporting_date_time_grain = TimeGrain.DAY + + valuation_amount = Column(Numeric(20, 2)) + valuation_amount_description = ( + "Mark-to-market value (Art. 11(2)), signed from the reporting " + "counterparty's perspective. Value, not size: do not confuse with notional." + ) + valuation_amount_privacy_level = PrivacyLevel.RESTRICTED + + contract_status = Column(String(4), nullable=False) + contract_status_description = ( + "'OUTS' outstanding, 'TERM' terminated, 'MATU' matured, 'ERRO' error. " + "Exclude non-OUTS from exposure aggregates unless asked otherwise." + ) + contract_status_sample_values = ["OUTS", "TERM"] + + trade = relationship("Trade", back_populates="states") + trade_relationship_description = "The trade this state report belongs to." + + +@semantic_table( + description=( + "Role bridge between trades and counterparties. A trade has AT LEAST " + "two rows here (reporting + other counterparty), often more." + ), + business_context=( + "FAN-OUT GUARD: joining trade -> this table -> counterparty multiplies " + "trade rows by role count; any SUM over trade amounts after this join " + "double-counts. Filter to a single role (usually 'RPTG') before " + "aggregating trade-level amounts." + ), +) +class CounterpartyTradeRole(SemanticDeclarativeBase): + __tablename__ = "counterparty_trade_role" + + id = Column(Integer, primary_key=True) + uti = Column(String(52), ForeignKey("trade.uti"), nullable=False) + lei = Column(String(20), ForeignKey("counterparty.lei"), nullable=False) + + role = Column(String(4), nullable=False) + role_description = ( + "'RPTG' reporting counterparty, 'OTHR' other counterparty, 'BRKR' " + "broker, 'CLRM' clearing member. Exactly one RPTG row per UTI." + ) + role_sample_values = ["RPTG", "OTHR"] + + trade = relationship("Trade", back_populates="roles") + trade_relationship_description = "Trade this role row belongs to." + counterparty = relationship("Counterparty", back_populates="roles") + counterparty_relationship_description = "Entity playing this role." + + +@semantic_table( + description="Daily collateral/margin per UTI (Art. 11(3)).", + business_context=( + "SIGN CONVENTION (different from trade_state!): amounts are UNSIGNED; " + "direction is carried by which column they sit in. Net collateral = " + "posted - received, computed, never read from a column." + ), + time_dimension="reporting_date", +) +class CollateralReport(SemanticDeclarativeBase): + __tablename__ = "collateral_report" + + id = Column(Integer, primary_key=True) + uti = Column(String(52), ForeignKey("trade.uti"), nullable=False) + reporting_date = Column(Date, nullable=False) + reporting_date_time_grain = TimeGrain.DAY + + initial_margin_posted = Column(Numeric(20, 2)) + initial_margin_posted_description = "IM posted BY the reporting counterparty. Unsigned." + initial_margin_received = Column(Numeric(20, 2)) + initial_margin_received_description = ( + "IM received. Unsigned; do not sum with posted — subtract." + ) + collateral_currency = Column(String(3)) + collateral_currency_description = "ISO 4217 currency of margin amounts." + + +@semantic_table( + description=( + "Trade repository submission log. Grain: one row per submission " + "attempt — a trade may have many." + ), + business_context=( + "Rejection-rate questions resolve here, per attempt, not per trade, " + "unless deduplicated. Rejection rate = NACK / all attempts." + ), + time_dimension="submission_timestamp", +) +class SubmissionLog(SemanticDeclarativeBase): + __tablename__ = "submission_log" + + id = Column(Integer, primary_key=True) + uti = Column(String(52), ForeignKey("trade.uti"), nullable=False) + submission_timestamp = Column(DateTime, nullable=False) + submission_timestamp_time_grain = TimeGrain.SECOND + + status = Column(String(4), nullable=False) + status_description = "'ACKD' accepted, 'NACK' rejected, 'PEND' pending." + status_sample_values = ["ACKD", "NACK"] + + nack_reason = Column(String(200)) + nack_reason_description = "TR rejection reason text; NULL unless NACK." diff --git a/converters/semantido/tests/helpers.py b/converters/semantido/tests/helpers.py new file mode 100644 index 00000000..d7093dce --- /dev/null +++ b/converters/semantido/tests/helpers.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared test helpers.""" + +from pathlib import Path + +from ossie_semantido.loaders import load_from_module + +FIXTURES = Path(__file__).parent / "fixtures" + + +def load_emir_layer(): + return load_from_module( + "models.emir_reporting", sys_path=str(FIXTURES / "emir_reporting") + ) diff --git a/converters/semantido/tests/test_osi_to_semantido.py b/converters/semantido/tests/test_osi_to_semantido.py new file mode 100644 index 00000000..b03f8ad1 --- /dev/null +++ b/converters/semantido/tests/test_osi_to_semantido.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from ossie_semantido.osi_to_semantido import osi_to_semantido_source +from ossie_semantido.semantido_to_osi import semantic_layer_to_osi +from tests.helpers import load_emir_layer + + +def test_generated_source_compiles(): + layer = load_emir_layer() + document = semantic_layer_to_osi(layer, model_name="emir_reporting").output + result = osi_to_semantido_source(document) + compile(result.output, "", "exec") + + +def test_roundtrip_preserves_governance_annotations(): + layer = load_emir_layer() + document = semantic_layer_to_osi(layer, model_name="emir_reporting").output + source = osi_to_semantido_source(document).output + assert "sql_filters=" in source + assert "PrivacyLevel.CONFIDENTIAL" in source + assert 'time_dimension="reporting_date"' in source or "time_dimension='reporting_date'" in source + assert "TimeGrain.DAY" in source diff --git a/converters/semantido/tests/test_semantido_to_osi.py b/converters/semantido/tests/test_semantido_to_osi.py new file mode 100644 index 00000000..4b65a666 --- /dev/null +++ b/converters/semantido/tests/test_semantido_to_osi.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +from ossie import OSIDocument + +from ossie_semantido.semantido_to_osi import VENDOR_NAME, semantic_layer_to_osi +from tests.helpers import load_emir_layer + + +def _vendor_data(entity): + for ext in entity.custom_extensions or []: + if ext.vendor_name == VENDOR_NAME: + return json.loads(ext.data) + return {} + + +def test_converts_all_tables_to_datasets(): + layer = load_emir_layer() + result = semantic_layer_to_osi(layer, model_name="emir_reporting") + model = result.output.semantic_model[0] + assert {d.name for d in model.datasets} == set(layer.tables) + + +def test_relationships_are_parsed_not_dropped(): + layer = load_emir_layer() + result = semantic_layer_to_osi(layer, model_name="emir_reporting") + model = result.output.semantic_model[0] + assert model.relationships, "expected FK relationships in the fixture" + assert not result.issues, f"unexpected conversion issues: {result.issues}" + rel = model.relationships[0] + assert rel.from_columns and rel.to_columns + + +def test_governance_metadata_preserved_in_extensions(): + layer = load_emir_layer() + model = semantic_layer_to_osi(layer, model_name="emir_reporting").output.semantic_model[0] + + trade_state = next(d for d in model.datasets if d.name == "trade_state") + assert _vendor_data(trade_state).get("sql_filters"), "sql_filters must survive" + assert _vendor_data(trade_state).get("time_dimension") == "reporting_date" + + counterparty = next(d for d in model.datasets if d.name == "counterparty") + legal_name = next(f for f in counterparty.fields if f.name == "legal_name") + assert _vendor_data(legal_name).get("privacy_level") == "confidential" + + +def test_time_dimension_marked_on_field(): + layer = load_emir_layer() + model = semantic_layer_to_osi(layer, model_name="emir_reporting").output.semantic_model[0] + trade = next(d for d in model.datasets if d.name == "trade") + ts_field = next(f for f in trade.fields if f.name == "execution_timestamp") + assert ts_field.dimension is not None and ts_field.dimension.is_time is True + + +def test_yaml_output_is_valid_document(): + layer = load_emir_layer() + yaml_text = semantic_layer_to_osi(layer, model_name="emir_reporting").output.to_osi_yaml() + import yaml as pyyaml + + OSIDocument.model_validate(pyyaml.safe_load(yaml_text))