From 93efb8fb736fe8758bbcf801401ba5971cde4703 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Fri, 8 May 2026 13:46:42 -0400 Subject: [PATCH 01/89] First draft adding support for ontologies to the core spec. --- core-spec/spec.md | 247 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 232 insertions(+), 15 deletions(-) diff --git a/core-spec/spec.md b/core-spec/spec.md index 3798251..1765927 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -13,10 +13,12 @@ 1. [Enumerations](#enumerations) 2. [Semantic Model](#semantic-model) 3. [Datasets](#datasets) -4. [Relationships](#relationships) +4. [Join paths](#join-paths) 5. [Fields](#fields) 6. [Metrics](#metrics) -7. [Examples](#examples) +7. [Concepts](#concepts) +8. [Mappings](#mappings) +9. [Complete Example](#complete-example) --- @@ -37,6 +39,15 @@ Supported SQL and expression language dialects for metrics and field definitions | `DATABRICKS` | Databricks SQL | | `MAQL` | GoodData MAQL (Metric Analysis and Query Language) | +### Multiplicities + +The allowable multiplicities of relationships defined in the [Concepts](#concepts) section. + +| Multiplicity | Description | +|---------|-------------| +| `ManyToOne` | The last role of a relationship is uniquely determined by the other roles | +| `OneToOne` | The relationship is ManyToMany in both directions (only for binary relationships) | + ### Vendors Supported vendors for custom extensions and integrations. @@ -52,7 +63,7 @@ Supported vendors for custom extensions and integrations. ## Semantic Model -The top-level container that represents a complete semantic model, including datasets, relationships, and metrics. +The top-level container that represents a complete semantic model, including datasets, join paths, and metrics. ### Schema @@ -62,7 +73,7 @@ The top-level container that represents a complete semantic model, including dat | `description` | string | No | Human-readable description | | `ai_context` | string/object | No | Additional context for AI tools (e.g., custom instructions) | | `datasets` | array | Yes | Collection of logical datasets (fact and dimension tables) | -| `relationships` | array | No | Defines how logical datasets are connected | +| `join_paths` | array | No | Defines how logical datasets are connected | | `metrics` | array | No | Quantifiable measures defined as aggregate expessions on fields from logical datsets | | `custom_extensions` | array | No | Vendor-specific attributes for extensibility | @@ -75,7 +86,7 @@ semantic_model: ai_context: instructions: "Use this model for sales analysis and customer insights" datasets: [] - relationships: [] + join_paths: [] metrics: [] custom_extensions: - vendor_name: DBT @@ -143,17 +154,17 @@ datasets: --- -## Relationships +## Join paths -Relationships define how logical datasets are connected through foreign key constraints. They support both simple and composite keys. +Join paths define how logical datasets are connected through foreign key constraints. They support both simple and composite keys. ### Schema | Field | Type | Required | Description | |-------|------|----------|-------------| -| `name` | string | Yes | Unique identifier for the relationship | -| `from` | string | Yes | The logical dataset on the many side of the relationship | -| `to` | string | Yes | The logical dataset on the one side of the relationship | +| `name` | string | Yes | Unique identifier for the join path | +| `from` | string | Yes | The logical dataset on the many side of the join path | +| `to` | string | Yes | The logical dataset on the one side of the join path | | `from_columns` | array | Yes | Array of column names in the "from" dataset (foreign key columns) | | `to_columns` | array | Yes | Array of column names in the "to" dataset (primary or unique key columns) | | `ai_context` | string/object | No | Additional context for AI tools | @@ -163,12 +174,12 @@ Relationships define how logical datasets are connected through foreign key cons - The order of columns in `from_columns` must correspond to the order in `to_columns` - Both arrays must have the same number of columns -- For simple relationships, use a single column: `[column1]` -- For composite relationships, use multiple columns: `[column1, column2]` +- For simple join paths, use a single column: `[column1]` +- For composite join paths, use multiple columns: `[column1, column2]` ### Examples -**Simple Relationship:** +**Simple Join Path:** ```yaml - name: orders_to_customers @@ -178,7 +189,7 @@ Relationships define how logical datasets are connected through foreign key cons to_columns: [id] ``` -**Composite Relationship:** +**Composite Join Path:** ```yaml # order_lines.product_id = products.id AND order_lines.variant_id = products.variant_id @@ -412,6 +423,206 @@ custom_extensions: --- +## Concepts + +Concepts represent the types of things that have meaning in a business setting, e.g., person, company, or +salary. Each concept is either an entity type or a value type. Every ontology starts with a value-type +concept for each basic data type, like `Integer`, `Decimal`, and `String`, and an entity-type concept +called `Any`. Every other concept in the ontology extends one of these starter concepts. + +### Schema + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Unique identifier for this concept | +| `description` | string | No | Human-readable description | +| `relationships` | object | No | Relationships where this concept plays the first role | +| `extends` | list | No | Names of this concept's supertypes | +| `derived_by` | list | No | Expressions for deriving this concept's population | +| `identify_by` | list | No | Names of relationships that collectively identify this concept | +| `requires` | list | No | Expressions that constrain this concept's population | + +### Extends + +Every concept that a user declares extends one or more concepts in the ontology. The new concept +is a sutype of each concept that it extends, and the extended concepts are its supertypes. For instance,`SocialSecurityNr` extends `Integer`, `Person` extends `Any`, and `Employee` extends `Person`. Any +concept that directly or indirectly extends a value type like `Integer` or `String` is a value type. +A concept is an entity type if it is not a value type. Any concept with an empty extends list defaults +to being a subtype of `Any` and so is an entity type. + +### Relationships + +Relationships relate objects of one or more concepts and declare how to verbalize links among those objects. +For instance, a relationship named `earns` links persons to salaries. Each link pairs some `Person` with +some `Salary` and verbalizes as, “Person earns Salary annually.” Relationships have set (as opposed to bag) +semantics, and links do not contain nulls. + +#### Schema + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Part of the identifier for this relationship | +| `description` | string | No | Human-readable description | +| `multiplicity` | enum | No | Multiplicity constraint | +| `roles` | list | No | List of additional roles in this relationship | +| `derived_by` | list | No | Expressions that derive links of this relationship | +| `requires` | list | No | Expressions that constrain this relationship's population | +| `verbalization` | list | Yes | Patterns describing how to verbalize links | + +Each relationship is uniquely identified by a prepending its declared name with that of the containing +concept. For instance, a relationship named `earns`, declared within the `Person` concept is identified +by the string `Person.earns`. This convention naturally supports expressions that navigate over the links +of relationships using the “dot-join” operator in a manner that is familiar to object-oriented programming +languages. + +#### Multiplicities + +In a relationship that comprises more than one role, the last role might be functionally dermined +by the other roles. This is declared by providing a ManyToOne multiplicity on that relationship. +For relationships of ternary and higher arity, the multiplicity applies to the n-th role, meaning +the object that plays the n-th role is functionally determined by the tuple of objects that play +the first n-1 roles. A relationship like `Item.total_sales_in`, which records the total sales amount +of an item at a given store, will have a ManyToOne multiplicity to indicate that for any pair +of `Item` and `Store` at most one `Amount` will be recorded as the total sales amount for that +`Item` and that `Store`. + +In the special case of a binary relationship, one might declare a OneToOne multiplicity, which +indicates the relationship is ManyToOne in both directions. For instance, the `Person.ssn` +relationship will have a OneToOne multiplicity because each person is assigned at most one social +security number and each social security number is assigned to at most one person. + +In the absence of any multiplicity, we make no assumptions of functional dependencies among +any of the roles. + +#### Roles + +Objects play roles in the links of a relationship. If you think of a relationship as a narrow table, +then links are its rows and roles are its columns. Each role is played by some concept, which +indicates the type of objects that can play that role in the relationship's links. In the +`Person.earns` relationship, `Person` and `Salary` play the first and second roles respectively. + +By convention, the first role of any relationship will be played by the concept under which the +relationship is declared. Any additional roles are declared in the roles field as follows: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `player` | string | Yes | Name of the concept that plays this role | +| `name` | string | No | Optional role name | + +A unary relationship like `Person.files_married_joint` declares no additional roles, +while a ternary relationship like `Person.purchased_on` declares two additional roles +played by `Vehicle` and `Date` respectively. + +The role player often suffices to distinguish the role within a given relationship. +However, the same concept can play multiple roles, such as in the ternary relationship +`Store.ships_to_in_days` that connects pairs of `Store` objects to the number of days +required to ship inventory from one store to another. When this happens, the user must +declare a distinguising name for any additional role whose player does not suffice to +distinguish it from other roles in the same relationship. + +### Identifying relationships + +Many conceptual models distinguish one or more relationships that should be used when +referecing entity-type objects in expressions and queries. For instance, the relationship +`Person.ssn` can be used to reference a person by their social security number; while +the pair of relationships `License.acct` and `License.seat_nr` can be used to reference +a license by the account and seat number. These relationships are always binary, and +their first roles are always played by the concept the relationship is used to reference. +When a set of identifying relationships is known for a concept, this knowledge can be +expressed using the identify_by list. + +### Derivation expressions + +Concepts and relationships may be derived using expressions. A derived relationship is one +whose links are derived from those of other relationships. For instance: + +```yaml +- name: Person + relationships: + parent_of: + roles: + - player: Person + name: "child" + verbalizes: [ "{Person} is a parent of {Person:child}", "{Person:child} is a child of {Person}" ] + ancestor_of: + roles: + - player: Person + name: "descendant" + derived_by: + - "Person.parent_of(descendant)" + "Person.ancestor_of.parent_of(descendant)" + taxed_at: + roles: + - player: TaxRate + derived_by: + - "10.0 WHERE ( Person.files_single AND Person.earns <= 11925 )" + - "10.0 WHERE ( Person.files_married_joint AND Person.earns <= 23850 )" + - ... +``` + +derives a link of `Person.taxed_at` for each object of the first role player (Person) +using expressions that return a TaxRate based on the person's filing status and how +much they earn. If, for some person, none of the expressions can be evaluated, then +the relationship will have no link involving that person. + +Expressions here are interpreted as rules for assembling the links of the relationship +in the same way that a SQL query is interpreted as a rule for assembling the rows of +a new table. Each expression must therefore reference each role of the relationship, +either explicitly or implicitly. When an expression is relational, then it must explicitly +reference each role. On the other handm if an expression returns a value (like 10.0 in the +two examples here) then that value will implicitly play the last role, and the expression +must reference each of the other roles explicitly. + +A derived concept is one whose population is derived from those of its supertype concepts +using one or more expressions. For instance: + +```yaml +- name: Employee + extends: [Person] + derived_by: [ "EXISTS ( Person.earns )" ] +``` + +declares that the population of Employee is derived from the population of Person by +classifying each Person who earns some salary as a Employee. + +### Requires + +The requires list contains expressions that give additional semantics to a concept or relationship +by declaring conditions that must hold over their populations. When applied to a concept, each +expression must reference the concept itself or one of its supertypes, as in: + +```yaml +- name: SocialSecurityNr + extends: [Integer] + requires: [ "0 < SocialSecurityNr", "SocialSecurityNr <= 999999999" ] +``` + +When applied to a relationship, each expression must reference one or more roles of the +relationship. For instance, in: + +```yaml +- name: Item + extends: [Integer] + relationships: + offers_in: + roles: + - player: Store + verbalizations: [ "{Item} is offered for sale in {Store}", "{Store} offers sale of {Item}" ] + total_sales_in: + roles: + - player: Store + - player: Amount + verbalizations: [ "{Item} sold for cumulative {Amount} in {Store}" ] + requires: + - "Amount > 0.0" + - "Item.offers_in(Store)" +``` + +the first expression requires any value that plays the `Amount` role to be positive while the second +requires any item that has sales in some store to be offered in that store. + +--- + ## Complete Example Here's a complete semantic model example showing all components working together: @@ -478,7 +689,7 @@ semantic_model: expression: email description: Customer email - relationships: + join_paths: - name: orders_to_customers from: orders to: customers @@ -551,6 +762,12 @@ ai_context: ## Version History +- **0.1.2** (2026-05-08): Support for both logical and conceptual modeling layers + - Core conceptual model structure + - Support for concepts, relationships, and logical -> conceptual schema mappings + - Renamed relationships in the logical (semantic) layer to join paths to avoid + conflict with relationships in the conceptual layer + - **0.1.1** (2025-12-11): Initial release - Core semantic model structure - Support for datasets, relationships, fields, and metrics From 717279877a3597fe90257a96464eaa9fef1dce01 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Mon, 11 May 2026 17:55:31 -0400 Subject: [PATCH 02/89] Checkpointing --- core-spec/spec.md | 418 +++++++++++++++++++++++++++++++++------------- 1 file changed, 302 insertions(+), 116 deletions(-) diff --git a/core-spec/spec.md b/core-spec/spec.md index 1765927..7d99568 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -16,9 +16,8 @@ 4. [Join paths](#join-paths) 5. [Fields](#fields) 6. [Metrics](#metrics) -7. [Concepts](#concepts) -8. [Mappings](#mappings) -9. [Complete Example](#complete-example) +7. [Ontology](#ontology) +8. [Complete Example](#complete-example) --- @@ -41,7 +40,7 @@ Supported SQL and expression language dialects for metrics and field definitions ### Multiplicities -The allowable multiplicities of relationships defined in the [Concepts](#concepts) section. +The allowable multiplicities of relationships defined in the [Ontology](#ontology) section. | Multiplicity | Description | |---------|-------------| @@ -97,7 +96,7 @@ semantic_model: ## Datasets -Logical datasets represent business entities or concepts (fact and dimension tables). They contain fields and define the structure of the data. +Logical datasets represent business entities (fact and dimension tables). They contain fields and define the structure of the data. ### Schema @@ -423,39 +422,69 @@ custom_extensions: --- -## Concepts +## Ontology -Concepts represent the types of things that have meaning in a business setting, e.g., person, company, or -salary. Each concept is either an entity type or a value type. Every ontology starts with a value-type -concept for each basic data type, like `Integer`, `Decimal`, and `String`, and an entity-type concept -called `Any`. Every other concept in the ontology extends one of these starter concepts. +Enterprise data are commonly modeled at different levels. The logical level comprises datasets and +fields, while the conceptual level is an ontology that comprises concepts, relationships, and +business rules. This section describes how to declare an ontology and, when a model also +declares datasets and fields, logical to conceptual schema mappings. + +Ontologies and schema mappings are organized hierarchically. At the top level are the concepts, +under which relationships, business rules, and schema mappings are then defined. + +### Concepts + +Concepts represent the types of things that have meaning in a business setting, e.g., person, company, +or salary. Each concept is either an entity type or a value type. Each model implicitly includes a +value-type for each basic data type, like `Integer`, `Decimal`, and `String`, and an entity type +called `Any`. Every other concept in an ontology extends (is a subtype of) one of these concepts. ### Schema | Field | Type | Required | Description | |-------|------|----------|-------------| -| `name` | string | Yes | Unique identifier for this concept | +| `concept` | string | Yes | Unique name of a concept | | `description` | string | No | Human-readable description | -| `relationships` | object | No | Relationships where this concept plays the first role | +| `relationships` | list | No | Relationships where this concept plays the first role | | `extends` | list | No | Names of this concept's supertypes | | `derived_by` | list | No | Expressions for deriving this concept's population | -| `identify_by` | list | No | Names of relationships that collectively identify this concept | +| `identify_by` | list | No | Names of relationships to use to referencing objects in expressions and queries | | `requires` | list | No | Expressions that constrain this concept's population | +| `entity_mappings` | list | No | Mappings from fields to concept objects | +| `relationship_mappings` | list | No | Mappings from fields to relatioship links | ### Extends Every concept that a user declares extends one or more concepts in the ontology. The new concept -is a sutype of each concept that it extends, and the extended concepts are its supertypes. For instance,`SocialSecurityNr` extends `Integer`, `Person` extends `Any`, and `Employee` extends `Person`. Any -concept that directly or indirectly extends a value type like `Integer` or `String` is a value type. -A concept is an entity type if it is not a value type. Any concept with an empty extends list defaults -to being a subtype of `Any` and so is an entity type. +is a sutype of each concept that it extends, and the extended concepts are its supertypes. + +Any concept that directly or indirectly extends a value type like `Integer` or `String` is a value type. +Any concept that does not extend some value type is an entity type, and if a concept declares no extends +list, then it is assumed to extend the built-in entity type `Any`. + +So if `SocialSecurityNr` extends `Integer` and `Employee` extends `Person`, which declares +no extends list, then `SocialSecurityNr` is a value type and both `Person` and `Employee` +are entity types. ### Relationships -Relationships relate objects of one or more concepts and declare how to verbalize links among those objects. -For instance, a relationship named `earns` links persons to salaries. Each link pairs some `Person` with -some `Salary` and verbalizes as, “Person earns Salary annually.” Relationships have set (as opposed to bag) -semantics, and links do not contain nulls. +Relationships relate objects of one or more concepts and declare how to verbalize links among +those objects. For instance, the following relationships: + +```yaml +ontology: + - concept: Person + relationships: + - name: earns + roles: + - player: Salary + multiplicity: ManyToOne + verbalizes: [ "{Person} earns {Salary}" ] + ... +``` + +links `Person` and `Salary` objects and verbalizes each link as “Person earns Salary.” +Relationships have set (as opposed to bag) semantics, and links do not contain nulls. #### Schema @@ -470,116 +499,124 @@ semantics, and links do not contain nulls. | `verbalization` | list | Yes | Patterns describing how to verbalize links | Each relationship is uniquely identified by a prepending its declared name with that of the containing -concept. For instance, a relationship named `earns`, declared within the `Person` concept is identified -by the string `Person.earns`. This convention naturally supports expressions that navigate over the links -of relationships using the “dot-join” operator in a manner that is familiar to object-oriented programming -languages. - -#### Multiplicities - -In a relationship that comprises more than one role, the last role might be functionally dermined -by the other roles. This is declared by providing a ManyToOne multiplicity on that relationship. -For relationships of ternary and higher arity, the multiplicity applies to the n-th role, meaning -the object that plays the n-th role is functionally determined by the tuple of objects that play -the first n-1 roles. A relationship like `Item.total_sales_in`, which records the total sales amount -of an item at a given store, will have a ManyToOne multiplicity to indicate that for any pair -of `Item` and `Store` at most one `Amount` will be recorded as the total sales amount for that -`Item` and that `Store`. - -In the special case of a binary relationship, one might declare a OneToOne multiplicity, which -indicates the relationship is ManyToOne in both directions. For instance, the `Person.ssn` -relationship will have a OneToOne multiplicity because each person is assigned at most one social -security number and each social security number is assigned to at most one person. - -In the absence of any multiplicity, we make no assumptions of functional dependencies among -any of the roles. +concept. The relationship in the previous example is identified by the string `Person.earns`. +This convention naturally supports expressions that navigate over the links of relationships using +the “dot-join” operator in a manner that is familiar to object-oriented programming languages. #### Roles Objects play roles in the links of a relationship. If you think of a relationship as a narrow table, -then links are its rows and roles are its columns. Each role is played by some concept, which -indicates the type of objects that can play that role in the relationship's links. In the +then its links are like rows and its roles are like columns. Each role is played by a concept that +constraints the type of objects that can play the role in the relationship's links. In the `Person.earns` relationship, `Person` and `Salary` play the first and second roles respectively. By convention, the first role of any relationship will be played by the concept under which the -relationship is declared. Any additional roles are declared in the roles field as follows: +relationship is declared. Any additional roles are enumerated in the roles list using this schema: | Field | Type | Required | Description | |-------|------|----------|-------------| | `player` | string | Yes | Name of the concept that plays this role | | `name` | string | No | Optional role name | -A unary relationship like `Person.files_married_joint` declares no additional roles, -while a ternary relationship like `Person.purchased_on` declares two additional roles -played by `Vehicle` and `Date` respectively. +A ternary relationship like `Person.purchased_on` declares two additional roles played by +`Vehicle` and `Date` respectively, while a unary relationship like +`Person.files_married_joint` will have an empty roles list. -The role player often suffices to distinguish the role within a given relationship. +The role player often suffices to distinguish the role within its relationship. However, the same concept can play multiple roles, such as in the ternary relationship `Store.ships_to_in_days` that connects pairs of `Store` objects to the number of days required to ship inventory from one store to another. When this happens, the user must declare a distinguising name for any additional role whose player does not suffice to distinguish it from other roles in the same relationship. +#### Multiplicities + +In a relationship that comprises more than one role, objects that play the last role could +be functionally dermined by a tuple of objects that play the other roles. This knowledge is +declared using a `ManyToOne` multiplicity constraint. In the previous example, the constraint +declares that each person earns at most one salary. + +For relationships of ternary and higher arity, the multiplicity applies to the n-th role, meaning +the object that plays the n-th role is functionally determined by the tuple of objects that play +the first n-1 roles. A relationship like `Item.total_sales_in`, which records the total sales amount +of an item at a given store, is many-to-one because for any pair of `Item` and `Store` at most one +`Amount` represents the total sales for that `Item` and that `Store`. + +In the special case of a binary relationship, one might declare a `OneToOne` multiplicity, which +indicates the relationship is many-to-one in both directions. For instance, the `Person.ssn` +relationship is one-to-one because each person is assigned at most one social security number +and each social security number is assigned to at most one person. + +In the absence of any multiplicity, we make no assumptions of functional dependencies among +any of the roles. + ### Identifying relationships -Many conceptual models distinguish one or more relationships that should be used when -referecing entity-type objects in expressions and queries. For instance, the relationship -`Person.ssn` can be used to reference a person by their social security number; while -the pair of relationships `License.acct` and `License.seat_nr` can be used to reference -a license by the account and seat number. These relationships are always binary, and -their first roles are always played by the concept the relationship is used to reference. -When a set of identifying relationships is known for a concept, this knowledge can be -expressed using the identify_by list. +Many conceptual models distinguish one or more relationships to use when referecing entity-type +objects in expressions and queries. The `Person.ssn` relationship can be used to reference a +person by their social security number; while the pair of relationships `License.acct` and +`License.seat_nr` can be used to reference a license by its associated account and seat number. +These identifier relationships are always binary, and their first role is always played by the +concept the relationship is used to reference. ### Derivation expressions -Concepts and relationships may be derived using expressions. A derived relationship is one -whose links are derived from those of other relationships. For instance: +Concepts and relationships may be derived using expressions. Think of a derived relationship +as a conceptual view whose links are derived from those of other relationships. For instance: ```yaml -- name: Person - relationships: - parent_of: - roles: - - player: Person - name: "child" - verbalizes: [ "{Person} is a parent of {Person:child}", "{Person:child} is a child of {Person}" ] - ancestor_of: - roles: - - player: Person - name: "descendant" - derived_by: - - "Person.parent_of(descendant)" - "Person.ancestor_of.parent_of(descendant)" - taxed_at: - roles: - - player: TaxRate - derived_by: - - "10.0 WHERE ( Person.files_single AND Person.earns <= 11925 )" - - "10.0 WHERE ( Person.files_married_joint AND Person.earns <= 23850 )" - - ... +ontology: + - concept: Person + relationships: + - name: parent_of + roles: + - player: Person + name: "child" + verbalizes: [ "{Person} is a parent of {Person:child}", "{Person:child} is a child of {Person}" ] + - name: ancestor_of + roles: + - player: Person + name: "descendant" + derived_by: + - "Person.parent_of(descendant)" + "Person.ancestor_of.parent_of(descendant)" + taxed_at: + roles: + - player: TaxRate + derived_by: + - "10.0 WHERE ( Person.files_single AND Person.earns <= 11925 )" + - "10.0 WHERE ( Person.files_married_joint AND Person.earns <= 23850 )" + - ... ``` -derives a link of `Person.taxed_at` for each object of the first role player (Person) -using expressions that return a TaxRate based on the person's filing status and how -much they earn. If, for some person, none of the expressions can be evaluated, then -the relationship will have no link involving that person. - -Expressions here are interpreted as rules for assembling the links of the relationship -in the same way that a SQL query is interpreted as a rule for assembling the rows of -a new table. Each expression must therefore reference each role of the relationship, -either explicitly or implicitly. When an expression is relational, then it must explicitly -reference each role. On the other handm if an expression returns a value (like 10.0 in the -two examples here) then that value will implicitly play the last role, and the expression -must reference each of the other roles explicitly. +declares two derived relationships -- `ancestor_of` and `taxed_at`. Each link of `Person.ancestor_of` +relates a person to one of its descendants. The two expressions form the base and recursive cases for +this calculation. In the base case, a `Person` as an ancestor of some `descendant` if that `Person` +is the parent of that descendant. And in the recursive case, a `Person` is an ancestor of some +`descendant` if that `Person` is an ancestor of the parent of that `descendant`. Notice in this +example how role names are used to disambiguate the two `Person` roles in this relationship. + +Each link of `Person.taxed_at` links a `Person` object to a `TaxRate` that is derived using +expressions that determine the rate based on the person's filing status and how much they earn. +If, for some person, none of the expressions can be evaluated, then the relationship will have +no link involving that person. + +Expressions that derive a relationship are interpreted as rules for constructing the links of the +relationship in the same way that a SQL query is interpreted as a rule for constructing the rows +of a new table. Each expression must therefore reference each role of the relationship, either +explicitly or implicitly. If an expression evaluates to some object (like 10.0 in the two examples +here) then that object will implicitly play the last role, and the expression must reference each +of the other roles explicitly. If an expression does not evaluate to any object, then it must +explicitly reference each role. A derived concept is one whose population is derived from those of its supertype concepts using one or more expressions. For instance: ```yaml -- name: Employee - extends: [Person] - derived_by: [ "EXISTS ( Person.earns )" ] +ontology: + - concept: Employee + extends: [Person] + derived_by: [ "EXISTS ( Person.earns )" ] ``` declares that the population of Employee is derived from the population of Person by @@ -589,38 +626,187 @@ classifying each Person who earns some salary as a Employee. The requires list contains expressions that give additional semantics to a concept or relationship by declaring conditions that must hold over their populations. When applied to a concept, each -expression must reference the concept itself or one of its supertypes, as in: +expression must reference the concept, as in: ```yaml -- name: SocialSecurityNr - extends: [Integer] - requires: [ "0 < SocialSecurityNr", "SocialSecurityNr <= 999999999" ] +ontology: + - concept: SocialSecurityNr + extends: [Integer] + requires: [ "0 < SocialSecurityNr", "SocialSecurityNr <= 999999999" ] ``` When applied to a relationship, each expression must reference one or more roles of the relationship. For instance, in: ```yaml -- name: Item - extends: [Integer] - relationships: - offers_in: - roles: - - player: Store - verbalizations: [ "{Item} is offered for sale in {Store}", "{Store} offers sale of {Item}" ] - total_sales_in: - roles: - - player: Store - - player: Amount - verbalizations: [ "{Item} sold for cumulative {Amount} in {Store}" ] - requires: - - "Amount > 0.0" - - "Item.offers_in(Store)" +ontology: + - concept: Item + extends: [Integer] + relationships: + - name: offers_in + roles: + - player: Store + verbalizations: [ "{Item} is offered for sale in {Store}", "{Store} offers sale of {Item}" ] + - name: total_sales_in + roles: + - player: Store + - player: Amount + verbalizations: [ "{Item} sold for cumulative {Amount} in {Store}" ] + requires: + - "Amount > 0.0" + - "Item.offers_in(Store)" ``` the first expression requires any value that plays the `Amount` role to be positive while the second requires any item that has sales in some store to be offered in that store. +### Mappings + +Logical to conceptual schema mappings declare how field values map to conceptual objects and +relationship links among conceptual objects. The key idea is to map a SQL expression that +references one or more fields to some role that is played by a value-typed concept. + +#### Entity mappings + +Entity mappings declare how and under what conditions fields in the logical level map to objects of +some entity type in the population of the model by mapping to the value-typed roles of its identifying +relationsips: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `description` | string | No | Human-readable description | +| `identify_by` | list | Yes | Role bindings for each identifying relationship | + +For instance, this entity mapping: + +```yaml +ontology: + - concept: Person + relationships: + - name: nr + roles: + - player: SocialSecurityNr + multiplicity: OneToOne + verbalizes: [ "{Person} is identified by {SocialSecurityNr}" ] + identify_by: [ nr ] + entity_mappings: + - identify_by: + - role: Person.nr + expr: PERSONS.SSN + ... +``` + +uses one role binding that maps values from the `SSN` field of dataset `PERSONS` to objects +that play the `SocialSecurityNr` role in the `nr` relationship. Because each link in that +relationship associates a `SocialSecurityNr` object to some unique `Person` object, this +one role binding suffices to associate each distinct `SSN` value in the dataset to a distinct +`Person` object in the ontology. + +A more interesting example maps fields to instances of the `OrderLineItem` concept, whose identifier +involves two relationships: + +```yaml + - concept: OrderLineItem + relationships: + - name: nr + roles: [ concept: LineNr ] + multiplicity: ManyToOne + - name: order + roles: [ concept: CustOrder ] + multiplicity: ManyToOne + identify_by: + requires: [ "nr", "order" ] + entity_mappings: + - identify_by: + - role: OrderLineItem.nr + expr: LINEITEMS.L_LINENUMBER + - role: OrderLineItem.order + identify_by: + - role: CustOrder.nr + expr: LINEITEMS.L_ORDERKEY + +``` + +Notice how this entity mapping uses two role bindings -- one that maps the `L_LINENUMBER` +field to the `LineNr` role of the `nr` relationship, and one that uses a more complex +structure to map an object to the `CustOrder` role of the `order` relationship. Because +`LineNr` is a value-typed concept, we can directly map an expression involving fields to +that role just as in the previous example. But because `CustOrder` is an entity typed +concept, we cannot directly map field values to its objects but must instead use its +identifier, which here involves one relationship called `CustOrder.nr`. + +#### Relationship mappings + +Relationship mappings declare how and under what conditions fields refer to objects that +play roles in the links of relationships. Unlike entity mappings, which declare how to +construct objects from fields, relationship mappings declare how to look up objects using +fields and then form tuples that populate relationships in the model. + +Relationship mappings are organized hierarchically into trees with common tuple prefixes +to allow mapping to links of relationships of many different arities while minimizing +redundancy. Each node in the tree has this schema: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `concept` | string | No | Concept that plays the role mapped to by this node in the tree | +| `expr` | string | No | Maps fields to objects when the concept is a value type | +| `identify_by` | list | Yes | Maps fields to identifying relationships when the concept is an entity type | +| `relationship` | string | No | Relationship to populate using tuple mapped to by this level in the tree | +| `children` | list | List of relationship child mappings | + +```yaml +ontology: + - concept: Item + relationships: + - name: nr + roles: [ concept: SkuNr ] + multiplicity: OneToOne + verbalises: "{Item} is identified by {SkuNr}" + - name: active # A unary relationship + verbalizes: "{Item} is actively sold" + - name: active_in + roles: [ concept: Store ] + verbalises: "{Item} is actively sold in {Store}" + - name: returned_in + roles: [ concept: Store, concept: Amount ] + verbalizes: "{Item} returned in {Store} for {Amount}" + multiplicitly: ManyToOne + - name: sold_in + roles: [ concept: Store, concept: Amount ] + verbalizes: "{Item} sells in {Store} for {Amount}" + multiplicitly: ManyToOne + identify_by: [ nr ] + entity_mappings: + - identify_by: + role: Item.nr + expr: ITEMS.SKU + relationship_mappings: + - identify_by: + role: Item.nr + expr: METRICS.SKU + relationship: Item.active + children: + - concept: Store + identify_by: + role: Store.nr + expr: METRICS.STORE + relationship: Item.active_in + children: + - concept: Amount + expr: METRICS.SALES + relationship: Item.sold_in + - concept: Amount + expr: METRICS.RETURNS + relationship: Item.returned_in +``` + +This relationship mapping maps fields of the `METRICS` dataset to links of four different relationships. +The mapping has a tree structure in which paths from the root declare how to use fields to look up +objects that form tuples of various lengths and that at each level in the tree, the tuple of objects +formed at that level can be used to form links of the relationship that is named at that level. This +hierarchical structure simplifies the mapping by not declaring how to map the `SKU` field four times +and the `STORE` field three times. + --- ## Complete Example From 6f2d404bd77de8356361a39f763227fb2bc3a499 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Mon, 11 May 2026 22:10:25 -0400 Subject: [PATCH 03/89] Checkpointing --- core-spec/spec.md | 245 +++++++++++++++++++++++++------------------- core-spec/spec.yaml | 110 +++++++++++++++++++- 2 files changed, 245 insertions(+), 110 deletions(-) diff --git a/core-spec/spec.md b/core-spec/spec.md index 7d99568..6f1ba2c 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -424,52 +424,65 @@ custom_extensions: ## Ontology -Enterprise data are commonly modeled at different levels. The logical level comprises datasets and -fields, while the conceptual level is an ontology that comprises concepts, relationships, and -business rules. This section describes how to declare an ontology and, when a model also -declares datasets and fields, logical to conceptual schema mappings. - -Ontologies and schema mappings are organized hierarchically. At the top level are the concepts, -under which relationships, business rules, and schema mappings are then defined. +Enterprise data are often modeled at both the logical level, in terms of datasets and fields, +and the conceptual level in the form of an ontology that comprises concepts, relationships, +and business rules. This section describes how to declare an ontology and schema mappings +from logical-level fields to concepts and relationships in the ontology. Ontologies are +organized hierarchically in top-level collection of tree structures, each root of which +is a concept, under which relationships and schema mappings are defined. ### Concepts Concepts represent the types of things that have meaning in a business setting, e.g., person, company, -or salary. Each concept is either an entity type or a value type. Each model implicitly includes a +or salary. Each concept is either an entity type or a value type. Ontologies implicitly include a value-type for each basic data type, like `Integer`, `Decimal`, and `String`, and an entity type -called `Any`. Every other concept in an ontology extends (is a subtype of) one of these concepts. +called `Any`. Every other concept ontology extends (is a subtype of) one of these concepts. -### Schema +Concepts conform to the following schema: | Field | Type | Required | Description | |-------|------|----------|-------------| -| `concept` | string | Yes | Unique name of a concept | +| `concept` | string | Yes | Unique name of this concept | | `description` | string | No | Human-readable description | | `relationships` | list | No | Relationships where this concept plays the first role | | `extends` | list | No | Names of this concept's supertypes | -| `derived_by` | list | No | Expressions for deriving this concept's population | -| `identify_by` | list | No | Names of relationships to use to referencing objects in expressions and queries | +| `derived_by` | list | No | Expressions that derive this concept's population | +| `identify_by` | list | No | Names of relationships that uniquely reference objects of this concept | | `requires` | list | No | Expressions that constrain this concept's population | -| `entity_mappings` | list | No | Mappings from fields to concept objects | -| `relationship_mappings` | list | No | Mappings from fields to relatioship links | +| `entity_mappings` | list | No | Mappings from field values to concept objects | +| `relationship_mappings` | list | No | Mappings from field values to relatioship links | ### Extends -Every concept that a user declares extends one or more concepts in the ontology. The new concept +Every user-declared concept extends one or more concepts in the ontology. The new concept is a sutype of each concept that it extends, and the extended concepts are its supertypes. Any concept that directly or indirectly extends a value type like `Integer` or `String` is a value type. Any concept that does not extend some value type is an entity type, and if a concept declares no extends -list, then it is assumed to extend the built-in entity type `Any`. - -So if `SocialSecurityNr` extends `Integer` and `Employee` extends `Person`, which declares -no extends list, then `SocialSecurityNr` is a value type and both `Person` and `Employee` -are entity types. +list, then it is assumed to extend the built-in entity type `Any`. If `SocialSecurityNr` extends `Integer` +and `Employee` extends `Person`, which declares no extends list, then `SocialSecurityNr` is a value type +and both `Person` and `Employee` are entity types. ### Relationships Relationships relate objects of one or more concepts and declare how to verbalize links among -those objects. For instance, the following relationships: +those objects. Relationships have set (as opposed to bag) semantics, and links do not contain +nulls. + +Each relationship that is declared under a concept conforms to the following schema: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Part of the identifier for this relationship | +| `description` | string | No | Human-readable description | +| `multiplicity` | enum | No | Multiplicity constraint | +| `roles` | list | No | List of additional roles in this relationship | +| `derived_by` | list | No | Expressions that derive links of this relationship | +| `requires` | list | No | Expressions that constrain this relationship's population | +| `verbalizes` | list | Yes | Patterns describing how to verbalize links | + +Each relationship is uniquely identified by a prepending its declared name with that of the containing +concept. For instance, in: ```yaml ontology: @@ -483,67 +496,86 @@ ontology: ... ``` -links `Person` and `Salary` objects and verbalizes each link as “Person earns Salary.” -Relationships have set (as opposed to bag) semantics, and links do not contain nulls. - -#### Schema - -| Field | Type | Required | Description | -|-------|------|----------|-------------| -| `name` | string | Yes | Part of the identifier for this relationship | -| `description` | string | No | Human-readable description | -| `multiplicity` | enum | No | Multiplicity constraint | -| `roles` | list | No | List of additional roles in this relationship | -| `derived_by` | list | No | Expressions that derive links of this relationship | -| `requires` | list | No | Expressions that constrain this relationship's population | -| `verbalization` | list | Yes | Patterns describing how to verbalize links | - -Each relationship is uniquely identified by a prepending its declared name with that of the containing -concept. The relationship in the previous example is identified by the string `Person.earns`. -This convention naturally supports expressions that navigate over the links of relationships using -the “dot-join” operator in a manner that is familiar to object-oriented programming languages. +the relationship is identified by the string `Person.earns`. This convention naturally supports +expressions that navigate over the links of relationships using the “dot-join” operator in a +manner that is familiar to object-oriented programming languages. This relationship links +`Person` and `Salary` objects and verbalizes each link as “Person earns Salary.” #### Roles Objects play roles in the links of a relationship. If you think of a relationship as a narrow table, then its links are like rows and its roles are like columns. Each role is played by a concept that -constraints the type of objects that can play the role in the relationship's links. In the -`Person.earns` relationship, `Person` and `Salary` play the first and second roles respectively. +constrains the type of objects that can play that role in any link. In `Person.earns`, `Person` and +`Salary` play the first and second roles respectively. -By convention, the first role of any relationship will be played by the concept under which the -relationship is declared. Any additional roles are enumerated in the roles list using this schema: +By convention, the first role of any relationship is played by the concept under which the +relationship is declared. Any additional roles are enumerated in order in the roles list +using this schema: | Field | Type | Required | Description | |-------|------|----------|-------------| | `player` | string | Yes | Name of the concept that plays this role | | `name` | string | No | Optional role name | -A ternary relationship like `Person.purchased_on` declares two additional roles played by -`Vehicle` and `Date` respectively, while a unary relationship like -`Person.files_married_joint` will have an empty roles list. +For instance, in: + +```yaml +ontology: + - concept: Person + relationships: + - name: files_married_joint + verbalizes: [ "{Person} files married filing joint" ] + - name: purchased_on + roles: + - player: Vehicle + - player: Date + multiplicity: ManyToOne + verbalizes: [ "{Person} puchased {Vehicle} on {Date}" ] +``` + +the unary relationship `Person.files_married_joint` has an empty roles list, while the +ternary relationship `Person.purchased_on` declares two additional roles played by +`Vehicle` and `Date` respectively, -The role player often suffices to distinguish the role within its relationship. -However, the same concept can play multiple roles, such as in the ternary relationship -`Store.ships_to_in_days` that connects pairs of `Store` objects to the number of days -required to ship inventory from one store to another. When this happens, the user must -declare a distinguising name for any additional role whose player does not suffice to -distinguish it from other roles in the same relationship. +The role player often suffices to distinguish the role within its relationship, but when +the same concept plays more than one role, the user must declare a distinguising name for +any additional role whose player does not suffice to distinguish it from other roles in +the same relationship. For instance, in: -#### Multiplicities +```yaml +ontology: + - concept: Store + relationships: + - name: ships_to_in_days + roles: + - player: Store + name: destination + - player: NrDays + multiplicity: ManyToOne + verbalizes: [ "{Store} ships to {destination} in {NrDays}" ] +``` + +the role name `destination` distinguishes the second `Store`-playing role from the first in +this relationship. + +Expressions that are used to define derived_by rules and requires constraints will refer to +roles by name -- the name defaulting to the concept that plays the role unless an explicit +role name is provided. In any expression that involving links of the `Store.ships_to_in_days` +relationship can then use the variables `Store` and `destination` to refer to objecs that +play these two `Store`-playing roles without ambiguity. -In a relationship that comprises more than one role, objects that play the last role could -be functionally dermined by a tuple of objects that play the other roles. This knowledge is -declared using a `ManyToOne` multiplicity constraint. In the previous example, the constraint -declares that each person earns at most one salary. +#### Multiplicities -For relationships of ternary and higher arity, the multiplicity applies to the n-th role, meaning +If a relationship comprises more than one role, objects that play the last role could be functionally +dermined by a tuple of objects that play the other roles. This knowledge is declared using a `ManyToOne` +multiplicity constraint. In the examples above, the constraint declares that each person earns at most +one salary and that for each pair of stores, the former ships to the latter in at most one number of +days. For relationships of ternary and higher arity, the multiplicity applies to the n-th role, meaning the object that plays the n-th role is functionally determined by the tuple of objects that play -the first n-1 roles. A relationship like `Item.total_sales_in`, which records the total sales amount -of an item at a given store, is many-to-one because for any pair of `Item` and `Store` at most one -`Amount` represents the total sales for that `Item` and that `Store`. +the first n-1 roles. In the special case of a binary relationship, one might declare a `OneToOne` multiplicity, which -indicates the relationship is many-to-one in both directions. For instance, the `Person.ssn` +indicates the relationship is many-to-one in both directions. For instance, the `Person.nr` relationship is one-to-one because each person is assigned at most one social security number and each social security number is assigned to at most one person. @@ -552,8 +584,8 @@ any of the roles. ### Identifying relationships -Many conceptual models distinguish one or more relationships to use when referecing entity-type -objects in expressions and queries. The `Person.ssn` relationship can be used to reference a +Many conceptual models distinguish one or more relationships to use when referencing entity-type +objects in expressions and queries. The `Person.nr` relationship can be used to reference a person by their social security number; while the pair of relationships `License.acct` and `License.seat_nr` can be used to reference a license by its associated account and seat number. These identifier relationships are always binary, and their first role is always played by the @@ -561,8 +593,9 @@ concept the relationship is used to reference. ### Derivation expressions -Concepts and relationships may be derived using expressions. Think of a derived relationship -as a conceptual view whose links are derived from those of other relationships. For instance: +Concepts and relationships may be derived using expressions. Think of a derived concept or +relationship as a conceptual view whose objects or links are derived from those of other +concepts or relationships. For instance: ```yaml ontology: @@ -663,21 +696,19 @@ requires any item that has sales in some store to be offered in that store. ### Mappings Logical to conceptual schema mappings declare how field values map to conceptual objects and -relationship links among conceptual objects. The key idea is to map a SQL expression that -references one or more fields to some role that is played by a value-typed concept. +relationship links among conceptual objects. The key idea is to map a SQL expression that computes +a value from one or more fields to some role that is played by a value-typed concept. #### Entity mappings Entity mappings declare how and under what conditions fields in the logical level map to objects of -some entity type in the population of the model by mapping to the value-typed roles of its identifying -relationsips: +some entity type in the population of the model. Each mapping conforms to the following schema: | Field | Type | Required | Description | |-------|------|----------|-------------| -| `description` | string | No | Human-readable description | -| `identify_by` | list | Yes | Role bindings for each identifying relationship | +| `entity_map` | list | Yes | Role bindings for each identifying relationship | -For instance, this entity mapping: +For instance, the entity mapping in: ```yaml ontology: @@ -690,7 +721,7 @@ ontology: verbalizes: [ "{Person} is identified by {SocialSecurityNr}" ] identify_by: [ nr ] entity_mappings: - - identify_by: + - entity_map: - role: Person.nr expr: PERSONS.SSN ... @@ -699,10 +730,10 @@ ontology: uses one role binding that maps values from the `SSN` field of dataset `PERSONS` to objects that play the `SocialSecurityNr` role in the `nr` relationship. Because each link in that relationship associates a `SocialSecurityNr` object to some unique `Person` object, this -one role binding suffices to associate each distinct `SSN` value in the dataset to a distinct +role binding suffices to associate each distinct `SSN` value in the dataset to a distinct `Person` object in the ontology. -A more interesting example maps fields to instances of the `OrderLineItem` concept, whose identifier +A more interesting example maps fields to the `OrderLineItem` concept, whose identifier involves two relationships: ```yaml @@ -714,26 +745,25 @@ involves two relationships: - name: order roles: [ concept: CustOrder ] multiplicity: ManyToOne - identify_by: - requires: [ "nr", "order" ] + identify_by: [ "nr", "order" ] + requires: [ "OrderLineItem.nr", "OrderLineItem.order" ] entity_mappings: - - identify_by: + - entity_map: - role: OrderLineItem.nr expr: LINEITEMS.L_LINENUMBER - role: OrderLineItem.order - identify_by: + entity_map: - role: CustOrder.nr expr: LINEITEMS.L_ORDERKEY ``` -Notice how this entity mapping uses two role bindings -- one that maps the `L_LINENUMBER` -field to the `LineNr` role of the `nr` relationship, and one that uses a more complex -structure to map an object to the `CustOrder` role of the `order` relationship. Because -`LineNr` is a value-typed concept, we can directly map an expression involving fields to -that role just as in the previous example. But because `CustOrder` is an entity typed -concept, we cannot directly map field values to its objects but must instead use its -identifier, which here involves one relationship called `CustOrder.nr`. +This entity mapping uses two role bindings -- one that maps the `L_LINENUMBER` field to the `LineNr` +role of the `nr` relationship, and one that uses a more complex structure to map `CustOrder` objects +to the role that concept plays in the `order` relationship. Because `LineNr` is a value-typed concept, +we can directly map an expression involving fields to that role just as in the previous example. But +because `CustOrder` is an entity type, we cannot directly map field values to its objects but must +instead use its identifier, which here involves one relationship called `CustOrder.nr`. #### Relationship mappings @@ -749,10 +779,12 @@ redundancy. Each node in the tree has this schema: | Field | Type | Required | Description | |-------|------|----------|-------------| | `concept` | string | No | Concept that plays the role mapped to by this node in the tree | -| `expr` | string | No | Maps fields to objects when the concept is a value type | -| `identify_by` | list | Yes | Maps fields to identifying relationships when the concept is an entity type | -| `relationship` | string | No | Relationship to populate using tuple mapped to by this level in the tree | -| `children` | list | List of relationship child mappings | +| `expr` | string | No | Maps fields when the concept is a value type | +| `entity_map` | list | Yes | Maps fields when the concept is an entity type | +| `relationship` | string | No | Relationship whose links are mapped to by this level in the tree | +| `children` | list | No | List of relationship child mappings | + +For instance, the relationship mapping in: ```yaml ontology: @@ -763,35 +795,35 @@ ontology: multiplicity: OneToOne verbalises: "{Item} is identified by {SkuNr}" - name: active # A unary relationship - verbalizes: "{Item} is actively sold" + verbalizes: [ "{Item} is actively sold" ] - name: active_in roles: [ concept: Store ] - verbalises: "{Item} is actively sold in {Store}" + verbalises: [ "{Item} is actively sold in {Store}" ] - name: returned_in roles: [ concept: Store, concept: Amount ] - verbalizes: "{Item} returned in {Store} for {Amount}" + verbalizes: [ "{Item} returned in {Store} for {Amount}" ] multiplicitly: ManyToOne - name: sold_in roles: [ concept: Store, concept: Amount ] - verbalizes: "{Item} sells in {Store} for {Amount}" + verbalizes: [ "{Item} sells in {Store} for {Amount}" ] multiplicitly: ManyToOne identify_by: [ nr ] entity_mappings: - - identify_by: + - entity_map: role: Item.nr expr: ITEMS.SKU relationship_mappings: - - identify_by: + - entity_map: role: Item.nr expr: METRICS.SKU relationship: Item.active - children: + relationship_mappings: - concept: Store - identify_by: + entity_map: role: Store.nr expr: METRICS.STORE relationship: Item.active_in - children: + relationship_mappings: - concept: Amount expr: METRICS.SALES relationship: Item.sold_in @@ -800,12 +832,9 @@ ontology: relationship: Item.returned_in ``` -This relationship mapping maps fields of the `METRICS` dataset to links of four different relationships. -The mapping has a tree structure in which paths from the root declare how to use fields to look up -objects that form tuples of various lengths and that at each level in the tree, the tuple of objects -formed at that level can be used to form links of the relationship that is named at that level. This -hierarchical structure simplifies the mapping by not declaring how to map the `SKU` field four times -and the `STORE` field three times. +maps fields of the `METRICS` dataset to links of four different relationships. The hierarchical +structure simplifies the mapping by not declaring how to map the `SKU` field four times and the +`STORE` field three times. --- diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index d8b5545..6595d82 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -20,6 +20,9 @@ dialects: - "MAQL" # GoodData MAQL (Multi-Dimensional Analytical Query Language) +multiplicities: + - "ManyToOne" + - "OneToOne" # Supported vendors for custom extensions vendors: @@ -48,7 +51,10 @@ semantic_model: # Optional: Defines how logical datasets are connected # See Relationships section below for detailed structure - relationships: [] + join_paths: [] + + # Optional: Defines the ontology used to give meaning to the data + ontology: [] # Optional: # These metrics can span one or more logical datasets and use relationships @@ -118,7 +124,7 @@ datasets: # Relationship Schema # Defines how logical datasets or semantic models are connected # Represents foreign key relationships (many-to-one or one-to-one) -relationships: +join_paths: # Required: Unique identifier for the relationship - name: string @@ -217,3 +223,103 @@ metrics: custom_extensions: - vendor_name: string # Must be one of the values from 'vendors' enum above data: string + +--- +# Ontology Schema +ontology: + + # Required: Unique identifier for the concept + - concept: string + + # Optional: Human-readable description of the concept + description: string + + # Optional: Relationships in which this concept plays the first role + relationships: [] + + # Optional: Array of names of relationships that form the preferred identifier of this concept + identify_by: [] + + # Optional: Array of names of supertype concepts that this concept extends + extends: [] + + # Optional: Array of expressions that derive the population of this concept + derived_by: [] + + # Optional: Array of expressions that constrain the population of this concept + requires: [] + + # Optional: Array of entity mappings that map logical fields to concept objects + entity_mappings: [] + + # Optional: Array of relationship mappings that map logical fields to relationship links + relationship_mappings: [] + +relationships: + + # Required: unique name of the relationship in the context of the declaring concept + - name: string + + # Optional: Human-readable description + description: string + + # Optional: Multiplicity constraint + multiplicity: multiplicities + + # Optional: List of additional roles in this relationship + roles: [] + + # Optional: Array of expressions that derive links of this relationship + derived_by: [] + + # Optional: Array of expressions that constrain this relationship's population + requires: [] + + # Optional Array of string patterns that describe how to verbalize links of + # this relationship + verbalizes: [] + +roles: + + # Required: Name of the concept that plays this role + - player: string + + # Optional: Role name + name: string + +entity_mappings: + + - entity_map: [] + +entity_map: + + # Required: String naming the idenfier relationship whose role is mapped to using either the + # expr if the role is played by a value type or the nested entity map if the role is played + # by an entity type + - role: string + + # Optional: SQL expression whose value maps to the role + expr: string + + # Optional: Nested entity map that returns an entity object that maps to the role + entity_map: [] + +relationship_mappings: + + - relationship_map: [] + +relationship_map: + + # Optional: Name of the concept that plays this role in the relationship map + - concept: string + + # Optional: Maps fields to this role when the player is a value type + expr: string + + # Optional: Maps fields to this role when the player is an entity type + entity_map: [] + + # Optional: Name of relationship whose links are mapped to by this level in the tree + relationship: string + + relationship_mappings: [] \ No newline at end of file From 4f0e87967b2c0ffb83614d2d4c9e0280077c63d3 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Tue, 12 May 2026 16:18:21 -0400 Subject: [PATCH 04/89] Checkpointing after improving description of mappings --- core-spec/spec.md | 171 ++++++++++++++++++++++++++++------------------ 1 file changed, 105 insertions(+), 66 deletions(-) diff --git a/core-spec/spec.md b/core-spec/spec.md index 6f1ba2c..2b82b61 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -449,8 +449,8 @@ Concepts conform to the following schema: | `derived_by` | list | No | Expressions that derive this concept's population | | `identify_by` | list | No | Names of relationships that uniquely reference objects of this concept | | `requires` | list | No | Expressions that constrain this concept's population | -| `entity_mappings` | list | No | Mappings from field values to concept objects | -| `relationship_mappings` | list | No | Mappings from field values to relatioship links | +| `entities` | list | No | Mappings that determine this concept's objects (when an entity type) | +| `links` | list | No | Mappings from field values to relatioship links | ### Extends @@ -552,7 +552,7 @@ ontology: name: destination - player: NrDays multiplicity: ManyToOne - verbalizes: [ "{Store} ships to {destination} in {NrDays}" ] + verbalizes: [ "{Store} ships to {Store:destination} in {NrDays}" ] ``` the role name `destination` distinguishes the second `Store`-playing role from the first in @@ -695,20 +695,47 @@ requires any item that has sales in some store to be offered in that store. ### Mappings -Logical to conceptual schema mappings declare how field values map to conceptual objects and -relationship links among conceptual objects. The key idea is to map a SQL expression that computes -a value from one or more fields to some role that is played by a value-typed concept. +Logical to conceptual schema mappings declare how to map field values to conceptual objects +and links. The most basic kind of mapping is a value map, which has the following schema: -#### Entity mappings +| Field | Type | Required | Description | +|--------------|----------|-----|-------| +| `role` | string | Yes | Expression that indicates the role that the mapped objects will play | +| `value` | string | Yes | Expression that calculates the value to map to the role | + +These declare how to map a SQL expression that computes a value from one or more fields to a +role of some relationship, where the role must be played by a value type. -Entity mappings declare how and under what conditions fields in the logical level map to objects of -some entity type in the population of the model. Each mapping conforms to the following schema: +Entity-type objects (entities) are opaque and must be referenced by their identifying +relationships. These relationships are always binary, with one role played by the entity +type and the other played by some other concept. In the common case where an entity type +is identified by a single relationship whose other role is played by a value type, a value +map suffices to declare the existence of entities of that type and also to look them up +when mapping to links of a relationship. And if an entity type is identified by a pair +of such relationships, a pair of value maps suffice for the same purpose. + +An entity map is an array of structures, each of which is either a value map or a nested entity +map that looks up an object to map to some role of an identifier relationship. These structures +have the following schema: | Field | Type | Required | Description | -|-------|------|----------|-------------| -| `entity_map` | list | Yes | Role bindings for each identifying relationship | +|--------------|----------|-----|-------| +| `role` | string | Yes | Expression that indicates the role the mapped objects will play | +| `value` | string | if no `entity` | Expression that maps to a value (when role played by a value type) | +| `entity` | list | if no `value` | Entity map that maps to an entity (when role played by an entity type) | + +where each must provide either a `value` expression (in which case the structure reduces to a value map) +or a nested `entity` map but not both. + +Building on these structures, we can declare precisely how fields determine the entities of a concept +and the links of every relationship in which the concept plays the first role. + +#### Entities (entity maps that determine the existence of objects of some entity type) + +When a concept is an entity type, it can declare an `entities` array, each element of which +is an entity map. -For instance, the entity mapping in: +For instance, in: ```yaml ontology: @@ -720,17 +747,18 @@ ontology: multiplicity: OneToOne verbalizes: [ "{Person} is identified by {SocialSecurityNr}" ] identify_by: [ nr ] - entity_mappings: - - entity_map: + + entities: + - entity: - role: Person.nr - expr: PERSONS.SSN + value: PERSONS.SSN ... ``` -uses one role binding that maps values from the `SSN` field of dataset `PERSONS` to objects -that play the `SocialSecurityNr` role in the `nr` relationship. Because each link in that +uses a single value map to declare how values from the `SSN` field of dataset `PERSONS` +map to `SocialSecurityNr` objects in the `nr` relationship. Because each link in that relationship associates a `SocialSecurityNr` object to some unique `Person` object, this -role binding suffices to associate each distinct `SSN` value in the dataset to a distinct +value map suffices to associate each distinct `SSN` value in the dataset to a distinct `Person` object in the ontology. A more interesting example maps fields to the `OrderLineItem` concept, whose identifier @@ -740,51 +768,57 @@ involves two relationships: - concept: OrderLineItem relationships: - name: nr - roles: [ concept: LineNr ] + roles: [ player: LineNr ] multiplicity: ManyToOne - name: order - roles: [ concept: CustOrder ] + roles: [ player: CustOrder ] multiplicity: ManyToOne identify_by: [ "nr", "order" ] requires: [ "OrderLineItem.nr", "OrderLineItem.order" ] - entity_mappings: - - entity_map: + + entities: + - entity: - role: OrderLineItem.nr - expr: LINEITEMS.L_LINENUMBER + value: LINEITEMS.L_LINENUMBER - role: OrderLineItem.order - entity_map: + entity: - role: CustOrder.nr - expr: LINEITEMS.L_ORDERKEY + value: LINEITEMS.L_ORDERKEY ``` -This entity mapping uses two role bindings -- one that maps the `L_LINENUMBER` field to the `LineNr` -role of the `nr` relationship, and one that uses a more complex structure to map `CustOrder` objects -to the role that concept plays in the `order` relationship. Because `LineNr` is a value-typed concept, -we can directly map an expression involving fields to that role just as in the previous example. But -because `CustOrder` is an entity type, we cannot directly map field values to its objects but must -instead use its identifier, which here involves one relationship called `CustOrder.nr`. +This mapping contains a single entity map with two components -- a value map that maps the +`L_LINENUMBER` field to the `LineNr` role of the `nr` relationship, and a nested entity map +that maps `CustOrder` objects to the role that concept plays in the `order` relationship. -#### Relationship mappings +#### Links (mappings that determine the existence of relationship links) -Relationship mappings declare how and under what conditions fields refer to objects that -play roles in the links of relationships. Unlike entity mappings, which declare how to -construct objects from fields, relationship mappings declare how to look up objects using -fields and then form tuples that populate relationships in the model. +Each of an ontology's relationships is grouped under the concept that plays its first role. +The concept's `links` array declares schema mappings from field values to links of these +relationships. Each element in the array is a tree structure that concisely declares how +to map to tuples that form the links of one or more of these relationships. The path from +the root of a tree to each node describes how to map to tuples of objects that form the +links of the relationship that is named by the node. These structures leverage the hierarchical +nature of YAML to prevent the duplication in the typical case when the fields of a single +dataset map to many relationships. -Relationship mappings are organized hierarchically into trees with common tuple prefixes -to allow mapping to links of relationships of many different arities while minimizing -redundancy. Each node in the tree has this schema: +Each tree node has the following schema: | Field | Type | Required | Description | |-------|------|----------|-------------| -| `concept` | string | No | Concept that plays the role mapped to by this node in the tree | -| `expr` | string | No | Maps fields when the concept is a value type | -| `entity_map` | list | Yes | Maps fields when the concept is an entity type | +| `concept` | string | when level > 1 | Concept whose objects are looked up by this node | +| `value` | string | if no `entity` | Expression that computes a value (when concept is a value type) | +| `entity` | list | if no `value` | Entity map that looks up an object (when concept is an entity type) | | `relationship` | string | No | Relationship whose links are mapped to by this level in the tree | -| `children` | list | No | List of relationship child mappings | +| `children` | list | No | List of child nodes in the tree | + -For instance, the relationship mapping in: +Each node must provide either a `value` or an `entity` by which to look up objects but never both. +The level of each node coincides with the arity of the associated relationship. So a top-level +(root) node could map to a unary relationship, a node at level 2 could map to a binary +relationship, and so forth. + +For instance, the relationship mapping declared here: ```yaml ontology: @@ -799,42 +833,47 @@ ontology: - name: active_in roles: [ concept: Store ] verbalises: [ "{Item} is actively sold in {Store}" ] - - name: returned_in + - name: returned_in_for roles: [ concept: Store, concept: Amount ] verbalizes: [ "{Item} returned in {Store} for {Amount}" ] multiplicitly: ManyToOne - - name: sold_in + - name: sold_in_for roles: [ concept: Store, concept: Amount ] verbalizes: [ "{Item} sells in {Store} for {Amount}" ] multiplicitly: ManyToOne identify_by: [ nr ] - entity_mappings: - - entity_map: - role: Item.nr - expr: ITEMS.SKU - relationship_mappings: - - entity_map: - role: Item.nr - expr: METRICS.SKU + + entities: + - entity: + - role: Item.nr + value_map: ITEMS.SKU + + links: + - entity: + - role: Item.nr + value_map: METRICS.SKU relationship: Item.active - relationship_mappings: + children: - concept: Store - entity_map: - role: Store.nr - expr: METRICS.STORE + entity: + - role: Store.nr + value: METRICS.STORE relationship: Item.active_in - relationship_mappings: + children: - concept: Amount - expr: METRICS.SALES - relationship: Item.sold_in + value: METRICS.SALES + relationship: Item.sold_in_for - concept: Amount - expr: METRICS.RETURNS - relationship: Item.returned_in + value: METRICS.RETURNS + relationship: Item.returned_in_for ``` -maps fields of the `METRICS` dataset to links of four different relationships. The hierarchical -structure simplifies the mapping by not declaring how to map the `SKU` field four times and the -`STORE` field three times. +describes a tree with one root node, one node at level 2, and 2 nodes at level 3. Each node +maps fields of the `METRICS` dataset to links of four different relationships, and notice +how the mapping to `Item` objects is declared once even though `Item` plays a role in all +four of the relationships and that the mapping to `Store` objects is declared once even +though `Store` plays a role in three of the relationships. + --- From 8f022442af4a21872c4c80d708a9980ea71bfabc Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Tue, 12 May 2026 16:56:10 -0400 Subject: [PATCH 05/89] Ready for review --- core-spec/spec.md | 99 +++++++++++++++++++++------------------------ core-spec/spec.yaml | 43 +++++++++++--------- 2 files changed, 71 insertions(+), 71 deletions(-) diff --git a/core-spec/spec.md b/core-spec/spec.md index 2b82b61..c37b0fd 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -1,6 +1,6 @@ # OSI - Core Metadata Specification -**Version:** 0.1.1 +**Version:** 0.1.2 ## Goals @@ -696,27 +696,24 @@ requires any item that has sales in some store to be offered in that store. ### Mappings Logical to conceptual schema mappings declare how to map field values to conceptual objects -and links. The most basic kind of mapping is a value map, which has the following schema: +and links. The most basic kind of mapping is a value map: | Field | Type | Required | Description | |--------------|----------|-----|-------| | `role` | string | Yes | Expression that indicates the role that the mapped objects will play | | `value` | string | Yes | Expression that calculates the value to map to the role | -These declare how to map a SQL expression that computes a value from one or more fields to a +which declares how to map a SQL expression that computes a value from one or more fields to a role of some relationship, where the role must be played by a value type. -Entity-type objects (entities) are opaque and must be referenced by their identifying -relationships. These relationships are always binary, with one role played by the entity -type and the other played by some other concept. In the common case where an entity type -is identified by a single relationship whose other role is played by a value type, a value -map suffices to declare the existence of entities of that type and also to look them up -when mapping to links of a relationship. And if an entity type is identified by a pair -of such relationships, a pair of value maps suffice for the same purpose. +Entity-type objects (entities) are opaque and must be mapped to via their identifying +relationships. These relationships are always binary, with the first role played by +the entity type itself. In the common case where an entity type is identified by a +single relationship whose second role is played by a value type, a value map suffices +to declare the existence of entities of that type or to look them up when mapping to +links of a relationship. -An entity map is an array of structures, each of which is either a value map or a nested entity -map that looks up an object to map to some role of an identifier relationship. These structures -have the following schema: +An entity map is an array of identifier maps: | Field | Type | Required | Description | |--------------|----------|-----|-------| @@ -724,11 +721,11 @@ have the following schema: | `value` | string | if no `entity` | Expression that maps to a value (when role played by a value type) | | `entity` | list | if no `value` | Entity map that maps to an entity (when role played by an entity type) | -where each must provide either a `value` expression (in which case the structure reduces to a value map) -or a nested `entity` map but not both. +each of which is either a value map or a nested entity map that looks up an object to map to some +role of an identifier relationship. -Building on these structures, we can declare precisely how fields determine the entities of a concept -and the links of every relationship in which the concept plays the first role. +Building on value and entity maps, we can declare precisely how fields determine the entities of +a concept and the links of every relationship in which that concept plays the first role. #### Entities (entity maps that determine the existence of objects of some entity type) @@ -755,14 +752,14 @@ ontology: ... ``` -uses a single value map to declare how values from the `SSN` field of dataset `PERSONS` -map to `SocialSecurityNr` objects in the `nr` relationship. Because each link in that -relationship associates a `SocialSecurityNr` object to some unique `Person` object, this -value map suffices to associate each distinct `SSN` value in the dataset to a distinct -`Person` object in the ontology. +uses a value map to declare how values from the `SSN` field of dataset `PERSONS` are used +to form `SocialSecurityNr` values that map to the second role of the `nr` relationship. +Because each link in that relationship associates a `SocialSecurityNr` value to some +unique `Person` entity, this value map suffices to associate each distinct `SSN` value +in the dataset to a distinct `Person` object in the ontology. -A more interesting example maps fields to the `OrderLineItem` concept, whose identifier -involves two relationships: +A more interesting example maps field values to `OrderLineItem` entities, which are +identified using two relationships: ```yaml - concept: OrderLineItem @@ -787,20 +784,20 @@ involves two relationships: ``` -This mapping contains a single entity map with two components -- a value map that maps the -`L_LINENUMBER` field to the `LineNr` role of the `nr` relationship, and a nested entity map -that maps `CustOrder` objects to the role that concept plays in the `order` relationship. +This mapping contains a single entity map with two role mappings -- a value map that maps the +`L_LINENUMBER` field to the second role of the `nr` relationship, and a nested entity map that +maps `CustOrder` objects to the second role in the `order` relationship. #### Links (mappings that determine the existence of relationship links) -Each of an ontology's relationships is grouped under the concept that plays its first role. -The concept's `links` array declares schema mappings from field values to links of these -relationships. Each element in the array is a tree structure that concisely declares how -to map to tuples that form the links of one or more of these relationships. The path from -the root of a tree to each node describes how to map to tuples of objects that form the -links of the relationship that is named by the node. These structures leverage the hierarchical -nature of YAML to prevent the duplication in the typical case when the fields of a single -dataset map to many relationships. +Each relationships is grouped under the concept that plays its first role. A concept's +`links` array declares schema mappings from field values to links of these relationships. +Each array element is a tree that concisely declares how to map fields to the links of +one or more of these relationships. More precisely, the path from the root of a tree +to a node describes how to map to tuples of objects that form the links of the relationship +that is named by the node. These structures leverage the hierarchical nature of YAML to +avoid duplication in the typical case when the fields of a single dataset map to many +relationships. Each tree node has the following schema: @@ -808,17 +805,16 @@ Each tree node has the following schema: |-------|------|----------|-------------| | `concept` | string | when level > 1 | Concept whose objects are looked up by this node | | `value` | string | if no `entity` | Expression that computes a value (when concept is a value type) | -| `entity` | list | if no `value` | Entity map that looks up an object (when concept is an entity type) | -| `relationship` | string | No | Relationship whose links are mapped to by this level in the tree | +| `entity` | list | if no `value` | Entity map that looks up an entity (when concept is an entity type) | +| `relationship` | string | No | Relationship whose links are mapped to by the path to this node | | `children` | list | No | List of child nodes in the tree | - Each node must provide either a `value` or an `entity` by which to look up objects but never both. -The level of each node coincides with the arity of the associated relationship. So a top-level -(root) node could map to a unary relationship, a node at level 2 could map to a binary -relationship, and so forth. +The level of each node coincides with the arity of the associated relationship. So a root node +could map to a unary relationship, a node at level 2 could map to a binary relationship, and so +forth. -For instance, the relationship mapping declared here: +For instance, the `links` array declared here: ```yaml ontology: @@ -868,12 +864,11 @@ ontology: relationship: Item.returned_in_for ``` -describes a tree with one root node, one node at level 2, and 2 nodes at level 3. Each node -maps fields of the `METRICS` dataset to links of four different relationships, and notice -how the mapping to `Item` objects is declared once even though `Item` plays a role in all -four of the relationships and that the mapping to `Store` objects is declared once even -though `Store` plays a role in three of the relationships. - +describes a tree with one root node, one node at level 2, and two nodes at level 3. +Each node maps fields of the `METRICS` dataset to links of four different relationships, +and notice how the mapping to `Item` objects is declared once even though `Item` plays a +role in all four of the relationships and that the mapping to `Store` objects is declared +once even though `Store` plays a role in three of the relationships. --- @@ -1016,11 +1011,11 @@ ai_context: ## Version History -- **0.1.2** (2026-05-08): Support for both logical and conceptual modeling layers - - Core conceptual model structure +- **0.1.2** (2026-05-12): Support for both logical and conceptual modeling layers (ontologies) + - Core ontology structure - Support for concepts, relationships, and logical -> conceptual schema mappings - - Renamed relationships in the logical (semantic) layer to join paths to avoid - conflict with relationships in the conceptual layer + - Renamed relationships in the logical layer to join paths to avoid conflict with + relationships in the conceptual layer - **0.1.1** (2025-12-11): Initial release - Core semantic model structure diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 6595d82..021ef41 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -1,5 +1,5 @@ # OSI - Core Metadata Spec (YAML Schema) -version: 0.1.1 +version: 0.1.2 # # Goals: # - Standardization: Establish uniform language and structure for semantic model definitions @@ -249,11 +249,11 @@ ontology: # Optional: Array of expressions that constrain the population of this concept requires: [] - # Optional: Array of entity mappings that map logical fields to concept objects - entity_mappings: [] + # Optional: Array of entity maps that map logical fields to entities (when concept is an entity type) + entities: [] - # Optional: Array of relationship mappings that map logical fields to relationship links - relationship_mappings: [] + # Optional: Array of link trees that map logical fields to relationships + links: [] relationships: @@ -287,11 +287,12 @@ roles: # Optional: Role name name: string -entity_mappings: +entities: - - entity_map: [] + # Array of entity maps + - entity: [] -entity_map: +entity: # Required: String naming the idenfier relationship whose role is mapped to using either the # expr if the role is played by a value type or the nested entity map if the role is played @@ -299,27 +300,31 @@ entity_map: - role: string # Optional: SQL expression whose value maps to the role - expr: string + value: string # Optional: Nested entity map that returns an entity object that maps to the role - entity_map: [] + entity: [] -relationship_mappings: +links: - - relationship_map: [] + - links_tree_node: [] -relationship_map: +links_tree_node: - # Optional: Name of the concept that plays this role in the relationship map + # Name of the concept whose objects are looked up by this node. + # Optional when the node is the root of a tree. - concept: string - # Optional: Maps fields to this role when the player is a value type - expr: string + # Optional: Maps fields to a value (when concept is a value type) + # Must be present when entity is absent + value: string - # Optional: Maps fields to this role when the player is an entity type - entity_map: [] + # Optional: Maps fields to an entity (when concept is an entity type) + # Must be present when value is absent + entity: [] # Optional: Name of relationship whose links are mapped to by this level in the tree relationship: string - relationship_mappings: [] \ No newline at end of file + # Optional: Array of links_tree_node children of this node + children: [] \ No newline at end of file From d3161399f75579a71459155a8d89888539f9e6ad Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Tue, 12 May 2026 17:02:35 -0400 Subject: [PATCH 06/89] Small fix --- core-spec/spec.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/core-spec/spec.md b/core-spec/spec.md index c37b0fd..f17ea6b 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -791,13 +791,12 @@ maps `CustOrder` objects to the second role in the `order` relationship. #### Links (mappings that determine the existence of relationship links) Each relationships is grouped under the concept that plays its first role. A concept's -`links` array declares schema mappings from field values to links of these relationships. -Each array element is a tree that concisely declares how to map fields to the links of -one or more of these relationships. More precisely, the path from the root of a tree -to a node describes how to map to tuples of objects that form the links of the relationship -that is named by the node. These structures leverage the hierarchical nature of YAML to -avoid duplication in the typical case when the fields of a single dataset map to many -relationships. +`links` array maps field schema to relationships. Each array element is a tree that +concisely declares how field values map to the links of one or more of these relationships. +More precisely, the path from the root of a tree to a node describes how to map to tuples +of objects that form the links of the relationship that is named by the node. These structures +leverage the hierarchical nature of YAML to avoid duplication in the typical case when the +fields of a single dataset map to many relationships. Each tree node has the following schema: @@ -842,12 +841,12 @@ ontology: entities: - entity: - role: Item.nr - value_map: ITEMS.SKU + value: ITEMS.SKU links: - entity: - role: Item.nr - value_map: METRICS.SKU + value: METRICS.SKU relationship: Item.active children: - concept: Store From 5ce7e1d09841d6a13a6995d8d76bbd9a886f9222 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Tue, 12 May 2026 18:59:39 -0400 Subject: [PATCH 07/89] Fixed some typos --- core-spec/spec.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core-spec/spec.md b/core-spec/spec.md index f17ea6b..aba6572 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -45,7 +45,7 @@ The allowable multiplicities of relationships defined in the [Ontology](#ontolog | Multiplicity | Description | |---------|-------------| | `ManyToOne` | The last role of a relationship is uniquely determined by the other roles | -| `OneToOne` | The relationship is ManyToMany in both directions (only for binary relationships) | +| `OneToOne` | The relationship is ManyToOne in both directions (only for binary relationships) | ### Vendors @@ -428,7 +428,7 @@ Enterprise data are often modeled at both the logical level, in terms of dataset and the conceptual level in the form of an ontology that comprises concepts, relationships, and business rules. This section describes how to declare an ontology and schema mappings from logical-level fields to concepts and relationships in the ontology. Ontologies are -organized hierarchically in top-level collection of tree structures, each root of which +organized hierarchically in a top-level collection of tree structures, each root of which is a concept, under which relationships and schema mappings are defined. ### Concepts @@ -450,7 +450,7 @@ Concepts conform to the following schema: | `identify_by` | list | No | Names of relationships that uniquely reference objects of this concept | | `requires` | list | No | Expressions that constrain this concept's population | | `entities` | list | No | Mappings that determine this concept's objects (when an entity type) | -| `links` | list | No | Mappings from field values to relatioship links | +| `links` | list | No | Mappings from field values to relationship links | ### Extends From 6ad8e0f83751442762efefafd4490ca07c63da0f Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Wed, 20 May 2026 16:20:25 -0400 Subject: [PATCH 08/89] Reverting previous changes to core spec in favor of a new first-class ontology spec that refs the core spec so that both may evolve independently. --- core-spec/spec.md | 499 ++--------------------------------------- core-spec/spec.yaml | 117 +--------- ontology/ontology.json | 352 +++++++++++++++++++++++++++++ 3 files changed, 372 insertions(+), 596 deletions(-) create mode 100644 ontology/ontology.json diff --git a/core-spec/spec.md b/core-spec/spec.md index aba6572..3798251 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -1,6 +1,6 @@ # OSI - Core Metadata Specification -**Version:** 0.1.2 +**Version:** 0.1.1 ## Goals @@ -13,11 +13,10 @@ 1. [Enumerations](#enumerations) 2. [Semantic Model](#semantic-model) 3. [Datasets](#datasets) -4. [Join paths](#join-paths) +4. [Relationships](#relationships) 5. [Fields](#fields) 6. [Metrics](#metrics) -7. [Ontology](#ontology) -8. [Complete Example](#complete-example) +7. [Examples](#examples) --- @@ -38,15 +37,6 @@ Supported SQL and expression language dialects for metrics and field definitions | `DATABRICKS` | Databricks SQL | | `MAQL` | GoodData MAQL (Metric Analysis and Query Language) | -### Multiplicities - -The allowable multiplicities of relationships defined in the [Ontology](#ontology) section. - -| Multiplicity | Description | -|---------|-------------| -| `ManyToOne` | The last role of a relationship is uniquely determined by the other roles | -| `OneToOne` | The relationship is ManyToOne in both directions (only for binary relationships) | - ### Vendors Supported vendors for custom extensions and integrations. @@ -62,7 +52,7 @@ Supported vendors for custom extensions and integrations. ## Semantic Model -The top-level container that represents a complete semantic model, including datasets, join paths, and metrics. +The top-level container that represents a complete semantic model, including datasets, relationships, and metrics. ### Schema @@ -72,7 +62,7 @@ The top-level container that represents a complete semantic model, including dat | `description` | string | No | Human-readable description | | `ai_context` | string/object | No | Additional context for AI tools (e.g., custom instructions) | | `datasets` | array | Yes | Collection of logical datasets (fact and dimension tables) | -| `join_paths` | array | No | Defines how logical datasets are connected | +| `relationships` | array | No | Defines how logical datasets are connected | | `metrics` | array | No | Quantifiable measures defined as aggregate expessions on fields from logical datsets | | `custom_extensions` | array | No | Vendor-specific attributes for extensibility | @@ -85,7 +75,7 @@ semantic_model: ai_context: instructions: "Use this model for sales analysis and customer insights" datasets: [] - join_paths: [] + relationships: [] metrics: [] custom_extensions: - vendor_name: DBT @@ -96,7 +86,7 @@ semantic_model: ## Datasets -Logical datasets represent business entities (fact and dimension tables). They contain fields and define the structure of the data. +Logical datasets represent business entities or concepts (fact and dimension tables). They contain fields and define the structure of the data. ### Schema @@ -153,17 +143,17 @@ datasets: --- -## Join paths +## Relationships -Join paths define how logical datasets are connected through foreign key constraints. They support both simple and composite keys. +Relationships define how logical datasets are connected through foreign key constraints. They support both simple and composite keys. ### Schema | Field | Type | Required | Description | |-------|------|----------|-------------| -| `name` | string | Yes | Unique identifier for the join path | -| `from` | string | Yes | The logical dataset on the many side of the join path | -| `to` | string | Yes | The logical dataset on the one side of the join path | +| `name` | string | Yes | Unique identifier for the relationship | +| `from` | string | Yes | The logical dataset on the many side of the relationship | +| `to` | string | Yes | The logical dataset on the one side of the relationship | | `from_columns` | array | Yes | Array of column names in the "from" dataset (foreign key columns) | | `to_columns` | array | Yes | Array of column names in the "to" dataset (primary or unique key columns) | | `ai_context` | string/object | No | Additional context for AI tools | @@ -173,12 +163,12 @@ Join paths define how logical datasets are connected through foreign key constra - The order of columns in `from_columns` must correspond to the order in `to_columns` - Both arrays must have the same number of columns -- For simple join paths, use a single column: `[column1]` -- For composite join paths, use multiple columns: `[column1, column2]` +- For simple relationships, use a single column: `[column1]` +- For composite relationships, use multiple columns: `[column1, column2]` ### Examples -**Simple Join Path:** +**Simple Relationship:** ```yaml - name: orders_to_customers @@ -188,7 +178,7 @@ Join paths define how logical datasets are connected through foreign key constra to_columns: [id] ``` -**Composite Join Path:** +**Composite Relationship:** ```yaml # order_lines.product_id = products.id AND order_lines.variant_id = products.variant_id @@ -422,455 +412,6 @@ custom_extensions: --- -## Ontology - -Enterprise data are often modeled at both the logical level, in terms of datasets and fields, -and the conceptual level in the form of an ontology that comprises concepts, relationships, -and business rules. This section describes how to declare an ontology and schema mappings -from logical-level fields to concepts and relationships in the ontology. Ontologies are -organized hierarchically in a top-level collection of tree structures, each root of which -is a concept, under which relationships and schema mappings are defined. - -### Concepts - -Concepts represent the types of things that have meaning in a business setting, e.g., person, company, -or salary. Each concept is either an entity type or a value type. Ontologies implicitly include a -value-type for each basic data type, like `Integer`, `Decimal`, and `String`, and an entity type -called `Any`. Every other concept ontology extends (is a subtype of) one of these concepts. - -Concepts conform to the following schema: - -| Field | Type | Required | Description | -|-------|------|----------|-------------| -| `concept` | string | Yes | Unique name of this concept | -| `description` | string | No | Human-readable description | -| `relationships` | list | No | Relationships where this concept plays the first role | -| `extends` | list | No | Names of this concept's supertypes | -| `derived_by` | list | No | Expressions that derive this concept's population | -| `identify_by` | list | No | Names of relationships that uniquely reference objects of this concept | -| `requires` | list | No | Expressions that constrain this concept's population | -| `entities` | list | No | Mappings that determine this concept's objects (when an entity type) | -| `links` | list | No | Mappings from field values to relationship links | - -### Extends - -Every user-declared concept extends one or more concepts in the ontology. The new concept -is a sutype of each concept that it extends, and the extended concepts are its supertypes. - -Any concept that directly or indirectly extends a value type like `Integer` or `String` is a value type. -Any concept that does not extend some value type is an entity type, and if a concept declares no extends -list, then it is assumed to extend the built-in entity type `Any`. If `SocialSecurityNr` extends `Integer` -and `Employee` extends `Person`, which declares no extends list, then `SocialSecurityNr` is a value type -and both `Person` and `Employee` are entity types. - -### Relationships - -Relationships relate objects of one or more concepts and declare how to verbalize links among -those objects. Relationships have set (as opposed to bag) semantics, and links do not contain -nulls. - -Each relationship that is declared under a concept conforms to the following schema: - -| Field | Type | Required | Description | -|-------|------|----------|-------------| -| `name` | string | Yes | Part of the identifier for this relationship | -| `description` | string | No | Human-readable description | -| `multiplicity` | enum | No | Multiplicity constraint | -| `roles` | list | No | List of additional roles in this relationship | -| `derived_by` | list | No | Expressions that derive links of this relationship | -| `requires` | list | No | Expressions that constrain this relationship's population | -| `verbalizes` | list | Yes | Patterns describing how to verbalize links | - -Each relationship is uniquely identified by a prepending its declared name with that of the containing -concept. For instance, in: - -```yaml -ontology: - - concept: Person - relationships: - - name: earns - roles: - - player: Salary - multiplicity: ManyToOne - verbalizes: [ "{Person} earns {Salary}" ] - ... -``` - -the relationship is identified by the string `Person.earns`. This convention naturally supports -expressions that navigate over the links of relationships using the “dot-join” operator in a -manner that is familiar to object-oriented programming languages. This relationship links -`Person` and `Salary` objects and verbalizes each link as “Person earns Salary.” - -#### Roles - -Objects play roles in the links of a relationship. If you think of a relationship as a narrow table, -then its links are like rows and its roles are like columns. Each role is played by a concept that -constrains the type of objects that can play that role in any link. In `Person.earns`, `Person` and -`Salary` play the first and second roles respectively. - -By convention, the first role of any relationship is played by the concept under which the -relationship is declared. Any additional roles are enumerated in order in the roles list -using this schema: - -| Field | Type | Required | Description | -|-------|------|----------|-------------| -| `player` | string | Yes | Name of the concept that plays this role | -| `name` | string | No | Optional role name | - -For instance, in: - -```yaml -ontology: - - concept: Person - relationships: - - name: files_married_joint - verbalizes: [ "{Person} files married filing joint" ] - - name: purchased_on - roles: - - player: Vehicle - - player: Date - multiplicity: ManyToOne - verbalizes: [ "{Person} puchased {Vehicle} on {Date}" ] -``` - -the unary relationship `Person.files_married_joint` has an empty roles list, while the -ternary relationship `Person.purchased_on` declares two additional roles played by -`Vehicle` and `Date` respectively, - -The role player often suffices to distinguish the role within its relationship, but when -the same concept plays more than one role, the user must declare a distinguising name for -any additional role whose player does not suffice to distinguish it from other roles in -the same relationship. For instance, in: - -```yaml -ontology: - - concept: Store - relationships: - - name: ships_to_in_days - roles: - - player: Store - name: destination - - player: NrDays - multiplicity: ManyToOne - verbalizes: [ "{Store} ships to {Store:destination} in {NrDays}" ] -``` - -the role name `destination` distinguishes the second `Store`-playing role from the first in -this relationship. - -Expressions that are used to define derived_by rules and requires constraints will refer to -roles by name -- the name defaulting to the concept that plays the role unless an explicit -role name is provided. In any expression that involving links of the `Store.ships_to_in_days` -relationship can then use the variables `Store` and `destination` to refer to objecs that -play these two `Store`-playing roles without ambiguity. - -#### Multiplicities - -If a relationship comprises more than one role, objects that play the last role could be functionally -dermined by a tuple of objects that play the other roles. This knowledge is declared using a `ManyToOne` -multiplicity constraint. In the examples above, the constraint declares that each person earns at most -one salary and that for each pair of stores, the former ships to the latter in at most one number of -days. For relationships of ternary and higher arity, the multiplicity applies to the n-th role, meaning -the object that plays the n-th role is functionally determined by the tuple of objects that play -the first n-1 roles. - -In the special case of a binary relationship, one might declare a `OneToOne` multiplicity, which -indicates the relationship is many-to-one in both directions. For instance, the `Person.nr` -relationship is one-to-one because each person is assigned at most one social security number -and each social security number is assigned to at most one person. - -In the absence of any multiplicity, we make no assumptions of functional dependencies among -any of the roles. - -### Identifying relationships - -Many conceptual models distinguish one or more relationships to use when referencing entity-type -objects in expressions and queries. The `Person.nr` relationship can be used to reference a -person by their social security number; while the pair of relationships `License.acct` and -`License.seat_nr` can be used to reference a license by its associated account and seat number. -These identifier relationships are always binary, and their first role is always played by the -concept the relationship is used to reference. - -### Derivation expressions - -Concepts and relationships may be derived using expressions. Think of a derived concept or -relationship as a conceptual view whose objects or links are derived from those of other -concepts or relationships. For instance: - -```yaml -ontology: - - concept: Person - relationships: - - name: parent_of - roles: - - player: Person - name: "child" - verbalizes: [ "{Person} is a parent of {Person:child}", "{Person:child} is a child of {Person}" ] - - name: ancestor_of - roles: - - player: Person - name: "descendant" - derived_by: - - "Person.parent_of(descendant)" - "Person.ancestor_of.parent_of(descendant)" - taxed_at: - roles: - - player: TaxRate - derived_by: - - "10.0 WHERE ( Person.files_single AND Person.earns <= 11925 )" - - "10.0 WHERE ( Person.files_married_joint AND Person.earns <= 23850 )" - - ... -``` - -declares two derived relationships -- `ancestor_of` and `taxed_at`. Each link of `Person.ancestor_of` -relates a person to one of its descendants. The two expressions form the base and recursive cases for -this calculation. In the base case, a `Person` as an ancestor of some `descendant` if that `Person` -is the parent of that descendant. And in the recursive case, a `Person` is an ancestor of some -`descendant` if that `Person` is an ancestor of the parent of that `descendant`. Notice in this -example how role names are used to disambiguate the two `Person` roles in this relationship. - -Each link of `Person.taxed_at` links a `Person` object to a `TaxRate` that is derived using -expressions that determine the rate based on the person's filing status and how much they earn. -If, for some person, none of the expressions can be evaluated, then the relationship will have -no link involving that person. - -Expressions that derive a relationship are interpreted as rules for constructing the links of the -relationship in the same way that a SQL query is interpreted as a rule for constructing the rows -of a new table. Each expression must therefore reference each role of the relationship, either -explicitly or implicitly. If an expression evaluates to some object (like 10.0 in the two examples -here) then that object will implicitly play the last role, and the expression must reference each -of the other roles explicitly. If an expression does not evaluate to any object, then it must -explicitly reference each role. - -A derived concept is one whose population is derived from those of its supertype concepts -using one or more expressions. For instance: - -```yaml -ontology: - - concept: Employee - extends: [Person] - derived_by: [ "EXISTS ( Person.earns )" ] -``` - -declares that the population of Employee is derived from the population of Person by -classifying each Person who earns some salary as a Employee. - -### Requires - -The requires list contains expressions that give additional semantics to a concept or relationship -by declaring conditions that must hold over their populations. When applied to a concept, each -expression must reference the concept, as in: - -```yaml -ontology: - - concept: SocialSecurityNr - extends: [Integer] - requires: [ "0 < SocialSecurityNr", "SocialSecurityNr <= 999999999" ] -``` - -When applied to a relationship, each expression must reference one or more roles of the -relationship. For instance, in: - -```yaml -ontology: - - concept: Item - extends: [Integer] - relationships: - - name: offers_in - roles: - - player: Store - verbalizations: [ "{Item} is offered for sale in {Store}", "{Store} offers sale of {Item}" ] - - name: total_sales_in - roles: - - player: Store - - player: Amount - verbalizations: [ "{Item} sold for cumulative {Amount} in {Store}" ] - requires: - - "Amount > 0.0" - - "Item.offers_in(Store)" -``` - -the first expression requires any value that plays the `Amount` role to be positive while the second -requires any item that has sales in some store to be offered in that store. - -### Mappings - -Logical to conceptual schema mappings declare how to map field values to conceptual objects -and links. The most basic kind of mapping is a value map: - -| Field | Type | Required | Description | -|--------------|----------|-----|-------| -| `role` | string | Yes | Expression that indicates the role that the mapped objects will play | -| `value` | string | Yes | Expression that calculates the value to map to the role | - -which declares how to map a SQL expression that computes a value from one or more fields to a -role of some relationship, where the role must be played by a value type. - -Entity-type objects (entities) are opaque and must be mapped to via their identifying -relationships. These relationships are always binary, with the first role played by -the entity type itself. In the common case where an entity type is identified by a -single relationship whose second role is played by a value type, a value map suffices -to declare the existence of entities of that type or to look them up when mapping to -links of a relationship. - -An entity map is an array of identifier maps: - -| Field | Type | Required | Description | -|--------------|----------|-----|-------| -| `role` | string | Yes | Expression that indicates the role the mapped objects will play | -| `value` | string | if no `entity` | Expression that maps to a value (when role played by a value type) | -| `entity` | list | if no `value` | Entity map that maps to an entity (when role played by an entity type) | - -each of which is either a value map or a nested entity map that looks up an object to map to some -role of an identifier relationship. - -Building on value and entity maps, we can declare precisely how fields determine the entities of -a concept and the links of every relationship in which that concept plays the first role. - -#### Entities (entity maps that determine the existence of objects of some entity type) - -When a concept is an entity type, it can declare an `entities` array, each element of which -is an entity map. - -For instance, in: - -```yaml -ontology: - - concept: Person - relationships: - - name: nr - roles: - - player: SocialSecurityNr - multiplicity: OneToOne - verbalizes: [ "{Person} is identified by {SocialSecurityNr}" ] - identify_by: [ nr ] - - entities: - - entity: - - role: Person.nr - value: PERSONS.SSN - ... -``` - -uses a value map to declare how values from the `SSN` field of dataset `PERSONS` are used -to form `SocialSecurityNr` values that map to the second role of the `nr` relationship. -Because each link in that relationship associates a `SocialSecurityNr` value to some -unique `Person` entity, this value map suffices to associate each distinct `SSN` value -in the dataset to a distinct `Person` object in the ontology. - -A more interesting example maps field values to `OrderLineItem` entities, which are -identified using two relationships: - -```yaml - - concept: OrderLineItem - relationships: - - name: nr - roles: [ player: LineNr ] - multiplicity: ManyToOne - - name: order - roles: [ player: CustOrder ] - multiplicity: ManyToOne - identify_by: [ "nr", "order" ] - requires: [ "OrderLineItem.nr", "OrderLineItem.order" ] - - entities: - - entity: - - role: OrderLineItem.nr - value: LINEITEMS.L_LINENUMBER - - role: OrderLineItem.order - entity: - - role: CustOrder.nr - value: LINEITEMS.L_ORDERKEY - -``` - -This mapping contains a single entity map with two role mappings -- a value map that maps the -`L_LINENUMBER` field to the second role of the `nr` relationship, and a nested entity map that -maps `CustOrder` objects to the second role in the `order` relationship. - -#### Links (mappings that determine the existence of relationship links) - -Each relationships is grouped under the concept that plays its first role. A concept's -`links` array maps field schema to relationships. Each array element is a tree that -concisely declares how field values map to the links of one or more of these relationships. -More precisely, the path from the root of a tree to a node describes how to map to tuples -of objects that form the links of the relationship that is named by the node. These structures -leverage the hierarchical nature of YAML to avoid duplication in the typical case when the -fields of a single dataset map to many relationships. - -Each tree node has the following schema: - -| Field | Type | Required | Description | -|-------|------|----------|-------------| -| `concept` | string | when level > 1 | Concept whose objects are looked up by this node | -| `value` | string | if no `entity` | Expression that computes a value (when concept is a value type) | -| `entity` | list | if no `value` | Entity map that looks up an entity (when concept is an entity type) | -| `relationship` | string | No | Relationship whose links are mapped to by the path to this node | -| `children` | list | No | List of child nodes in the tree | - -Each node must provide either a `value` or an `entity` by which to look up objects but never both. -The level of each node coincides with the arity of the associated relationship. So a root node -could map to a unary relationship, a node at level 2 could map to a binary relationship, and so -forth. - -For instance, the `links` array declared here: - -```yaml -ontology: - - concept: Item - relationships: - - name: nr - roles: [ concept: SkuNr ] - multiplicity: OneToOne - verbalises: "{Item} is identified by {SkuNr}" - - name: active # A unary relationship - verbalizes: [ "{Item} is actively sold" ] - - name: active_in - roles: [ concept: Store ] - verbalises: [ "{Item} is actively sold in {Store}" ] - - name: returned_in_for - roles: [ concept: Store, concept: Amount ] - verbalizes: [ "{Item} returned in {Store} for {Amount}" ] - multiplicitly: ManyToOne - - name: sold_in_for - roles: [ concept: Store, concept: Amount ] - verbalizes: [ "{Item} sells in {Store} for {Amount}" ] - multiplicitly: ManyToOne - identify_by: [ nr ] - - entities: - - entity: - - role: Item.nr - value: ITEMS.SKU - - links: - - entity: - - role: Item.nr - value: METRICS.SKU - relationship: Item.active - children: - - concept: Store - entity: - - role: Store.nr - value: METRICS.STORE - relationship: Item.active_in - children: - - concept: Amount - value: METRICS.SALES - relationship: Item.sold_in_for - - concept: Amount - value: METRICS.RETURNS - relationship: Item.returned_in_for -``` - -describes a tree with one root node, one node at level 2, and two nodes at level 3. -Each node maps fields of the `METRICS` dataset to links of four different relationships, -and notice how the mapping to `Item` objects is declared once even though `Item` plays a -role in all four of the relationships and that the mapping to `Store` objects is declared -once even though `Store` plays a role in three of the relationships. - ---- - ## Complete Example Here's a complete semantic model example showing all components working together: @@ -937,7 +478,7 @@ semantic_model: expression: email description: Customer email - join_paths: + relationships: - name: orders_to_customers from: orders to: customers @@ -1010,12 +551,6 @@ ai_context: ## Version History -- **0.1.2** (2026-05-12): Support for both logical and conceptual modeling layers (ontologies) - - Core ontology structure - - Support for concepts, relationships, and logical -> conceptual schema mappings - - Renamed relationships in the logical layer to join paths to avoid conflict with - relationships in the conceptual layer - - **0.1.1** (2025-12-11): Initial release - Core semantic model structure - Support for datasets, relationships, fields, and metrics diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 021ef41..d8b5545 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -1,5 +1,5 @@ # OSI - Core Metadata Spec (YAML Schema) -version: 0.1.2 +version: 0.1.1 # # Goals: # - Standardization: Establish uniform language and structure for semantic model definitions @@ -20,9 +20,6 @@ dialects: - "MAQL" # GoodData MAQL (Multi-Dimensional Analytical Query Language) -multiplicities: - - "ManyToOne" - - "OneToOne" # Supported vendors for custom extensions vendors: @@ -51,10 +48,7 @@ semantic_model: # Optional: Defines how logical datasets are connected # See Relationships section below for detailed structure - join_paths: [] - - # Optional: Defines the ontology used to give meaning to the data - ontology: [] + relationships: [] # Optional: # These metrics can span one or more logical datasets and use relationships @@ -124,7 +118,7 @@ datasets: # Relationship Schema # Defines how logical datasets or semantic models are connected # Represents foreign key relationships (many-to-one or one-to-one) -join_paths: +relationships: # Required: Unique identifier for the relationship - name: string @@ -223,108 +217,3 @@ metrics: custom_extensions: - vendor_name: string # Must be one of the values from 'vendors' enum above data: string - ---- -# Ontology Schema -ontology: - - # Required: Unique identifier for the concept - - concept: string - - # Optional: Human-readable description of the concept - description: string - - # Optional: Relationships in which this concept plays the first role - relationships: [] - - # Optional: Array of names of relationships that form the preferred identifier of this concept - identify_by: [] - - # Optional: Array of names of supertype concepts that this concept extends - extends: [] - - # Optional: Array of expressions that derive the population of this concept - derived_by: [] - - # Optional: Array of expressions that constrain the population of this concept - requires: [] - - # Optional: Array of entity maps that map logical fields to entities (when concept is an entity type) - entities: [] - - # Optional: Array of link trees that map logical fields to relationships - links: [] - -relationships: - - # Required: unique name of the relationship in the context of the declaring concept - - name: string - - # Optional: Human-readable description - description: string - - # Optional: Multiplicity constraint - multiplicity: multiplicities - - # Optional: List of additional roles in this relationship - roles: [] - - # Optional: Array of expressions that derive links of this relationship - derived_by: [] - - # Optional: Array of expressions that constrain this relationship's population - requires: [] - - # Optional Array of string patterns that describe how to verbalize links of - # this relationship - verbalizes: [] - -roles: - - # Required: Name of the concept that plays this role - - player: string - - # Optional: Role name - name: string - -entities: - - # Array of entity maps - - entity: [] - -entity: - - # Required: String naming the idenfier relationship whose role is mapped to using either the - # expr if the role is played by a value type or the nested entity map if the role is played - # by an entity type - - role: string - - # Optional: SQL expression whose value maps to the role - value: string - - # Optional: Nested entity map that returns an entity object that maps to the role - entity: [] - -links: - - - links_tree_node: [] - -links_tree_node: - - # Name of the concept whose objects are looked up by this node. - # Optional when the node is the root of a tree. - - concept: string - - # Optional: Maps fields to a value (when concept is a value type) - # Must be present when entity is absent - value: string - - # Optional: Maps fields to an entity (when concept is an entity type) - # Must be present when value is absent - entity: [] - - # Optional: Name of relationship whose links are mapped to by this level in the tree - relationship: string - - # Optional: Array of links_tree_node children of this node - children: [] \ No newline at end of file diff --git a/ontology/ontology.json b/ontology/ontology.json new file mode 100644 index 0000000..fa9cfff --- /dev/null +++ b/ontology/ontology.json @@ -0,0 +1,352 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/ontology/ontology.json", + "title": "OSI Core Metadata Specification", + "description": "JSON Schema for validating OSI (Open Semantic Interoperability) ontology definitions", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "0.1.1", + "description": "Ontology specification version" + }, + "concept_types": { + "type": "array", + "description": "Concept types (enumeration definition)", + "items": { + "$ref": "#/$defs/ConceptType" + } + }, + "multiplicities": { + "type": "array", + "description": "Relationship multiplicities (enumeration definition)", + "items": { + "$ref": "#/$defs/Multiplicity" + } + }, + "ontologies": { + "type": "array", + "description": "Collection of ontologies", + "items": { + "$ref": "#/$defs/Ontology" + } + }, + "ontology_maps": { + "type": "array", + "description": "Collection of ontology maps from logical models", + "items": { + "$ref": "#/$defs/OntologyMap" + } + } + }, + "required": ["version", "ontologies"], + "additionalProperties": false, + "$defs": { + "Component": { + "type": "object", + "description": "Ontology component that defines a single concept and any relationships that are keyed primarily by that concept", + "properties": { + "name": { + "type": "string", + "description": "Name of the component" + }, + "description": { + "type": "string", + "description": "Human-readable description of the component" + }, + "concept": { + "$ref": "#/$defs/Concept" + }, + "relationships": { + "type": "array", + "items": { + "$ref": "#/$defs/Relationship" + }, + "description": "Defines relationships that pertain primarily to the concept defined in this component" + } + }, + "required": ["name"], + "additionalProperties": false + }, + "Expression": { + "type": "string", + "description": "ANSI SQL expression" + }, + "Relationship": { + "type": "object", + "description": "Relationship between concepts in the ontology", + "properties": { + "name": { + "type": "string", + "description": "Name of the relationship" + }, + "description": { + "type": "string", + "description": "Human-readable description of the relationship" + }, + "roles": { + "type": "array", + "items": { + "$ref": "#/$defs/Role" + }, + "description": "Additional roles in this relationship" + }, + "multiplicity": { + "$ref": "#/$defs/Multiplicity" + }, + "derived_by": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that define how this concept is derived" + }, + "verbalizes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Natural language expressions that verbalize this relationship" + } + }, + "required": ["name", "verbalizes"], + "additionalProperties": false + }, + "Concept": { + "type": "object", + "description": "Defines a concept in the ontology", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the concept" + }, + "type": { + "$ref": "#/$defs/ConceptType" + }, + "description": { + "type": "string", + "description": "Human-readable description of the concept" + }, + "extends": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Indicates that this concept extends one or more other concepts" + }, + "derived_by": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that define how this concept is derived" + }, + "identify_by": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Names of relationships to use as the preferred identifier of this concept" + }, + "requires": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that constrain the population of this concept" + } + }, + "required": ["name", "type"], + "additionalProperties": false + }, + "ComponentMap": { + "type": "object", + "description": "Mappings from logical model constructs to some ontology component", + "properties": { + "component": { + "type": "string", + "description": "Name of the ontology component whose concept and/or relationships are being mapped to" + }, + "entity_maps": { + "type": "array", + "items": { + "$ref": "#/$defs/EntityMap" + }, + "description": "Mappings from logical constructs that populate the concept in this component. Valid only when the concept is an entity type" + }, + "relationship_maps": { + "type": "array", + "items": { + "$ref": "#/$defs/RelationshipMap" + }, + "description": "Mappings from logical model relationships to ontology relationships pertaining to the mapped concept" + } + }, + "required": ["component"], + "additionalProperties": false + }, + "ConceptType": { + "type": "string", + "enum": [ "EntityType", "ValueType" ], + "description": "A concept is either an entity type or a value type" + }, + "EntityMap": { + "type": "object", + "description": "Maps logical-model constructs to the objects of some entity type in the ontology", + "properties": { + "entity": { + "type": "string", + "description": "Name of the entity type being mapped to" + }, + "referent_maps": { + "type": "array", + "items": { + "$ref": "#/$defs/ReferentMap" + }, + "description": "Maps logical-model constructs to referent relationships of this entity type" + } + }, + "required": ["entity"], + "additionalProperties": false + }, + "ReferentMap": { + "type": "object", + "description": "Mapping from logical model constructs to a relationship used to references some entity type in the ontology", + "properties": { + "relationship": { + "type": "string", + "description": "Name of referent relationship" + }, + "entity": { + "$ref": "#/$defs/EntityReference" + }, + "value": { + "$ref": "#/$defs/Expression" + } + }, + "required": ["relationship"], + "additionalProperties": false + }, + "Role": { + "type": "object", + "description": "Role in some relationship (the container)", + "properties": { + "player": { + "type": "string", + "description": "Name of the concept playing this role" + }, + "name": { + "type": "string", + "description": "Optional name of this role, used when the same concept plays multiple roles in the same relationship" + } + }, + "required": ["player"], + "additionalProperties": false + }, + "EntityReference": { + "type": "object", + "description": "Reference to an entity type in the ontology, used in relationship maps to indicate that the value of a logical model relationship should populate a relationship to an instance of this entity type", + "properties": { + "entity": { + "type": "string", + "description": "Name of the entity type being referenced" + }, + "referent_maps": { + "type": "array", + "items": { + "$ref": "#/$defs/ReferentMap" + }, + "description": "Maps logical-model constructs to referent relationships of this entity type" + } + }, + "required": ["entity"], + "additionalProperties": false + }, + "RelationshipMap": { + "type": "object", + "description": "Mapping from logical model constructs to the links of some relationship in the ontology", + "properties": { + "relationship": { + "type": "string", + "description": "Name of relationship being populated by this map" + }, + "entity": { + "$ref": "#/$defs/EntityReference" + }, + "value": { + "$ref": "#/$defs/Expression" + }, + "children": { + "type": "array", + "items": { + "$ref": "#/$defs/RelationshipMap" + }, + "description": "Relationship maps at the next level in this hierarchy" + } + }, + "additionalProperties": false + }, + "Multiplicity": { + "type": "string", + "enum": [ "ManyToOne", "OneToOne" ], + "description": "Relationship multiplicity" + }, + "OntologyMap": { + "type": "object", + "description": "Map from the constructs of some logical model to some ontology", + "properties": { + "name": { + "type": "string", + "description": "Name of this ontology map" + }, + "description": { + "type": "string", + "description": "Human-readable description of this ontology map" + }, + "ontology": { + "type": "string", + "description": "Name of the ontology being mapped to" + }, + "logical_model": { + "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/SemanticModel" + }, + "components": { + "type": "array", + "items": { + "$ref": "#/$defs/ComponentMap" + }, + "description": "Maps logical model constructs to some concept and its relationships in the ontology" + } + }, + "required": ["logical_model", "ontology", "components"], + "additionalProperties": false + }, + "Ontology": { + "type": "object", + "description": "Top-level container representing a complete ontology", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the ontology" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/AIContext" + }, + "components": { + "type": "array", + "items": { + "$ref": "#/$defs/Component" + }, + "minItems": 1, + "description": "Components that define the concepts and relationships in this ontology" + } + }, + "required": ["name", "components"], + "additionalProperties": false + } + } +} From 274fdaf94654c3182b59ea7add3cf6088c880c07 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Wed, 20 May 2026 23:25:33 -0400 Subject: [PATCH 09/89] Updated schema to reflect discussions with governance team --- examples/flights.yaml | 1103 ++++++++++++++++++++++++++++++++++++++++ ontology/ontology.json | 61 +-- ontology/ontology.md | 597 ++++++++++++++++++++++ validation/validate.py | 24 +- 4 files changed, 1740 insertions(+), 45 deletions(-) create mode 100644 examples/flights.yaml create mode 100644 ontology/ontology.md diff --git a/examples/flights.yaml b/examples/flights.yaml new file mode 100644 index 0000000..959db95 --- /dev/null +++ b/examples/flights.yaml @@ -0,0 +1,1103 @@ +version: 0.1.1 +ontologies: + - name: flights + description: Ontology for flight-related concepts and relationships + components: + - name: Example_Runway + concept: + name: Example_Runway + type: EntityType + identify_by: [ id ] + relationships: + - name: id + roles: + - player: String + verbalizes: [ '{Example_Runway} id {String}' ] + multiplicity: ManyToOne + - name: shapelength + roles: + - player: Float + verbalizes: [ '{Example_Runway} shapeLength {Float}' ] + multiplicity: ManyToOne + - name: airportid + roles: + - player: String + verbalizes: [ '{Example_Runway} airportId {String}' ] + multiplicity: ManyToOne + - name: length + roles: + - player: Float + verbalizes: [ '{Example_Runway} length {Float}' ] + multiplicity: ManyToOne + - name: geometry + roles: + - player: String + verbalizes: [ '{Example_Runway} geometry {String}' ] + multiplicity: ManyToOne + - name: designator + roles: + - player: String + verbalizes: [ '{Example_Runway} designator {String}' ] + multiplicity: ManyToOne + - name: airports + roles: + - player: Example_Airport + verbalizes: [ '{Example_Runway} airports {Example_Airport}' ] + multiplicity: ManyToOne + derived_by: [ 'Example_Runway.airportid == Example_Airport.airportid' ] + - name: Example_RouteAlertComment + concept: + name: Example_RouteAlertComment + type: EntityType + identify_by: [ "id" ] + relationships: + - name: id + roles: + - player: String + verbalizes: [ '{Example_RouteAlertComment} id {String}' ] + multiplicity: ManyToOne + - name: routealertid + roles: + - player: String + verbalizes: [ '{Example_RouteAlertComment} routeAlertId {String}' ] + multiplicity: ManyToOne + - name: comment + roles: + - player: String + verbalizes: [ '{Example_RouteAlertComment} comment {String}' ] + multiplicity: ManyToOne + - name: routealert + roles: + - player: Example_RouteAlert + verbalizes: [ '{Example_RouteAlertComment} routeAlerts {Example_RouteAlert}' ] + multiplicity: ManyToOne + derived_by: [ 'Example_RouteAlertComment.routealertid == Example_RouteAlert.routealertid' ] + - name: Example_Airport + concept: + name: Example_Airport + type: EntityType + identify_by: [ "airportid" ] + relationships: + - name: airportid + roles: + - player: String + verbalizes: [ '{Example_Airport} airportId {String}' ] + multiplicity: ManyToOne + - name: displayairportname + roles: + - player: String + verbalizes: [ '{Example_Airport} displayAirportName {String}' ] + multiplicity: ManyToOne + - name: displaycitymarketnamefull + roles: + - player: String + verbalizes: [ '{Example_Airport} displayCityMarketNameFull {String}' ] + multiplicity: ManyToOne + - name: flights + roles: + - player: Example_Flight + verbalizes: [ '{Example_Airport} flights {Example_Flight}' ] + derived_by: [ 'Example_Airport.airportid == Example_Flight.originairportid OR Example_Airport.airportid == Example_Flight.destairportid' ] +ontology_maps: + - name: flights_map + description: Example mapping of logical fields to ontology concepts and relationships + ontology: flights + logical_model: + name: Logical_Flights_Model + description: Logical model for flight data + datasets: + - name: Example_Runway_example_runway + source: PBACON_ATT_DB.GANDALF.EXAMPLE_RUNWAY + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(id AS VARCHAR) + - name: shape_length + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(shape_length AS DOUBLE) + - name: airport_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(airport_id AS VARCHAR) + - name: length + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(length AS DOUBLE) + - name: geometry + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(geometry AS VARCHAR) + - name: designator + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(designator AS VARCHAR) + - name: Example_RouteAlertComment_example_route_alert_comment + source: PBACON_ATT_DB.GANDALF.EXAMPLE_ROUTE_ALERT_COMMENT + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(id AS VARCHAR) + - name: alert_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(alert_id AS VARCHAR) + - name: type + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(type AS VARCHAR) + - name: created_by + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(created_by AS VARCHAR) + - name: title + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(title AS VARCHAR) + - name: comment + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(comment AS VARCHAR) + - name: created_at + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(created_at AS TIMESTAMP) + - name: Example_Aircraft_example_aircraft + source: PBACON_ATT_DB.GANDALF.EXAMPLE_AIRCRAFT + description: Aircraft data with basic properties. Full 2023 flight history available + for a subset of aircraft where the "Complete Flight History" property is "true". + fields: + - name: flight_count + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(flight_count AS INTEGER) + - name: serial_number + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(serial_number AS VARCHAR) + - name: mode_scode_hex + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(mode_scode_hex AS VARCHAR) + - name: title + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(title AS VARCHAR) + - name: number_of_seats + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(number_of_seats AS INTEGER) + - name: tail_num + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(tail_num AS VARCHAR) + - name: op_carrier_airline_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(op_carrier_airline_id AS VARCHAR) + - name: carrier_name + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(carrier_name AS VARCHAR) + - name: mode_scode + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(mode_scode AS INTEGER) + - name: complete_flight_history + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(complete_flight_history AS BOOLEAN) + - name: manufacturer + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(manufacturer AS VARCHAR) + - name: latest_flight_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(latest_flight_id AS VARCHAR) + - name: model + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(model AS VARCHAR) + - name: carrier_iata_code + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(carrier_iata_code AS VARCHAR) + - name: year_mfr + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(year_mfr AS VARCHAR) + - name: acquisition_date + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(acquisition_date AS TIMESTAMP) + - name: capacity_in_pounds + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(capacity_in_pounds AS INTEGER) + - name: Example_Airport_example_airport + source: PBACON_ATT_DB.GANDALF.EXAMPLE_AIRPORT + description: Airports with various geospatial properties. Useful for map examples. + Full 2023 flight history available for a subset of airports where the "Complete + Flight History" property is "true". + fields: + - name: airport_state_code + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(airport_state_code AS VARCHAR) + - name: arriving_flight_count + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(arriving_flight_count AS INTEGER) + - name: display_city_market_name_full + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(display_city_market_name_full AS VARCHAR) + - name: average_dep_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(average_dep_delay AS DOUBLE) + - name: longitude + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(longitude AS DOUBLE) + - name: airport_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(airport_id AS VARCHAR) + - name: airport_start_date + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(airport_start_date AS TIMESTAMP) + - name: airport_state_name + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(airport_state_name AS VARCHAR) + - name: daily_avg_dep_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(daily_avg_dep_delay AS VARCHAR) + - name: average_arr_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(average_arr_delay AS DOUBLE) + - name: display_airport_city_name_full + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(display_airport_city_name_full AS VARCHAR) + - name: daily_avg_arr_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(daily_avg_arr_delay AS VARCHAR) + - name: daily_count_of_flights + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(daily_count_of_flights AS VARCHAR) + - name: complete_flight_history + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(complete_flight_history AS BOOLEAN) + - name: display_airport_name + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(display_airport_name AS VARCHAR) + - name: airport + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(airport AS VARCHAR) + - name: departing_flight_count + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(departing_flight_count AS INTEGER) + - name: geopoint + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(geopoint AS VARCHAR) + - name: latitude + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(latitude AS DOUBLE) + - name: Example_Flight_example_flight + source: PBACON_ATT_DB.GANDALF.EXAMPLE_FLIGHT + description: Represents each individual commercial passenger flight in the US + in 2023. With geospatial and timeseries property examples. + fields: + - name: dep_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(dep_delay AS INTEGER) + - name: air_time + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(air_time AS DOUBLE) + - name: flight_number + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(flight_number AS VARCHAR) + - name: airline_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(airline_id AS VARCHAR) + - name: late_aircraft_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(late_aircraft_delay AS INTEGER) + - name: actual_elapsed_time + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(actual_elapsed_time AS DOUBLE) + - name: taxi_in + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(taxi_in AS INTEGER) + - name: scheduled_departure_timestamp + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(scheduled_departure_timestamp AS TIMESTAMP) + - name: dest_airport_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(dest_airport_id AS VARCHAR) + - name: arr_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(arr_delay AS INTEGER) + - name: cancelled + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(cancelled AS BOOLEAN) + - name: carrier_code + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(carrier_code AS VARCHAR) + - name: destination_display_airport_city_name_full + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(destination_display_airport_city_name_full AS VARCHAR) + - name: carrier_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(carrier_delay AS INTEGER) + - name: cancellation_code + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(cancellation_code AS VARCHAR) + - name: origin_latitude + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_latitude AS DOUBLE) + - name: destination_geopoint + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(destination_geopoint AS VARCHAR) + - name: flights + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(flights AS DOUBLE) + - name: destination_display_city_market_name_full + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(destination_display_city_market_name_full AS VARCHAR) + - name: distance + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(distance AS DOUBLE) + - name: flight_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(flight_id AS VARCHAR) + - name: origin_geopoint + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_geopoint AS VARCHAR) + - name: scheduled_arrival_timestamp + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(scheduled_arrival_timestamp AS TIMESTAMP) + - name: destination_latitude + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(destination_latitude AS DOUBLE) + - name: wheels_on_timestamp + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(wheels_on_timestamp AS TIMESTAMP) + - name: longitude_sensor_series_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(longitude_sensor_series_id AS VARCHAR) + - name: nas_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(nas_delay AS INTEGER) + - name: latitude_sensor_series_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(latitude_sensor_series_id AS VARCHAR) + - name: destination_longitude + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(destination_longitude AS DOUBLE) + - name: origin_airport_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_airport_id AS VARCHAR) + - name: origin_display_airport_name + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_display_airport_name AS VARCHAR) + - name: security_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(security_delay AS INTEGER) + - name: origin_display_airport_city_name_full + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_display_airport_city_name_full AS VARCHAR) + - name: origin_longitude + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_longitude AS DOUBLE) + - name: taxi_out + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(taxi_out AS INTEGER) + - name: origin_display_city_market_name_full + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_display_city_market_name_full AS VARCHAR) + - name: destination_airport + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(destination_airport AS VARCHAR) + - name: origin_airport + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_airport AS VARCHAR) + - name: date + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(date AS DATE) + - name: destination_display_airport_name + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(destination_display_airport_name AS VARCHAR) + - name: wheels_off_timestamp + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(wheels_off_timestamp AS TIMESTAMP) + - name: scheduled_elapsed_time + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(scheduled_elapsed_time AS INTEGER) + - name: tail_num + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(tail_num AS VARCHAR) + - name: departure_timestamp + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(departure_timestamp AS TIMESTAMP) + - name: carrier_name + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(carrier_name AS VARCHAR) + - name: route_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(route_id AS VARCHAR) + - name: weather_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(weather_delay AS INTEGER) + - name: diverted + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(diverted AS BOOLEAN) + - name: arrival_timestamp + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(arrival_timestamp AS TIMESTAMP) + - name: flight_title + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(flight_title AS VARCHAR) + - name: route_title + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(route_title AS VARCHAR) + - name: Example_Carrier_example_carrier + source: PBACON_ATT_DB.GANDALF.EXAMPLE_CARRIER + description: Carriers with derived fleet metrics from associated aircraft. Full + 2023 flight history available for a subset of carriers where the "Complete Flight + History" property is "true". + fields: + - name: carrier + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(carrier AS VARCHAR) + - name: average_manufacture_year + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(average_manufacture_year AS DOUBLE) + - name: std_dev_manufacture_year + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(std_dev_manufacture_year AS DOUBLE) + - name: carrier_name + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(carrier_name AS VARCHAR) + - name: daily_avg_arr_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(daily_avg_arr_delay AS VARCHAR) + - name: daily_count_of_flights + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(daily_count_of_flights AS VARCHAR) + - name: complete_flight_history + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(complete_flight_history AS BOOLEAN) + - name: total_capacity_in_pounds + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(total_capacity_in_pounds AS INTEGER) + - name: airline_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(airline_id AS VARCHAR) + - name: total_seats + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(total_seats AS INTEGER) + - name: aircraft_count + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(aircraft_count AS INTEGER) + - name: daily_avg_dep_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(daily_avg_dep_delay AS VARCHAR) + - name: Example_FlightSensor_example_flight_sensor + source: PBACON_ATT_DB.GANDALF.EXAMPLE_FLIGHT_SENSOR + description: 'The flight sensor holds any sensor readings for the flight object. + It is linked to flights as a sensor object and therefore analyses on the flight + object can automatically access time series on the flight sensor. ' + fields: + - name: series_name + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(series_name AS VARCHAR) + - name: units + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(units AS VARCHAR) + - name: flight_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(flight_id AS VARCHAR) + - name: title + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(title AS VARCHAR) + - name: flight_sensor_series_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(flight_sensor_series_id AS VARCHAR) + - name: unique_sensor_flight_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(unique_sensor_flight_id AS VARCHAR) + - name: Example_RouteAlert_example_route_alert + source: PBACON_ATT_DB.GANDALF.EXAMPLE_ROUTE_ALERT + description: Alerts generated based on historical route performance, structured + as an operational object type to demonstrate basic status and assignment patterns. + fields: + - name: normalized_deviation_metric + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(normalized_deviation_metric AS DOUBLE) + - name: normalized_weighted_change_metric + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(normalized_weighted_change_metric AS DOUBLE) + - name: percent_change + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(percent_change AS DOUBLE) + - name: priority + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(priority AS VARCHAR) + - name: assignee + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(assignee AS VARCHAR) + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(id AS VARCHAR) + - name: flight_count + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(flight_count AS INTEGER) + - name: alert_type + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(alert_type AS VARCHAR) + - name: risk_level + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(risk_level AS INTEGER) + - name: deviation_metric + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(deviation_metric AS DOUBLE) + - name: floor_threshold + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(floor_threshold AS DOUBLE) + - name: description_markdown + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(description_markdown AS VARCHAR) + - name: status + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(status AS VARCHAR) + - name: title + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(title AS VARCHAR) + - name: alert_period_length + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(alert_period_length AS INTEGER) + - name: percent_change_threshold + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(percent_change_threshold AS DOUBLE) + - name: route_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(route_id AS VARCHAR) + - name: metric_description + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(metric_description AS VARCHAR) + - name: description + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(description AS VARCHAR) + - name: previous_deviation_metric + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(previous_deviation_metric AS DOUBLE) + - name: investigation_notes + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(investigation_notes AS VARCHAR) + - name: alert_period_start + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(alert_period_start AS DATE) + - name: percent_change_display + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(percent_change_display AS DOUBLE) + - name: Example_Explainer_example_explainer + source: PBACON_ATT_DB.GANDALF.EXAMPLE_EXPLAINER + description: Stores re-usable callout references to integrate into Workshop and + other one state examples + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(id AS VARCHAR) + - name: link_url + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(link_url AS VARCHAR) + - name: text + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(text AS VARCHAR) + - name: title + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(title AS VARCHAR) + - name: link_text + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(link_text AS VARCHAR) + - name: Example_Log_UpdateRouteAlertStatus_example_log_update_route_alert_status + source: PBACON_ATT_DB.GANDALF.EXAMPLE_LOG_UPDATE_ROUTE_ALERT_STATUS + description: 'This object type was automatically created when logging was enabled + for the Action # type: Update Route Alert Status' + fields: + - name: action_triggerer + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(action_triggerer AS VARCHAR) + - name: status + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(status AS VARCHAR) + - name: action_rid + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(action_rid AS VARCHAR) + - name: action_type_rid + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(action_type_rid AS VARCHAR) + - name: action_type_version + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(action_type_version AS VARCHAR) + - name: action_summary + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(action_summary AS VARCHAR) + - name: action_timestamp + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(action_timestamp AS TIMESTAMP) + - name: Example_Route_example_route + source: PBACON_ATT_DB.GANDALF.EXAMPLE_ROUTE + description: Derived from flights between pairs of departure and arrival airports. + Demonstrates the use of deriving "intermediate" object types from granular data + as well as pre-calculating default metrics in a pipeline. Full 2023 flight history + available for a subset of routes where the "Complete Flight History" property + is "true". + fields: + - name: average_dep_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(average_dep_delay AS DOUBLE) + - name: destination_longitude + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(destination_longitude AS DOUBLE) + - name: origin_airport_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_airport_id AS VARCHAR) + - name: destination_airport_1 + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(destination_airport_1 AS VARCHAR) + - name: origin_display_airport_name + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_display_airport_name AS VARCHAR) + - name: daily_avg_dep_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(daily_avg_dep_delay AS VARCHAR) + - name: origin_display_airport_city_name_full + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_display_airport_city_name_full AS VARCHAR) + - name: average_arr_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(average_arr_delay AS DOUBLE) + - name: origin_longitude + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_longitude AS DOUBLE) + - name: daily_count_of_flights + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(daily_count_of_flights AS VARCHAR) + - name: daily_avg_arr_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(daily_avg_arr_delay AS VARCHAR) + - name: dest_airport_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(dest_airport_id AS VARCHAR) + - name: origin_airport + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_airport AS VARCHAR) + - name: destination_display_airport_city_name_full + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(destination_display_airport_city_name_full AS VARCHAR) + - name: origin_latitude + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_latitude AS DOUBLE) + - name: actual_elapsed_time_standard_deviation + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(actual_elapsed_time_standard_deviation AS DOUBLE) + - name: destination_display_airport_name + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(destination_display_airport_name AS VARCHAR) + - name: destination_geopoint + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(destination_geopoint AS VARCHAR) + - name: average_actual_elapsed_time + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(average_actual_elapsed_time AS DOUBLE) + - name: flights_count + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(flights_count AS INTEGER) + - name: distance + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(distance AS DOUBLE) + - name: distance_group + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(distance_group AS INTEGER) + - name: route_id + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(route_id AS VARCHAR) + - name: origin_geopoint + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(origin_geopoint AS VARCHAR) + - name: complete_flight_history + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(complete_flight_history AS BOOLEAN) + - name: destination_latitude + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(destination_latitude AS DOUBLE) + - name: route_title + expression: + dialects: + - dialect: ANSI_SQL + expression: CAST(route_title AS VARCHAR) + component_maps: + - component: Example_Runway + object_maps: + - referent: Example_Runway + identifier: + - relationship: id + object: + referent: String + expression: Example_Runway_example_runway.id + link_maps: + - maps: + referent: Example_Runway + identifier: + - relationship: id + object: + referent: String + expression: Example_Runway_example_runway.id + children: + - maps: + referent: Float + expression: Example_Runway_example_runway.length + populates: length + - maps: + referent: String + expression: Example_Runway_example_runway.geometry + populates: geometry + - maps: + referent: String + expression: Example_Runway_example_runway.designator + populates: designator + - maps: + referent: Float + expression: Example_Runway_example_runway.shape_length + populates: shape_length + - maps: + referent: String + expression: Example_Runway_example_runway.airport_id + populates: airport_id diff --git a/ontology/ontology.json b/ontology/ontology.json index fa9cfff..071b777 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -167,14 +167,14 @@ "type": "string", "description": "Name of the ontology component whose concept and/or relationships are being mapped to" }, - "entity_maps": { + "object_maps": { "type": "array", "items": { - "$ref": "#/$defs/EntityMap" + "$ref": "#/$defs/ObjectRef" }, "description": "Mappings from logical constructs that populate the concept in this component. Valid only when the concept is an entity type" }, - "relationship_maps": { + "link_maps": { "type": "array", "items": { "$ref": "#/$defs/RelationshipMap" @@ -190,25 +190,6 @@ "enum": [ "EntityType", "ValueType" ], "description": "A concept is either an entity type or a value type" }, - "EntityMap": { - "type": "object", - "description": "Maps logical-model constructs to the objects of some entity type in the ontology", - "properties": { - "entity": { - "type": "string", - "description": "Name of the entity type being mapped to" - }, - "referent_maps": { - "type": "array", - "items": { - "$ref": "#/$defs/ReferentMap" - }, - "description": "Maps logical-model constructs to referent relationships of this entity type" - } - }, - "required": ["entity"], - "additionalProperties": false - }, "ReferentMap": { "type": "object", "description": "Mapping from logical model constructs to a relationship used to references some entity type in the ontology", @@ -217,11 +198,8 @@ "type": "string", "description": "Name of referent relationship" }, - "entity": { - "$ref": "#/$defs/EntityReference" - }, - "value": { - "$ref": "#/$defs/Expression" + "object": { + "$ref": "#/$defs/ObjectRef" } }, "required": ["relationship"], @@ -243,38 +221,38 @@ "required": ["player"], "additionalProperties": false }, - "EntityReference": { + "ObjectRef": { "type": "object", - "description": "Reference to an entity type in the ontology, used in relationship maps to indicate that the value of a logical model relationship should populate a relationship to an instance of this entity type", + "description": "Pattern of logical-level expressions for identifying objects of some referent concept using the values in one or more fields", "properties": { - "entity": { + "referent": { "type": "string", - "description": "Name of the entity type being referenced" + "description": "Name of the referent concept" }, - "referent_maps": { + "identifier": { "type": "array", "items": { "$ref": "#/$defs/ReferentMap" }, "description": "Maps logical-model constructs to referent relationships of this entity type" + }, + "expression": { + "$ref": "#/$defs/Expression" } }, - "required": ["entity"], + "required": ["referent"], "additionalProperties": false }, "RelationshipMap": { "type": "object", "description": "Mapping from logical model constructs to the links of some relationship in the ontology", "properties": { - "relationship": { + "populates": { "type": "string", "description": "Name of relationship being populated by this map" }, - "entity": { - "$ref": "#/$defs/EntityReference" - }, - "value": { - "$ref": "#/$defs/Expression" + "maps": { + "$ref": "#/$defs/ObjectRef" }, "children": { "type": "array", @@ -284,6 +262,7 @@ "description": "Relationship maps at the next level in this hierarchy" } }, + "required": ["maps"], "additionalProperties": false }, "Multiplicity": { @@ -310,7 +289,7 @@ "logical_model": { "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/SemanticModel" }, - "components": { + "component_maps": { "type": "array", "items": { "$ref": "#/$defs/ComponentMap" @@ -318,7 +297,7 @@ "description": "Maps logical model constructs to some concept and its relationships in the ontology" } }, - "required": ["logical_model", "ontology", "components"], + "required": ["logical_model", "ontology", "component_maps"], "additionalProperties": false }, "Ontology": { diff --git a/ontology/ontology.md b/ontology/ontology.md new file mode 100644 index 0000000..2d8e70d --- /dev/null +++ b/ontology/ontology.md @@ -0,0 +1,597 @@ +# OSI - Ontology Specification + +**Version:** 0.1.1 + +## Table of Contents + +1. [Enumerations](#enumerations) +2. [Ontology](#ontologies) +3. [Ontology map](#ontology-maps) +4. [Complete Example](#complete-example) + +--- + +## Enumerations + +Standard enumeration values used throughout the specification. + +### Concept types + +Ontologies distinguish two different kinds of concepts. +An entity type is a concept that represents real-world objects that must be referenced using other information. +For instance, a person might be referenced by their social security number +or private e-mail address. +A value type is a concept that represents instances of some data type +(i.e SQL types like Integer or String) with additional semantics. +For instance, a social-security number is a string or positive integer that comprises exactly nine digits. + +| Multiplicity | Description | +|---------|-------------| +| `EntityType` | Real-world concept that must be referenced using other information | +| `ValueType` | A datatype with additional semantics | + +### Multiplicities + +The allowable multiplicities of relationships defined in the [Ontology](#ontology) section. + +| Multiplicity | Description | +|---------|-------------| +| `ManyToOne` | The last role of a relationship is uniquely determined by the other roles | +| `OneToOne` | The relationship is ManyToOne in both directions (only for binary relationships) | + +## Ontologies + +Enterprise data are often modeled conceptualy in the form of an ontology with concepts, relationships, +and business rules. This section describes how to represent ontologies hierarchically as a +top-level collection of components, each of which defines some concept and the relationships +that pertain primarily to that concept. + + +### Concepts + +Concepts represent the types of things that have meaning in a business setting, e.g., person, company, +or salary. Each concept is either an entity type or a value type. Ontologies implicitly include a +value-type for each basic data type, like `Integer`, `Decimal`, and `String`, and an entity type +called `Any`. Every other concept ontology extends (is a subtype of) one of these concepts. + +Concepts conform to the following schema: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `concept` | string | Yes | Unique name of this concept | +| `type` | ConceptType | Yes | Entity type or value type | +| `description` | string | No | Human-readable description | +| `relationships` | list | No | Relationships where this concept plays the first role | +| `extends` | list | No | Names of this concept's supertypes | +| `derived_by` | list | No | Expressions that derive this concept's population | +| `identify_by` | list | No | Names of relationships that uniquely reference objects of this concept | +| `requires` | list | No | Expressions that constrain this concept's population | + +### Extends + +Every user-declared concept extends one or more concepts in the ontology. The new concept +is a sutype of each concept that it extends, and the extended concepts are its supertypes. + +Any concept that directly or indirectly extends a value type like `Integer` or `String` is a value type. +Any concept that does not extend some value type is an entity type, and if a concept declares no extends +list, then it is assumed to extend the built-in entity type `Any`. If `SocialSecurityNr` extends `Integer` +and `Employee` extends `Person`, which declares no extends list, then `SocialSecurityNr` is a value type +and both `Person` and `Employee` are entity types. + +### Relationships + +Relationships relate objects of one or more concepts and declare how to verbalize links among +those objects. Relationships have set (as opposed to bag) semantics, and links do not contain +nulls. + +Each relationship that is declared under a concept conforms to the following schema: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Part of the identifier for this relationship | +| `description` | string | No | Human-readable description | +| `multiplicity` | enum | No | Multiplicity constraint | +| `roles` | list | No | List of additional roles in this relationship | +| `derived_by` | list | No | Expressions that derive links of this relationship | +| `requires` | list | No | Expressions that constrain this relationship's population | +| `verbalizes` | list | Yes | Patterns describing how to verbalize links | + +Each relationship is uniquely identified by a prepending its declared name with that of the containing +concept. For instance, in: + +```yaml +ontologies: + - name: EnterpriseOntology + components: + - concept: + name: Person + type: EntityType + identify_by: [ nr ] + relationships: + - name: nr + roles: + - player: SocialSecurityNr + verbalizes: [ '{Person} is identified by {SocialSecurityNr}' ] + multiplicity: OneToOne + - name: earns + roles: + - player: Salary + multiplicity: ManyToOne + verbalizes: [ "{Person} earns {Salary}" ] + ... +``` + +the relationship is identified by the string `Person.earns`. This convention naturally supports +expressions that navigate over the links of relationships using the “dot-join” operator in a +manner that is familiar to object-oriented programming languages. This relationship links +`Person` and `Salary` objects and verbalizes each link as “Person earns Salary.” + +#### Roles + +Objects play roles in the links of a relationship. If you think of a relationship as a narrow table, +then its links are like rows and its roles are like columns. Each role is played by a concept that +constrains the type of objects that can play that role in any link. In `Person.earns`, `Person` and +`Salary` play the first and second roles respectively. + +By convention, the first role of any relationship is played by the concept under which the +relationship is declared. Any additional roles are enumerated in order in the roles list +using this schema: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `player` | string | Yes | Name of the concept that plays this role | +| `name` | string | No | Optional role name | + +For instance, in: + +```yaml +ontologies: + - name: EnterpriseOntology + components: + - concept: + name: Person + type: EntityType + relationships: + - name: files_married_joint + verbalizes: [ "{Person} files married filing joint" ] + - name: purchased_on + roles: + - player: Vehicle + - player: Date + multiplicity: ManyToOne + verbalizes: [ "{Person} puchased {Vehicle} on {Date}" ] +``` + +the unary relationship `Person.files_married_joint` has an empty roles list, while the +ternary relationship `Person.purchased_on` declares two additional roles played by +`Vehicle` and `Date` respectively, + +The role player often suffices to distinguish the role within its relationship, but when +the same concept plays more than one role, the user must declare a distinguising name for +any additional role whose player does not suffice to distinguish it from other roles in +the same relationship. For instance, in: + +```yaml +ontologies: + - name: EnterpriseOntology + components: + - concept: + name: Store + type: EntityType + relationships: + - name: ships_to_in_days + roles: + - player: Store + name: destination + - player: NrDays + multiplicity: ManyToOne + verbalizes: [ "{Store} ships to {Store:destination} in {NrDays}" ] +``` + +the role name `destination` distinguishes the second `Store`-playing role from the first in +this relationship. + +Expressions that are used to define derived_by rules and requires constraints will refer to +roles by name -- the name defaulting to the concept that plays the role unless an explicit +role name is provided. In any expression that involving links of the `Store.ships_to_in_days` +relationship can then use the variables `Store` and `destination` to refer to objecs that +play these two `Store`-playing roles without ambiguity. + +#### Multiplicities + +If a relationship comprises more than one role, objects that play the last role could be functionally +dermined by a tuple of objects that play the other roles. This knowledge is declared using a `ManyToOne` +multiplicity constraint. In the examples above, the constraint declares that each person earns at most +one salary and that for each pair of stores, the former ships to the latter in at most one number of +days. For relationships of ternary and higher arity, the multiplicity applies to the n-th role, meaning +the object that plays the n-th role is functionally determined by the tuple of objects that play +the first n-1 roles. + +In the special case of a binary relationship, one might declare a `OneToOne` multiplicity, which +indicates the relationship is many-to-one in both directions. For instance, the `Person.nr` +relationship is one-to-one because each person is assigned at most one social security number +and each social security number is assigned to at most one person. + +In the absence of any multiplicity, we make no assumptions of functional dependencies among +any of the roles. + +### Identifying relationships + +Many conceptual models distinguish one or more relationships to use when referencing entity-type +objects in expressions and queries. The `Person.nr` relationship can be used to reference a +person by their social security number; while the pair of relationships `License.acct` and +`License.seat_nr` can be used to reference a license by its associated account and seat number. +These identifier relationships are always binary, and their first role is always played by the +concept the relationship is used to reference. + +### Derivation expressions + +Concepts and relationships may be derived using expressions. Think of a derived concept or +relationship as a conceptual view whose objects or links are derived from those of other +concepts or relationships. For instance: + +```yaml +ontologies: + - name: EnterpriseOntology + components: + - concept: + name: Person + type: EntityType + relationships: + - name: parent_of + roles: + - player: Person + name: "child" + verbalizes: [ "{Person} is a parent of {Person:child}", "{Person:child} is a child of {Person}" ] + - name: ancestor_of + roles: + - player: Person + name: "descendant" + derived_by: + - "Person.parent_of(descendant)" + "Person.ancestor_of.parent_of(descendant)" + - name: taxed_at + roles: + - player: TaxRate + derived_by: + - "10.0 WHERE ( Person.files_single AND Person.earns <= 11925 )" + - "10.0 WHERE ( Person.files_married_joint AND Person.earns <= 23850 )" + - ... +``` + +declares two derived relationships -- `ancestor_of` and `taxed_at`. Each link of `Person.ancestor_of` +relates a person to one of its descendants. The two expressions form the base and recursive cases for +this calculation. In the base case, a `Person` as an ancestor of some `descendant` if that `Person` +is the parent of that descendant. And in the recursive case, a `Person` is an ancestor of some +`descendant` if that `Person` is an ancestor of the parent of that `descendant`. Notice in this +example how role names are used to disambiguate the two `Person` roles in this relationship. + +Each link of `Person.taxed_at` links a `Person` object to a `TaxRate` that is derived using +expressions that determine the rate based on the person's filing status and how much they earn. +If, for some person, none of the expressions can be evaluated, then the relationship will have +no link involving that person. + +Expressions that derive a relationship are interpreted as rules for constructing the links of the +relationship in the same way that a SQL query is interpreted as a rule for constructing the rows +of a new table. Each expression must therefore reference each role of the relationship, either +explicitly or implicitly. If an expression evaluates to some object (like 10.0 in the two examples +here) then that object will implicitly play the last role, and the expression must reference each +of the other roles explicitly. If an expression does not evaluate to any object, then it must +explicitly reference each role. + +A derived concept is one whose population is derived from those of its supertype concepts +using one or more expressions. For instance: + +```yaml +ontologies: + - name: EnterpriseOntology + components: + - concept: + name: Employee + type: EntityType + extends: [Person] + derived_by: [ "EXISTS ( Person.earns )" ] +``` + +declares that the population of Employee is derived from the population of Person by +classifying each Person who earns some salary as a Employee. + +### Requires + +The requires list contains expressions that give additional semantics to a concept or relationship +by declaring conditions that must hold over their populations. When applied to a concept, each +expression must reference the concept, as in: + +```yaml +ontologies: + - name: EnterpriseOntology + components: + - concept: + name: SocialSecurityNr + type: ValueType + extends: [Integer] + requires: [ "0 < SocialSecurityNr", "SocialSecurityNr <= 999999999" ] +``` + +When applied to a relationship, each expression must reference one or more roles of the +relationship. For instance, in: + +```yaml +ontologies: + - name: EnterpriseOntology + components: + - concept: + name: Item + type: EntityType + relationships: + - name: offers_in + roles: + - player: Store + verbalizations: [ "{Item} is offered for sale in {Store}", "{Store} offers sale of {Item}" ] + - name: total_sales_in + roles: + - player: Store + - player: Amount + verbalizations: [ "{Item} sold for cumulative {Amount} in {Store}" ] + requires: + - "Amount > 0.0" + - "Item.offers_in(Store)" +``` + +the first expression requires any value that plays the `Amount` role to be positive while the second +requires any item that has sales in some store to be offered in that store. + +## Ontology maps + +Logical to conceptual schema mappings declare how to map field values to conceptual objects +and links. Just as ontologies are orgnaized into hierarchical components, so are ontology maps. +Each component map declares how to populate a concept with objects and any relationships that +are primarily keyed by that concept with links. These declarations are formed from patterns +of expressions that reference one or more fields in a logical model that is declared using +the OSI core semantic model spec. + +Entity-type objects (entities) are opaque and must be mapped to via their identifying +relationships. These relationships are always binary, with the first role played by +the entity type itself. In the common case where an entity type is identified by a +single relationship whose second role is played by a value type, a value map suffices +to declare the existence of entities of that type or to look them up when mapping to +links of a relationship. + +An entity map is an array of identifier maps: + +| Field | Type | Required | Description | +|--------------|----------|-----|-------| +| `role` | string | Yes | Expression that indicates the role the mapped objects will play | +| `value` | string | if no `entity` | Expression that maps to a value (when role played by a value type) | +| `entity` | list | if no `value` | Entity map that maps to an entity (when role played by an entity type) | + +each of which is either a value map or a nested entity map that looks up an object to map to some +role of an identifier relationship. + +Building on value and entity maps, we can declare precisely how fields determine the entities of +a concept and the links of every relationship in which that concept plays the first role. + +#### Entities (entity maps that determine the existence of objects of some entity type) + +When a concept is an entity type, it can declare an `entities` array, each element of which +is an entity map. + +For instance, in: + +```yaml +ontologies: + - name: EnterpriseOntology + components: + - name: PersonComponent + concept: + name: Person + type: EntityType + identify_by: [ nr ] + relationships: + - name: nr + roles: + - player: SocialSecurityNr + multiplicity: OneToOne + verbalizes: [ "{Person} is identified by {SocialSecurityNr}" ] +ontology_maps: + - name: EnterpriseOntologyMap + ontology: EnterpriseOntology + logical_model: + name: EnterpriseSemanticModel + datasets: + ... + component_maps: + - component: PersonComponentMap + object_maps: + - referent: Person + identifier: + - relationship: nr + object: + referent: SocialSecurityNr + expression: PERSONS.SSN + ... +``` + +uses a value map to declare how values from the `SSN` field of dataset `PERSONS` are used +to form `SocialSecurityNr` values that map to the second role of the `nr` relationship. +Because each link in that relationship associates a `SocialSecurityNr` value to some +unique `Person` entity, this value map suffices to associate each distinct `SSN` value +in the dataset to a distinct `Person` object in the ontology. + +A more interesting example maps field values to `OrderLineItem` entities, which are +identified using two relationships: + +```yaml +ontologies: + - name: EnterpriseOntology + components: + - concept: + name: OrderLineItem + type: EntityType + identify_by: [ "nr", "order" ] + requires: [ "OrderLineItem.nr", "OrderLineItem.order" ] + relationships: + - name: nr + roles: [ player: LineNr ] + multiplicity: ManyToOne + - name: order + roles: [ player: CustOrder ] + multiplicity: ManyToOne +ontology_maps: + - ontology: EnterpriseOntology + logical_model: + name: EnterpriseSemanticModel + datasets: + ... + component_maps: + - component: OrderLineItemComponentMap + object_maps: + - referent: OrderLineItem + identifier: + - relationship: nr + object: + referent: LineNr + expression: LINEITEMS.L_LINENUMBER + - relationship: order + object: + referent: Order + identifier: + relationship: CustOrder.nr + object: + referent: OrderNr + expression: LINEITEMS.L_ORDERKEY +``` + +This mapping contains a single entity map with two role mappings -- a value map that maps the +`L_LINENUMBER` field to the second role of the `nr` relationship, and a nested entity map that +maps `CustOrder` objects to the second role in the `order` relationship. + +#### Links (mappings that determine the existence of relationship links) + +Each relationships is grouped under the concept that plays its first role. A concept's +`links` array maps field schema to relationships. Each array element is a tree that +concisely declares how field values map to the links of one or more of these relationships. +More precisely, the path from the root of a tree to a node describes how to map to tuples +of objects that form the links of the relationship that is named by the node. These structures +leverage the hierarchical nature of YAML to avoid duplication in the typical case when the +fields of a single dataset map to many relationships. + +Each tree node has the following schema: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `concept` | string | when level > 1 | Concept whose objects are looked up by this node | +| `value` | string | if no `entity` | Expression that computes a value (when concept is a value type) | +| `entity` | list | if no `value` | Entity map that looks up an entity (when concept is an entity type) | +| `relationship` | string | No | Relationship whose links are mapped to by the path to this node | +| `children` | list | No | List of child nodes in the tree | + +Each node must provide either a `value` or an `entity` by which to look up objects but never both. +The level of each node coincides with the arity of the associated relationship. So a root node +could map to a unary relationship, a node at level 2 could map to a binary relationship, and so +forth. + +For instance, the `links` array declared here: + +```yaml +ontologies: + - name: EnterproseOntology + components: + - concept: + name: Item + type: EntityType + identify_by: [ nr ] + relationships: + - name: nr + roles: [ concept: SkuNr ] + multiplicity: OneToOne + verbalises: "{Item} is identified by {SkuNr}" + - name: active # A unary relationship + verbalizes: [ "{Item} is actively sold" ] + - name: active_in + roles: [ concept: Store ] + verbalises: [ "{Item} is actively sold in {Store}" ] + - name: returned_in_for + roles: [ concept: Store, concept: Amount ] + verbalizes: [ "{Item} returned in {Store} for {Amount}" ] + multiplicitly: ManyToOne + - name: sold_in_for + roles: [ concept: Store, concept: Amount ] + verbalizes: [ "{Item} sells in {Store} for {Amount}" ] + multiplicitly: ManyToOne +ontology_maps: + - name: flights_map + description: Example mapping of logical fields to ontology concepts and relationships + ontology: flights + logical_model: + name: Logical_Flights_Model + description: Logical model for flight data + datasets: + ... + component_maps: + - component: ItemComponentMap + object_maps: + - referent: Item + identifier: + relationship: Item.nr + object: + referent: ItemNr + expression: ITEMS.SKU + link_maps: + - maps: + referent: Item + identifier: + relationship: Item.nr + object: + referent: ItemNr + expression: METRICS.SKU + populates: Item.active + children: + - maps: + referent: Store + identifier: + relationship: Store.nr + object: + referent: StoreNr + expression: METRICS.STORE + populates: Item.active_in + children: + - maps: + referent: Amount + expression: METRICS.SALES + populates: Item.sold_in_for + - maps: + referent: Amount + expression: METRICS.RETURNS + populates: Item.returned_in_for +``` + +describes a tree with one root node, one node at level 2, and two nodes at level 3. +Each node maps fields of the `METRICS` dataset to links of four different relationships, +and notice how the mapping to `Item` objects is declared once even though `Item` plays a +role in all four of the relationships and that the mapping to `Store` objects is declared +once even though `Store` plays a role in three of the relationships. + + +--- + +## Complete Example + +Here's a complete ontology example showing all components working together: + +```yaml +``` + +## Version History + +- **0.1.1** (2026-05-20): Support for both logical and conceptual modeling layers (ontologies) + - Core ontology structure + - Support for concepts, relationships, and logical -> conceptual schema mappings + - Renamed relationships in the logical layer to join paths to avoid conflict with + relationships in the conceptual layer + +--- + +## License + +See LICENSE file for details. diff --git a/validation/validate.py b/validation/validate.py index d79db1a..a33cf76 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -10,6 +10,7 @@ Usage: python validation/validate.py + python validation/validate.py --schema ontology/ontology.json python validation/validate.py examples/tpcds_semantic_model.yaml """ @@ -147,6 +148,10 @@ def validate_sql_expression(expr: str, dialect: str, context: str) -> str | None def validate_sql(data: dict) -> list[str]: """Validate SQL expressions in fields and metrics.""" + # Only semantic model files contain SQL expressions to validate. + if not data.get("semantic_model"): + return [] + if not SQLGLOT_AVAILABLE: return ["[SQL] Warning: sqlglot not installed, skipping SQL validation. Install with: pip install sqlglot"] @@ -191,8 +196,16 @@ def main(): print(__doc__) sys.exit(1) - yaml_path = Path(sys.argv[1]) + args = sys.argv[1:] + yaml_path = Path(args[0]) + schema_path = Path(__file__).parent.parent / "core-spec" / "osi-schema.json" + if len(args) > 1: + if len(args) == 3 and args[1] == "--schema": + schema_path = Path(args[2]) + else: + print("Usage: python validation/validate.py [--schema ]") + sys.exit(1) if not yaml_path.exists(): print(f"Error: File not found: {yaml_path}") @@ -216,9 +229,12 @@ def main(): # Run validations errors = [] errors.extend(validate_schema(data, schema)) - errors.extend(validate_unique_names(data)) - errors.extend(validate_references(data)) - errors.extend(validate_sql(data)) + + # Run semantic-model-specific checks only for semantic model payloads. + if data.get("semantic_model"): + errors.extend(validate_unique_names(data)) + errors.extend(validate_references(data)) + errors.extend(validate_sql(data)) # Report results if errors: From 9de418a01e50eb316ada5182fd23a59ec297b182 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Wed, 20 May 2026 23:38:03 -0400 Subject: [PATCH 10/89] Small change to JSON schema --- ontology/ontology.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ontology/ontology.json b/ontology/ontology.json index 071b777..1ab3e8d 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -170,14 +170,14 @@ "object_maps": { "type": "array", "items": { - "$ref": "#/$defs/ObjectRef" + "$ref": "#/$defs/ObjectExpr" }, "description": "Mappings from logical constructs that populate the concept in this component. Valid only when the concept is an entity type" }, "link_maps": { "type": "array", "items": { - "$ref": "#/$defs/RelationshipMap" + "$ref": "#/$defs/LinkExpr" }, "description": "Mappings from logical model relationships to ontology relationships pertaining to the mapped concept" } @@ -199,7 +199,7 @@ "description": "Name of referent relationship" }, "object": { - "$ref": "#/$defs/ObjectRef" + "$ref": "#/$defs/ObjectExpr" } }, "required": ["relationship"], @@ -221,7 +221,7 @@ "required": ["player"], "additionalProperties": false }, - "ObjectRef": { + "ObjectExpr": { "type": "object", "description": "Pattern of logical-level expressions for identifying objects of some referent concept using the values in one or more fields", "properties": { @@ -243,7 +243,7 @@ "required": ["referent"], "additionalProperties": false }, - "RelationshipMap": { + "LinkExpr": { "type": "object", "description": "Mapping from logical model constructs to the links of some relationship in the ontology", "properties": { @@ -252,12 +252,12 @@ "description": "Name of relationship being populated by this map" }, "maps": { - "$ref": "#/$defs/ObjectRef" + "$ref": "#/$defs/ObjectExpr" }, "children": { "type": "array", "items": { - "$ref": "#/$defs/RelationshipMap" + "$ref": "#/$defs/LinkExpr" }, "description": "Relationship maps at the next level in this hierarchy" } From 75a50f316da0c0772cc5ee8a8b24ed6dd1e3cd86 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Thu, 21 May 2026 16:03:49 -0400 Subject: [PATCH 11/89] Checkpointing changes --- examples/flights.yaml | 87 ++++++------- ontology/ontology.json | 77 ++++++------ ontology/ontology.md | 275 +++++++++++++++++++---------------------- 3 files changed, 206 insertions(+), 233 deletions(-) diff --git a/examples/flights.yaml b/examples/flights.yaml index 959db95..d126588 100644 --- a/examples/flights.yaml +++ b/examples/flights.yaml @@ -2,103 +2,100 @@ version: 0.1.1 ontologies: - name: flights description: Ontology for flight-related concepts and relationships - components: - - name: Example_Runway - concept: + concepts: + - concept: name: Example_Runway type: EntityType identify_by: [ id ] relationships: - name: id roles: - - player: String + - concept: String verbalizes: [ '{Example_Runway} id {String}' ] multiplicity: ManyToOne - name: shapelength roles: - - player: Float + - concept: Float verbalizes: [ '{Example_Runway} shapeLength {Float}' ] multiplicity: ManyToOne - name: airportid roles: - - player: String + - concept: String verbalizes: [ '{Example_Runway} airportId {String}' ] multiplicity: ManyToOne - name: length roles: - - player: Float + - concept: Float verbalizes: [ '{Example_Runway} length {Float}' ] multiplicity: ManyToOne - name: geometry roles: - - player: String + - concept: String verbalizes: [ '{Example_Runway} geometry {String}' ] multiplicity: ManyToOne - name: designator roles: - - player: String + - concept: String verbalizes: [ '{Example_Runway} designator {String}' ] multiplicity: ManyToOne - name: airports roles: - - player: Example_Airport + - concept: Example_Airport verbalizes: [ '{Example_Runway} airports {Example_Airport}' ] multiplicity: ManyToOne derived_by: [ 'Example_Runway.airportid == Example_Airport.airportid' ] - - name: Example_RouteAlertComment - concept: + - concept: name: Example_RouteAlertComment type: EntityType identify_by: [ "id" ] relationships: - name: id roles: - - player: String + - concept: String verbalizes: [ '{Example_RouteAlertComment} id {String}' ] multiplicity: ManyToOne - name: routealertid roles: - - player: String + - concept: String verbalizes: [ '{Example_RouteAlertComment} routeAlertId {String}' ] multiplicity: ManyToOne - name: comment roles: - - player: String + - concept: String verbalizes: [ '{Example_RouteAlertComment} comment {String}' ] multiplicity: ManyToOne - name: routealert roles: - - player: Example_RouteAlert + - concept: Example_RouteAlert verbalizes: [ '{Example_RouteAlertComment} routeAlerts {Example_RouteAlert}' ] multiplicity: ManyToOne derived_by: [ 'Example_RouteAlertComment.routealertid == Example_RouteAlert.routealertid' ] - - name: Example_Airport - concept: + - concept: name: Example_Airport type: EntityType identify_by: [ "airportid" ] relationships: - name: airportid roles: - - player: String + - concept: String verbalizes: [ '{Example_Airport} airportId {String}' ] multiplicity: ManyToOne - name: displayairportname roles: - - player: String + - concept: String verbalizes: [ '{Example_Airport} displayAirportName {String}' ] multiplicity: ManyToOne - name: displaycitymarketnamefull roles: - - player: String + - concept: String verbalizes: [ '{Example_Airport} displayCityMarketNameFull {String}' ] multiplicity: ManyToOne - name: flights roles: - - player: Example_Flight + - concept: Example_Flight verbalizes: [ '{Example_Airport} flights {Example_Flight}' ] derived_by: [ 'Example_Airport.airportid == Example_Flight.originairportid OR Example_Airport.airportid == Example_Flight.destairportid' ] -ontology_maps: +ontology_mappings: - name: flights_map description: Example mapping of logical fields to ontology concepts and relationships ontology: flights @@ -1063,41 +1060,35 @@ ontology_maps: dialects: - dialect: ANSI_SQL expression: CAST(route_title AS VARCHAR) - component_maps: - - component: Example_Runway - object_maps: - - referent: Example_Runway - identifier: + concept_mappings: + - concept: Example_Runway + object_mappings: + - referent_mappings: - relationship: id - object: - referent: String - expression: Example_Runway_example_runway.id - link_maps: - - maps: - referent: Example_Runway - identifier: + expression: Example_Runway_example_runway.id + link_mappings: + - object_mapping: + referent_mappings: - relationship: id - object: - referent: String - expression: Example_Runway_example_runway.id + expression: Example_Runway_example_runway.id children: - - maps: - referent: Float + - object_mapping: + concept: Float expression: Example_Runway_example_runway.length populates: length - - maps: - referent: String + - object_mapping: + concept: String expression: Example_Runway_example_runway.geometry populates: geometry - - maps: - referent: String + - object_mapping: + concept: String expression: Example_Runway_example_runway.designator populates: designator - - maps: - referent: Float + - object_mapping: + concept: Float expression: Example_Runway_example_runway.shape_length populates: shape_length - - maps: - referent: String + - object_mapping: + concept: String expression: Example_Runway_example_runway.airport_id populates: airport_id diff --git a/ontology/ontology.json b/ontology/ontology.json index 1ab3e8d..ad82de3 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -31,7 +31,7 @@ "$ref": "#/$defs/Ontology" } }, - "ontology_maps": { + "ontology_mappings": { "type": "array", "description": "Collection of ontology maps from logical models", "items": { @@ -46,10 +46,6 @@ "type": "object", "description": "Ontology component that defines a single concept and any relationships that are keyed primarily by that concept", "properties": { - "name": { - "type": "string", - "description": "Name of the component" - }, "description": { "type": "string", "description": "Human-readable description of the component" @@ -65,7 +61,7 @@ "description": "Defines relationships that pertain primarily to the concept defined in this component" } }, - "required": ["name"], + "required": ["concept"], "additionalProperties": false }, "Expression": { @@ -159,30 +155,30 @@ "required": ["name", "type"], "additionalProperties": false }, - "ComponentMap": { + "ConceptMapping": { "type": "object", "description": "Mappings from logical model constructs to some ontology component", "properties": { - "component": { + "concept": { "type": "string", - "description": "Name of the ontology component whose concept and/or relationships are being mapped to" + "description": "Name of the concept whose part of the ontology we are mapping to" }, - "object_maps": { + "object_mappings": { "type": "array", "items": { - "$ref": "#/$defs/ObjectExpr" + "$ref": "#/$defs/ObjectMapping" }, "description": "Mappings from logical constructs that populate the concept in this component. Valid only when the concept is an entity type" }, - "link_maps": { + "link_mappings": { "type": "array", "items": { - "$ref": "#/$defs/LinkExpr" + "$ref": "#/$defs/LinkMapping" }, "description": "Mappings from logical model relationships to ontology relationships pertaining to the mapped concept" } }, - "required": ["component"], + "required": ["concept"], "additionalProperties": false }, "ConceptType": { @@ -190,7 +186,7 @@ "enum": [ "EntityType", "ValueType" ], "description": "A concept is either an entity type or a value type" }, - "ReferentMap": { + "ReferentMapping": { "type": "object", "description": "Mapping from logical model constructs to a relationship used to references some entity type in the ontology", "properties": { @@ -198,8 +194,14 @@ "type": "string", "description": "Name of referent relationship" }, - "object": { - "$ref": "#/$defs/ObjectExpr" + "expression": { + "$ref": "#/$defs/Expression" + }, + "referent_mappings": { + "type": "array", + "items": { + "$ref": "#/$defs/ReferentMapping" + } } }, "required": ["relationship"], @@ -209,7 +211,7 @@ "type": "object", "description": "Role in some relationship (the container)", "properties": { - "player": { + "concept": { "type": "string", "description": "Name of the concept playing this role" }, @@ -218,21 +220,21 @@ "description": "Optional name of this role, used when the same concept plays multiple roles in the same relationship" } }, - "required": ["player"], + "required": ["concept"], "additionalProperties": false }, - "ObjectExpr": { + "ObjectMapping": { "type": "object", - "description": "Pattern of logical-level expressions for identifying objects of some referent concept using the values in one or more fields", + "description": "Pattern of logical-level expressions for identifying objects of some concept using the values in one or more fields", "properties": { - "referent": { + "concept": { "type": "string", - "description": "Name of the referent concept" + "description": "Name of the concept whose objects we are mapping to" }, - "identifier": { + "referent_mappings": { "type": "array", "items": { - "$ref": "#/$defs/ReferentMap" + "$ref": "#/$defs/ReferentMapping" }, "description": "Maps logical-model constructs to referent relationships of this entity type" }, @@ -240,29 +242,28 @@ "$ref": "#/$defs/Expression" } }, - "required": ["referent"], "additionalProperties": false }, - "LinkExpr": { + "LinkMapping": { "type": "object", - "description": "Mapping from logical model constructs to the links of some relationship in the ontology", + "description": "Mapping from logical schema to the links of relationships in the ontology", "properties": { "populates": { "type": "string", - "description": "Name of relationship being populated by this map" + "description": "Name of relationship being populated by this mapping node" }, - "maps": { - "$ref": "#/$defs/ObjectExpr" + "object_mapping": { + "$ref": "#/$defs/ObjectMapping" }, "children": { "type": "array", "items": { - "$ref": "#/$defs/LinkExpr" + "$ref": "#/$defs/LinkMapping" }, "description": "Relationship maps at the next level in this hierarchy" } }, - "required": ["maps"], + "required": ["object_mapping"], "additionalProperties": false }, "Multiplicity": { @@ -289,15 +290,15 @@ "logical_model": { "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/SemanticModel" }, - "component_maps": { + "concept_mappings": { "type": "array", "items": { - "$ref": "#/$defs/ComponentMap" + "$ref": "#/$defs/ConceptMapping" }, "description": "Maps logical model constructs to some concept and its relationships in the ontology" } }, - "required": ["logical_model", "ontology", "component_maps"], + "required": ["logical_model", "ontology", "concept_mappings"], "additionalProperties": false }, "Ontology": { @@ -315,7 +316,7 @@ "ai_context": { "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/AIContext" }, - "components": { + "concepts": { "type": "array", "items": { "$ref": "#/$defs/Component" @@ -324,7 +325,7 @@ "description": "Components that define the concepts and relationships in this ontology" } }, - "required": ["name", "components"], + "required": ["name", "concepts"], "additionalProperties": false } } diff --git a/ontology/ontology.md b/ontology/ontology.md index 2d8e70d..2400b29 100644 --- a/ontology/ontology.md +++ b/ontology/ontology.md @@ -41,11 +41,9 @@ The allowable multiplicities of relationships defined in the [Ontology](#ontolog ## Ontologies -Enterprise data are often modeled conceptualy in the form of an ontology with concepts, relationships, -and business rules. This section describes how to represent ontologies hierarchically as a -top-level collection of components, each of which defines some concept and the relationships -that pertain primarily to that concept. - +Ontologies are conceptual models of enterprise data that describe the enterprise in terms +of concepts, relationships, and business rules. This specification represents ontologies +hierarchically, grouping each relationship under the concept that plays its first role. ### Concepts @@ -102,7 +100,7 @@ concept. For instance, in: ```yaml ontologies: - name: EnterpriseOntology - components: + concepts: - concept: name: Person type: EntityType @@ -110,12 +108,12 @@ ontologies: relationships: - name: nr roles: - - player: SocialSecurityNr + - concept: SocialSecurityNr verbalizes: [ '{Person} is identified by {SocialSecurityNr}' ] multiplicity: OneToOne - name: earns roles: - - player: Salary + - concept: Salary multiplicity: ManyToOne verbalizes: [ "{Person} earns {Salary}" ] ... @@ -147,7 +145,7 @@ For instance, in: ```yaml ontologies: - name: EnterpriseOntology - components: + concepts: - concept: name: Person type: EntityType @@ -156,8 +154,8 @@ ontologies: verbalizes: [ "{Person} files married filing joint" ] - name: purchased_on roles: - - player: Vehicle - - player: Date + - concept: Vehicle + - concept: Date multiplicity: ManyToOne verbalizes: [ "{Person} puchased {Vehicle} on {Date}" ] ``` @@ -174,16 +172,16 @@ the same relationship. For instance, in: ```yaml ontologies: - name: EnterpriseOntology - components: + concepts: - concept: name: Store type: EntityType relationships: - name: ships_to_in_days roles: - - player: Store + - concept: Store name: destination - - player: NrDays + - concept: NrDays multiplicity: ManyToOne verbalizes: [ "{Store} ships to {Store:destination} in {NrDays}" ] ``` @@ -233,26 +231,26 @@ concepts or relationships. For instance: ```yaml ontologies: - name: EnterpriseOntology - components: + concepts: - concept: name: Person type: EntityType relationships: - name: parent_of roles: - - player: Person + - concept: Person name: "child" verbalizes: [ "{Person} is a parent of {Person:child}", "{Person:child} is a child of {Person}" ] - name: ancestor_of roles: - - player: Person + - concept: Person name: "descendant" derived_by: - "Person.parent_of(descendant)" "Person.ancestor_of.parent_of(descendant)" - name: taxed_at roles: - - player: TaxRate + - concept: TaxRate derived_by: - "10.0 WHERE ( Person.files_single AND Person.earns <= 11925 )" - "10.0 WHERE ( Person.files_married_joint AND Person.earns <= 23850 )" @@ -285,7 +283,7 @@ using one or more expressions. For instance: ```yaml ontologies: - name: EnterpriseOntology - components: + concepts: - concept: name: Employee type: EntityType @@ -305,7 +303,7 @@ expression must reference the concept, as in: ```yaml ontologies: - name: EnterpriseOntology - components: + concepts: - concept: name: SocialSecurityNr type: ValueType @@ -319,19 +317,19 @@ relationship. For instance, in: ```yaml ontologies: - name: EnterpriseOntology - components: + concepts: - concept: name: Item type: EntityType relationships: - name: offers_in roles: - - player: Store + - concept: Store verbalizations: [ "{Item} is offered for sale in {Store}", "{Store} offers sale of {Item}" ] - name: total_sales_in roles: - - player: Store - - player: Amount + - concept: Store + - concept: Amount verbalizations: [ "{Item} sold for cumulative {Amount} in {Store}" ] requires: - "Amount > 0.0" @@ -341,90 +339,89 @@ ontologies: the first expression requires any value that plays the `Amount` role to be positive while the second requires any item that has sales in some store to be offered in that store. -## Ontology maps - -Logical to conceptual schema mappings declare how to map field values to conceptual objects -and links. Just as ontologies are orgnaized into hierarchical components, so are ontology maps. -Each component map declares how to populate a concept with objects and any relationships that -are primarily keyed by that concept with links. These declarations are formed from patterns -of expressions that reference one or more fields in a logical model that is declared using -the OSI core semantic model spec. +## Ontology mappings -Entity-type objects (entities) are opaque and must be mapped to via their identifying -relationships. These relationships are always binary, with the first role played by -the entity type itself. In the common case where an entity type is identified by a -single relationship whose second role is played by a value type, a value map suffices -to declare the existence of entities of that type or to look them up when mapping to -links of a relationship. +Ontology mappings declare how to map the values of fields at the logical level to objects and links +in the ontology. Just as ontologies are partitioned by concept, ontology maps partition into concept +maps that group by some concept in the ontology. -An entity map is an array of identifier maps: +Each concept map declares how to populate a concept with objects and how to populate the relationships +that are primarily keyed by that concept with links. These declarations are formed from patterns +of expressions that reference fields in a logical model that is declared using the OSI core semantic +model spec. -| Field | Type | Required | Description | -|--------------|----------|-----|-------| -| `role` | string | Yes | Expression that indicates the role the mapped objects will play | -| `value` | string | if no `entity` | Expression that maps to a value (when role played by a value type) | -| `entity` | list | if no `value` | Entity map that maps to an entity (when role played by an entity type) | +### Object mappings -each of which is either a value map or a nested entity map that looks up an object to map to some -role of an identifier relationship. +An object mapping describes how to map to the objects of some concept using a pattern of SQL +expressions over the fields of one or more datasets. -Building on value and entity maps, we can declare precisely how fields determine the entities of -a concept and the links of every relationship in which that concept plays the first role. +An object mapping has the following schema: -#### Entities (entity maps that determine the existence of objects of some entity type) +| Field | Type | Required | Description | +|---------------|---------|-----|-------| +| `concept` | string | No | Names the concept being mapped to using this object map | +| `expression` | string | if no `referent_mappings` | SQL expression that computes a value from fields | +| `referent_mappings` | list | if no `expression` | Referent mappings that find entity objects using identifying realtionships | -When a concept is an entity type, it can declare an `entities` array, each element of which -is an entity map. +When the concept is a value type, an object mapping is just a SQL expression that computes its values. +For instance, an object map that computes `SocialSecurityNumber` values would either retrieve or stitch +together an integer value that satisfies any constraints. -For instance, in: +In addition, when a concept is an entity type with a simple identifier -- one relationship that uses some +value type to uniquely reference the concept -- the object map is just a SQL expression that computes the +values of the identifying value type and then maps those values to objects of tha entity type using the +identifier relationship. +Given this ontology: ```yaml ontologies: - name: EnterpriseOntology - components: - - name: PersonComponent - concept: + concepts: + - concept: name: Person type: EntityType identify_by: [ nr ] relationships: - name: nr roles: - - player: SocialSecurityNr + - concept: SocialSecurityNr multiplicity: OneToOne verbalizes: [ "{Person} is identified by {SocialSecurityNr}" ] -ontology_maps: - - name: EnterpriseOntologyMap - ontology: EnterpriseOntology - logical_model: - name: EnterpriseSemanticModel - datasets: - ... - component_maps: - - component: PersonComponentMap - object_maps: - - referent: Person - identifier: - - relationship: nr - object: - referent: SocialSecurityNr - expression: PERSONS.SSN - ... ``` +this concept mapping: + +```yaml +concept_mappings: + - concept: Person + object_mappings: + - expression: PERSONS.SSN + ... +``` +uses an object mapping to declare how values from the `SSN` field of dataset `PERSONS` are +used to form `Person` objects. More precisely, the mapping declares to use `SSN` values to +form `SocialSecurityNr` values that are supplied to `nr` relationship, which is used to +identify `Person`. These additional details can be inferred because the object mapping is +attempting to populate the entity type `Person`, which declares a simple preferred identifier. -uses a value map to declare how values from the `SSN` field of dataset `PERSONS` are used -to form `SocialSecurityNr` values that map to the second role of the `nr` relationship. -Because each link in that relationship associates a `SocialSecurityNr` value to some -unique `Person` entity, this value map suffices to associate each distinct `SSN` value -in the dataset to a distinct `Person` object in the ontology. +When the concept is an entity type that does not provide a simple preferred identifier, the +object mapping is an array of referent mappings, each of which declares how to use one of its +identifying relationships to find its objects given other objects (values and/or objects of +another entity type). -A more interesting example maps field values to `OrderLineItem` entities, which are -identified using two relationships: +A referent mapping has the following schema: + +| Field | Type | Required | Description | +|----------------|--------|-----|-------| +| `relationship` | string | Yes | Name of an identifying relationship | +| `expression` | string | if no `referent_mappings` | SQL expression that computes a value from fields | +| `referent_mappings` | list | if no `expression` | Referent mappings that find entity objects using identifying realtionships | + +For instance, consider this ontology snippet: ```yaml ontologies: - name: EnterpriseOntology - components: + concepts: - concept: name: OrderLineItem type: EntityType @@ -432,51 +429,45 @@ ontologies: requires: [ "OrderLineItem.nr", "OrderLineItem.order" ] relationships: - name: nr - roles: [ player: LineNr ] + roles: [ concept: LineNr ] multiplicity: ManyToOne - name: order - roles: [ player: CustOrder ] + roles: [ concept: CustOrder ] multiplicity: ManyToOne -ontology_maps: - - ontology: EnterpriseOntology - logical_model: - name: EnterpriseSemanticModel - datasets: - ... - component_maps: - - component: OrderLineItemComponentMap - object_maps: - - referent: OrderLineItem - identifier: - - relationship: nr - object: - referent: LineNr - expression: LINEITEMS.L_LINENUMBER - - relationship: order - object: - referent: Order - identifier: - relationship: CustOrder.nr - object: - referent: OrderNr - expression: LINEITEMS.L_ORDERKEY ``` -This mapping contains a single entity map with two role mappings -- a value map that maps the -`L_LINENUMBER` field to the second role of the `nr` relationship, and a nested entity map that -maps `CustOrder` objects to the second role in the `order` relationship. +and notice that `OrderLineItem` has a compound identifier. This concept mapping: -#### Links (mappings that determine the existence of relationship links) +```yaml +concept_mappings: + - concept: OrderLineItem + object_mappings: + - referent_mappings: + - relationship: nr + expression: LINEITEMS.L_LINENUMBER + - relationship: order + referent_mappings: + relationship: CustOrder.nr + expression: LINEITEMS.L_ORDERKEY +``` + +contains a single object mapping that uses two referent mappings, which: +1. map the `L_LINENUMBER` field to `LineNr` values for the `nr` relationship, and +2. use nested referent mappings to find `Order` objects to supply to the `order` relationship. +The nested referent mappings are needed because `Order` is an entity type. -Each relationships is grouped under the concept that plays its first role. A concept's -`links` array maps field schema to relationships. Each array element is a tree that -concisely declares how field values map to the links of one or more of these relationships. -More precisely, the path from the root of a tree to a node describes how to map to tuples -of objects that form the links of the relationship that is named by the node. These structures -leverage the hierarchical nature of YAML to avoid duplication in the typical case when the -fields of a single dataset map to many relationships. -Each tree node has the following schema: +#### Link mappings + +A concept mapping's `link_mappings` array describes how to map logical field schema to the +relationships that group under the concept associated with the mapping. +Each link mapping is a tree structure that concisely declares how field values map to the +links of one or more of these relationships. More precisely, the path from the root of a +tree to a node describes how to map to tuples of objects that form the links of the relationship +that is named by the node. These structures leverage the hierarchical nature of YAML to avoid +duplication in the typical case when the fields of a single dataset map to many relationships. + +Each link mapping has the following schema: | Field | Type | Required | Description | |-------|------|----------|-------------| @@ -495,8 +486,8 @@ For instance, the `links` array declared here: ```yaml ontologies: - - name: EnterproseOntology - components: + - name: EnterpriseOntology + concepts: - concept: name: Item type: EntityType @@ -519,7 +510,7 @@ ontologies: roles: [ concept: Store, concept: Amount ] verbalizes: [ "{Item} sells in {Store} for {Amount}" ] multiplicitly: ManyToOne -ontology_maps: +ontology_mappings: - name: flights_map description: Example mapping of logical fields to ontology concepts and relationships ontology: flights @@ -528,40 +519,30 @@ ontology_maps: description: Logical model for flight data datasets: ... - component_maps: - - component: ItemComponentMap - object_maps: - - referent: Item - identifier: + concept_mappings: + - concept: Item + object_mappings: + - referent_mappings: relationship: Item.nr - object: - referent: ItemNr - expression: ITEMS.SKU - link_maps: - - maps: - referent: Item - identifier: + expression: ITEMS.SKU + link_mappings: + - object_mapping: + referent_mappings: relationship: Item.nr - object: - referent: ItemNr - expression: METRICS.SKU + expression: METRICS.SKU populates: Item.active children: - - maps: - referent: Store - identifier: - relationship: Store.nr - object: - referent: StoreNr - expression: METRICS.STORE + - object_mapping: + concept: Store + expression: METRICS.STORE populates: Item.active_in children: - - maps: - referent: Amount + - object_mapping: + concept: Amount expression: METRICS.SALES populates: Item.sold_in_for - - maps: - referent: Amount + - object_mapping: + concept: Amount expression: METRICS.RETURNS populates: Item.returned_in_for ``` From 6e422be530faa93b08272799b77a3d623d00138f Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Thu, 21 May 2026 17:38:36 -0400 Subject: [PATCH 12/89] Checkpointing for final review --- ontology/ontology.md | 230 +++++++++++++++++++++++-------------------- 1 file changed, 122 insertions(+), 108 deletions(-) diff --git a/ontology/ontology.md b/ontology/ontology.md index 2400b29..a409604 100644 --- a/ontology/ontology.md +++ b/ontology/ontology.md @@ -6,8 +6,7 @@ 1. [Enumerations](#enumerations) 2. [Ontology](#ontologies) -3. [Ontology map](#ontology-maps) -4. [Complete Example](#complete-example) +3. [Ontology mappings](#ontology-mappings) --- @@ -17,19 +16,38 @@ Standard enumeration values used throughout the specification. ### Concept types -Ontologies distinguish two different kinds of concepts. -An entity type is a concept that represents real-world objects that must be referenced using other information. -For instance, a person might be referenced by their social security number -or private e-mail address. -A value type is a concept that represents instances of some data type -(i.e SQL types like Integer or String) with additional semantics. -For instance, a social-security number is a string or positive integer that comprises exactly nine digits. +Ontologies distinguish two different kinds of concepts: -| Multiplicity | Description | +| ConceptType | Description | |---------|-------------| | `EntityType` | Real-world concept that must be referenced using other information | | `ValueType` | A datatype with additional semantics | +An entity type is a concept that represents real-world objects that must be referenced using other +information. For instance, a person might be referenced by their social security number or private +e-mail address. In some modeling languages these are called either entities or object types. + +A value type is a concept that represents instances of some data type (i.e SQL types like Integer +or String) with additional semantics. For instance, a social-security number is a string or positive +integer that comprises exactly nine digits. In some modeling langauges these are called data types +or domains. + +### Built-in concepts + +Ontologies come with several built in concepts that can be referred to by name: + +| BuiltInConcept | Description | +|---------|-------------| +| `Any` | Most general entity type | +| `Boolean` | Most general boolean value type | +| `Date` | Most general date value type | +| `DateTime` | Most general datetime value type | +| `Decimal` | Most general decimal value type | +| `Float` | Most general floating point value type | +| `Integer` | Most general integer value type | +| `String` | Most general string value type | + + ### Multiplicities The allowable multiplicities of relationships defined in the [Ontology](#ontology) section. @@ -44,15 +62,18 @@ The allowable multiplicities of relationships defined in the [Ontology](#ontolog Ontologies are conceptual models of enterprise data that describe the enterprise in terms of concepts, relationships, and business rules. This specification represents ontologies hierarchically, grouping each relationship under the concept that plays its first role. +Every ontology implicitly includes all of the built-in concepts (see Built-in concepts +enumeration above) and may refer to them by name without declaring them. ### Concepts Concepts represent the types of things that have meaning in a business setting, e.g., person, company, or salary. Each concept is either an entity type or a value type. Ontologies implicitly include a -value-type for each basic data type, like `Integer`, `Decimal`, and `String`, and an entity type -called `Any`. Every other concept ontology extends (is a subtype of) one of these concepts. +set of [built-in concepts](#built-in-concepts), including a value-type for each basic data type, +like `Integer`, `Decimal`, and `String`, and a most-general entity type called `Any`. Every other +concept in an ontology extends (is a subtype of) one of these concepts. -Concepts conform to the following schema: +Concepts have to the following schema: | Field | Type | Required | Description | |-------|------|----------|-------------| @@ -70,11 +91,11 @@ Concepts conform to the following schema: Every user-declared concept extends one or more concepts in the ontology. The new concept is a sutype of each concept that it extends, and the extended concepts are its supertypes. -Any concept that directly or indirectly extends a value type like `Integer` or `String` is a value type. -Any concept that does not extend some value type is an entity type, and if a concept declares no extends -list, then it is assumed to extend the built-in entity type `Any`. If `SocialSecurityNr` extends `Integer` -and `Employee` extends `Person`, which declares no extends list, then `SocialSecurityNr` is a value type -and both `Person` and `Employee` are entity types. +Any value type concept that is introduced in an ontology must either directly or indirectly +extend one of the built-in value types like `Integer` and `String`. + +Entity type concepts can only extend other entity type concepts, and every entity type +implicitly extends the built-in concept `Any`. ### Relationships @@ -137,7 +158,7 @@ using this schema: | Field | Type | Required | Description | |-------|------|----------|-------------| -| `player` | string | Yes | Name of the concept that plays this role | +| `concept` | string | Yes | Name of the concept that plays this role | | `name` | string | No | Optional role name | For instance, in: @@ -166,7 +187,7 @@ ternary relationship `Person.purchased_on` declares two additional roles played The role player often suffices to distinguish the role within its relationship, but when the same concept plays more than one role, the user must declare a distinguising name for -any additional role whose player does not suffice to distinguish it from other roles in +any additional role whose player's name does not distinguish it from other roles in the same relationship. For instance, in: ```yaml @@ -207,11 +228,8 @@ the first n-1 roles. In the special case of a binary relationship, one might declare a `OneToOne` multiplicity, which indicates the relationship is many-to-one in both directions. For instance, the `Person.nr` -relationship is one-to-one because each person is assigned at most one social security number -and each social security number is assigned to at most one person. - -In the absence of any multiplicity, we make no assumptions of functional dependencies among -any of the roles. +relationship is one-to-one because each person is identified by at most one social security number +and each social security number identifies at most one person. ### Identifying relationships @@ -252,22 +270,22 @@ ontologies: roles: - concept: TaxRate derived_by: - - "10.0 WHERE ( Person.files_single AND Person.earns <= 11925 )" - - "10.0 WHERE ( Person.files_married_joint AND Person.earns <= 23850 )" + - "Person.files_single AND Person.earns <= 11925 AND TaxRate == 10.0" + - "Person.files_married_joint AND Person.earns <= 23850 AND TaxRate == 10.0" - ... ``` declares two derived relationships -- `ancestor_of` and `taxed_at`. Each link of `Person.ancestor_of` relates a person to one of its descendants. The two expressions form the base and recursive cases for -this calculation. In the base case, a `Person` as an ancestor of some `descendant` if that `Person` +this calculation. In the base case, a `Person` is an ancestor of some `descendant` if that `Person` is the parent of that descendant. And in the recursive case, a `Person` is an ancestor of some `descendant` if that `Person` is an ancestor of the parent of that `descendant`. Notice in this example how role names are used to disambiguate the two `Person` roles in this relationship. Each link of `Person.taxed_at` links a `Person` object to a `TaxRate` that is derived using expressions that determine the rate based on the person's filing status and how much they earn. -If, for some person, none of the expressions can be evaluated, then the relationship will have -no link involving that person. +If, for some person, none of the expressions can be evaluated, then the relationship will +not link that person. Expressions that derive a relationship are interpreted as rules for constructing the links of the relationship in the same way that a SQL query is interpreted as a rule for constructing the rows @@ -277,7 +295,7 @@ here) then that object will implicitly play the last role, and the expression mu of the other roles explicitly. If an expression does not evaluate to any object, then it must explicitly reference each role. -A derived concept is one whose population is derived from those of its supertype concepts +A derived concept is one whose population is derived from that of its supertype concepts using one or more expressions. For instance: ```yaml @@ -343,12 +361,21 @@ requires any item that has sales in some store to be offered in that store. Ontology mappings declare how to map the values of fields at the logical level to objects and links in the ontology. Just as ontologies are partitioned by concept, ontology maps partition into concept -maps that group by some concept in the ontology. +mappings that group by some concept. + +### Concept mappings + +Each concept mapping declares how to populate a concept with objects and how to populate the relationships +that group under that concept with links. These declarations are formed from patterns of expressions that +reference fields in a logical model that is declared using the OSI core semantic model spec. -Each concept map declares how to populate a concept with objects and how to populate the relationships -that are primarily keyed by that concept with links. These declarations are formed from patterns -of expressions that reference fields in a logical model that is declared using the OSI core semantic -model spec. +Concept mappings have the following schema: + +| Field | Type | Required | Description | +|---------------|---------|-----|-------| +| `concept` | string | Yes | Names the concept whose part of the ontology is covered by this concept mapping | +| `object_mappings` | string | if no `link_mappings` | Mappings that populate this concept | +| `link_mappings` | list | if no `object_mappings` | Mappings that populate the relationships grouped under this concept | ### Object mappings @@ -363,20 +390,18 @@ An object mapping has the following schema: | `expression` | string | if no `referent_mappings` | SQL expression that computes a value from fields | | `referent_mappings` | list | if no `expression` | Referent mappings that find entity objects using identifying realtionships | -When the concept is a value type, an object mapping is just a SQL expression that computes its values. -For instance, an object map that computes `SocialSecurityNumber` values would either retrieve or stitch -together an integer value that satisfies any constraints. +When the concept is a value type or an entity type with a simple identifier, then an object mapping is just +a SQL expression. For instance, given this ontology snippet: -In addition, when a concept is an entity type with a simple identifier -- one relationship that uses some -value type to uniquely reference the concept -- the object map is just a SQL expression that computes the -values of the identifying value type and then maps those values to objects of tha entity type using the -identifier relationship. - -Given this ontology: ```yaml ontologies: - name: EnterpriseOntology concepts: + - concept: + name: SocialSecurityNr + type: ValueType + extends: [ Integer ] + requires: [ "0 < SocialSecurityNr", "SocialSecurityNr <= 999999999" ] - concept: name: Person type: EntityType @@ -388,7 +413,15 @@ ontologies: multiplicity: OneToOne verbalizes: [ "{Person} is identified by {SocialSecurityNr}" ] ``` -this concept mapping: + +an object mapping that computes `SocialSecurityNumber` values would use a SQL expression to retrieve or +stitch together an integer value and check that the value satisfies the constraints on that concept. + +Because `Person` uses a simple identifier -- one that involves one relationship that uses some +value type to uniquely reference the concept -- an object mapping can find its objects using a +SQL expression that computes the values of its identifying value type (`SocialSecurityNr`) and +then mapping those values to `Person` objects using the declared identifier relationship. +The object mapping in: ```yaml concept_mappings: @@ -397,18 +430,16 @@ concept_mappings: - expression: PERSONS.SSN ... ``` -uses an object mapping to declare how values from the `SSN` field of dataset `PERSONS` are -used to form `Person` objects. More precisely, the mapping declares to use `SSN` values to -form `SocialSecurityNr` values that are supplied to `nr` relationship, which is used to -identify `Person`. These additional details can be inferred because the object mapping is -attempting to populate the entity type `Person`, which declares a simple preferred identifier. +maps values from the `SSN` field of dataset `PERSONS` into `Person` objects. More precisely, +the mapping declares to use `SSN` values to form `SocialSecurityNr` values that are supplied +to `nr` relationship, which is used to identify `Person`. -When the concept is an entity type that does not provide a simple preferred identifier, the -object mapping is an array of referent mappings, each of which declares how to use one of its -identifying relationships to find its objects given other objects (values and/or objects of -another entity type). +When an entity-type concept does not provide a simple identifier, the object mapping is an +array of referent mappings, each of which declares how to use one of its identifying +relationships to find its objects given other objects (values and/or objects of another +entity type). -A referent mapping has the following schema: +Referent mappings have the following schema: | Field | Type | Required | Description | |----------------|--------|-----|-------| @@ -459,30 +490,28 @@ The nested referent mappings are needed because `Order` is an entity type. #### Link mappings -A concept mapping's `link_mappings` array describes how to map logical field schema to the -relationships that group under the concept associated with the mapping. -Each link mapping is a tree structure that concisely declares how field values map to the -links of one or more of these relationships. More precisely, the path from the root of a -tree to a node describes how to map to tuples of objects that form the links of the relationship -that is named by the node. These structures leverage the hierarchical nature of YAML to avoid -duplication in the typical case when the fields of a single dataset map to many relationships. +Link mappings describe how to map logical field schema to the relationships that group under the +concept associated with the mapping. These mappings are organized into tree structures to avoid +duplication and clarify mapping intent in the typical case when fields map to objects that play +roles in many different relationships. -Each link mapping has the following schema: +Link mappings have the following schema: | Field | Type | Required | Description | |-------|------|----------|-------------| -| `concept` | string | when level > 1 | Concept whose objects are looked up by this node | -| `value` | string | if no `entity` | Expression that computes a value (when concept is a value type) | -| `entity` | list | if no `value` | Entity map that looks up an entity (when concept is an entity type) | -| `relationship` | string | No | Relationship whose links are mapped to by the path to this node | +| `object_mapping` | object | Yes | Maps to objects in the last position of mapped tuples | +| `relationship` | string | No | Relationship whose links include the tuples mapped to by this mapping | | `children` | list | No | List of child nodes in the tree | -Each node must provide either a `value` or an `entity` by which to look up objects but never both. -The level of each node coincides with the arity of the associated relationship. So a root node -could map to a unary relationship, a node at level 2 could map to a binary relationship, and so -forth. +Semantically, each link mapping uses a pattern of SQL expressions to map to object tuples, which +can then be used to form the links of the relationship that is named by the mapping as prefixes +of longer tuples that are mapped to by its child link mappings. -For instance, the `links` array declared here: +The level of a link mapping must coincide with the arity of the relationship it names. So a +top-level mapping could name a unary relationship, a mapping at level 2 could name to a binary +relationship, and so forth. + +For instance, this ontology snippet: ```yaml ontologies: @@ -510,66 +539,51 @@ ontologies: roles: [ concept: Store, concept: Amount ] verbalizes: [ "{Item} sells in {Store} for {Amount}" ] multiplicitly: ManyToOne -ontology_mappings: - - name: flights_map - description: Example mapping of logical fields to ontology concepts and relationships - ontology: flights - logical_model: - name: Logical_Flights_Model - description: Logical model for flight data - datasets: - ... +``` +declares one unary, one binary, and two ternary relationships whose links would be tuples +of the form (Item), (Item, Store), and (Item, Store, Amount) respectively. And suppose a +logical model declares a dataset called `METRICS` with fields like `SKU`, `STORE`, `SALES`, +and `RETURNS` among many others. This link mapping populates the relationships using these +fields: + +```yaml concept_mappings: - concept: Item - object_mappings: - - referent_mappings: - relationship: Item.nr - expression: ITEMS.SKU link_mappings: - object_mapping: referent_mappings: relationship: Item.nr expression: METRICS.SKU - populates: Item.active + relationship: Item.active children: - object_mapping: concept: Store expression: METRICS.STORE - populates: Item.active_in + relationship: Item.active_in children: - object_mapping: concept: Amount expression: METRICS.SALES - populates: Item.sold_in_for + relationship: Item.sold_in_for - object_mapping: concept: Amount expression: METRICS.RETURNS - populates: Item.returned_in_for + relationship: Item.returned_in_for ``` -describes a tree with one root node, one node at level 2, and two nodes at level 3. -Each node maps fields of the `METRICS` dataset to links of four different relationships, -and notice how the mapping to `Item` objects is declared once even though `Item` plays a -role in all four of the relationships and that the mapping to `Store` objects is declared -once even though `Store` plays a role in three of the relationships. - - ---- +The top level mapping is a tree with one root node, one node at level 2, and two nodes at +level 3. Each node maps fields of the `METRICS` dataset to links of four different relationships, +and notice how the mapping to `Item` objects is declared once even though `Item` plays a role in +all four of the relationships and that the mapping to `Store` objects is declared once even +though `Store` plays a role in three of the relationships. -## Complete Example - -Here's a complete ontology example showing all components working together: - -```yaml ``` ## Version History -- **0.1.1** (2026-05-20): Support for both logical and conceptual modeling layers (ontologies) - - Core ontology structure - - Support for concepts, relationships, and logical -> conceptual schema mappings - - Renamed relationships in the logical layer to join paths to avoid conflict with - relationships in the conceptual layer +- **0.1.1** (2026-05-21): Basic support for ontologies and logical schema mappings + - Core ontology structure: Concepts, relationships, and business rules (requires and derived_by) + - Schema mappings from one or more logical models into an ontology --- From 787a9b80dd79883b4c766f089bd501053c794ec9 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Thu, 21 May 2026 18:23:26 -0400 Subject: [PATCH 13/89] Final version ready for review --- ontology/ontology.md | 128 ++++++++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 56 deletions(-) diff --git a/ontology/ontology.md b/ontology/ontology.md index a409604..df6008e 100644 --- a/ontology/ontology.md +++ b/ontology/ontology.md @@ -50,7 +50,7 @@ Ontologies come with several built in concepts that can be referred to by name: ### Multiplicities -The allowable multiplicities of relationships defined in the [Ontology](#ontology) section. +The allowable multiplicities of relationships defined in the [Relationships](#relationships) section. | Multiplicity | Description | |---------|-------------| @@ -62,22 +62,25 @@ The allowable multiplicities of relationships defined in the [Ontology](#ontolog Ontologies are conceptual models of enterprise data that describe the enterprise in terms of concepts, relationships, and business rules. This specification represents ontologies hierarchically, grouping each relationship under the concept that plays its first role. -Every ontology implicitly includes all of the built-in concepts (see Built-in concepts -enumeration above) and may refer to them by name without declaring them. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Unique name of this ontology | +| `description` | string | No | Human-readable description | +| `ai_context` | string/object | No | Additional context for AI tools | +| `concepts` | list | Yes | Concepts and relationships they group that form this ontology | ### Concepts Concepts represent the types of things that have meaning in a business setting, e.g., person, company, -or salary. Each concept is either an entity type or a value type. Ontologies implicitly include a -set of [built-in concepts](#built-in-concepts), including a value-type for each basic data type, -like `Integer`, `Decimal`, and `String`, and a most-general entity type called `Any`. Every other -concept in an ontology extends (is a subtype of) one of these concepts. +or salary. Every ontology implicitly includes all of the [built-in concepts](#built-in-concepts) and +may refer to them by name without declaring them. -Concepts have to the following schema: +Concepts have the following schema: | Field | Type | Required | Description | |-------|------|----------|-------------| -| `concept` | string | Yes | Unique name of this concept | +| `name` | string | Yes | Unique name of this concept | | `type` | ConceptType | Yes | Entity type or value type | | `description` | string | No | Human-readable description | | `relationships` | list | No | Relationships where this concept plays the first role | @@ -86,17 +89,36 @@ Concepts have to the following schema: | `identify_by` | list | No | Names of relationships that uniquely reference objects of this concept | | `requires` | list | No | Expressions that constrain this concept's population | +Each concept is either an entity type or a value type, and each concept groups any relationships +where that concept plays the first role. + ### Extends Every user-declared concept extends one or more concepts in the ontology. The new concept -is a sutype of each concept that it extends, and the extended concepts are its supertypes. +is a subtype of each concept that it extends, and the extended concepts are its supertypes. -Any value type concept that is introduced in an ontology must either directly or indirectly -extend one of the built-in value types like `Integer` and `String`. +Any value type concept must either directly or indirectly extend one of the built-in value +types like `Integer` and `String`. Entity type concepts can only extend other entity type concepts, and every entity type implicitly extends the built-in concept `Any`. +This ontology snippet: +```yaml +ontologies: + - name: EnterpriseOntology + concepts: + - concept: + name: SocialSecurityNr + type: ValueType + extends: [Integer] + - concept: + name: Employee + type: EntityType + extends: [Person] +``` +declares two concepts that extend other concepts. + ### Relationships Relationships relate objects of one or more concepts and declare how to verbalize links among @@ -233,18 +255,21 @@ and each social security number identifies at most one person. ### Identifying relationships -Many conceptual models distinguish one or more relationships to use when referencing entity-type -objects in expressions and queries. The `Person.nr` relationship can be used to reference a -person by their social security number; while the pair of relationships `License.acct` and -`License.seat_nr` can be used to reference a license by its associated account and seat number. -These identifier relationships are always binary, and their first role is always played by the -concept the relationship is used to reference. +Entity-type objects cannot be referenced directly but must instead be referenced using one or more +relationships whose use allows to reference an entity type using other kinds of objects. Some modeling +methods distinguish a preferred identifier to use when referencing an entity type. For instance, the +`Person.nr` relationship can be used to reference a person by their social security number; while +the pair of relationships `License.acct` and `License.seat_nr` can be used to reference a license by +its associated account and seat number. These relationships are always binary, and their first role +is always played by the referent concept, i.e., the concept that the relationship is used to reference. +The `identify_by` array allows modelers to list the names of relationships that form the preferred +idnetifier of a concept. ### Derivation expressions Concepts and relationships may be derived using expressions. Think of a derived concept or -relationship as a conceptual view whose objects or links are derived from those of other -concepts or relationships. For instance: +relationship as a view whose objects or links are derived from those of other concepts or +relationships. For instance: ```yaml ontologies: @@ -290,10 +315,9 @@ not link that person. Expressions that derive a relationship are interpreted as rules for constructing the links of the relationship in the same way that a SQL query is interpreted as a rule for constructing the rows of a new table. Each expression must therefore reference each role of the relationship, either -explicitly or implicitly. If an expression evaluates to some object (like 10.0 in the two examples -here) then that object will implicitly play the last role, and the expression must reference each -of the other roles explicitly. If an expression does not evaluate to any object, then it must -explicitly reference each role. +explicitly or implicitly. If an expression evaluates to some object then that object will implicitly +play the last role, and the expression must reference each of the other roles explicitly. +If an expression does not evaluate to any object, then it must explicitly reference each role. A derived concept is one whose population is derived from that of its supertype concepts using one or more expressions. For instance: @@ -418,10 +442,9 @@ an object mapping that computes `SocialSecurityNumber` values would use a SQL ex stitch together an integer value and check that the value satisfies the constraints on that concept. Because `Person` uses a simple identifier -- one that involves one relationship that uses some -value type to uniquely reference the concept -- an object mapping can find its objects using a +value type to uniquely reference the concept -- an object mapping can map to its objects using a SQL expression that computes the values of its identifying value type (`SocialSecurityNr`) and -then mapping those values to `Person` objects using the declared identifier relationship. -The object mapping in: +then mapping those values to `Person` objects using the declared identifier. The object mapping in: ```yaml concept_mappings: @@ -430,14 +453,11 @@ concept_mappings: - expression: PERSONS.SSN ... ``` -maps values from the `SSN` field of dataset `PERSONS` into `Person` objects. More precisely, -the mapping declares to use `SSN` values to form `SocialSecurityNr` values that are supplied -to `nr` relationship, which is used to identify `Person`. +maps values from the `SSN` field of dataset `PERSONS` into `Person` objects. -When an entity-type concept does not provide a simple identifier, the object mapping is an -array of referent mappings, each of which declares how to use one of its identifying -relationships to find its objects given other objects (values and/or objects of another -entity type). +When an entity-type concept does not provide a simple identifier, the object mapping uses +an array of referent mappings, each of which declares how to use one of the concept's +identifying relationships to map to its objects from other objects. Referent mappings have the following schema: @@ -478,22 +498,25 @@ concept_mappings: expression: LINEITEMS.L_LINENUMBER - relationship: order referent_mappings: - relationship: CustOrder.nr - expression: LINEITEMS.L_ORDERKEY + - relationship: CustOrder.nr + expression: LINEITEMS.L_ORDERKEY ``` -contains a single object mapping that uses two referent mappings, which: -1. map the `L_LINENUMBER` field to `LineNr` values for the `nr` relationship, and -2. use nested referent mappings to find `Order` objects to supply to the `order` relationship. -The nested referent mappings are needed because `Order` is an entity type. - +contains a single object mapping that uses two referent mappings, which use: +1. an expression to map the `L_LINENUMBER` field to `LineNr` values to provide to the `nr` relationship, and +2. a nested referent mapping that maps to `Order` objects to provide to the `order` relationship. +The nested referent mapping is needed because `Order` is an entity type. #### Link mappings -Link mappings describe how to map logical field schema to the relationships that group under the -concept associated with the mapping. These mappings are organized into tree structures to avoid -duplication and clarify mapping intent in the typical case when fields map to objects that play -roles in many different relationships. +A concept mapping's link mappings describe how to map logical field schema to the relationships that +group under the concept associated with the mapping. + +These mappings are organized into tree structures to avoid duplication and clarify mapping intent in +the typical case when fields map to objects that play roles in many different relationships. Semantically, +each link mapping uses a pattern of SQL expressions to map to object tuples, which can then form the links +of some relationship or can form the prefixes of longer tuples that are mapped to by the mapping's +children. Link mappings have the following schema: @@ -503,10 +526,6 @@ Link mappings have the following schema: | `relationship` | string | No | Relationship whose links include the tuples mapped to by this mapping | | `children` | list | No | List of child nodes in the tree | -Semantically, each link mapping uses a pattern of SQL expressions to map to object tuples, which -can then be used to form the links of the relationship that is named by the mapping as prefixes -of longer tuples that are mapped to by its child link mappings. - The level of a link mapping must coincide with the arity of the relationship it names. So a top-level mapping could name a unary relationship, a mapping at level 2 could name to a binary relationship, and so forth. @@ -554,31 +573,28 @@ fields: referent_mappings: relationship: Item.nr expression: METRICS.SKU - relationship: Item.active + relationship: active children: - object_mapping: concept: Store expression: METRICS.STORE - relationship: Item.active_in + relationship: active_in children: - object_mapping: concept: Amount expression: METRICS.SALES - relationship: Item.sold_in_for + relationship: sold_in_for - object_mapping: concept: Amount expression: METRICS.RETURNS - relationship: Item.returned_in_for + relationship: returned_in_for ``` - The top level mapping is a tree with one root node, one node at level 2, and two nodes at level 3. Each node maps fields of the `METRICS` dataset to links of four different relationships, and notice how the mapping to `Item` objects is declared once even though `Item` plays a role in all four of the relationships and that the mapping to `Store` objects is declared once even though `Store` plays a role in three of the relationships. -``` - ## Version History - **0.1.1** (2026-05-21): Basic support for ontologies and logical schema mappings From 97a3c00b43e94d7dd60f9087bbc12ab8ec0115c5 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Fri, 22 May 2026 14:53:50 -0400 Subject: [PATCH 14/89] Addressed Kuhshboo's comment on naming of the semantic model reference --- examples/flights.yaml | 2 +- ontology/ontology.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/flights.yaml b/examples/flights.yaml index d126588..e0a39be 100644 --- a/examples/flights.yaml +++ b/examples/flights.yaml @@ -99,7 +99,7 @@ ontology_mappings: - name: flights_map description: Example mapping of logical fields to ontology concepts and relationships ontology: flights - logical_model: + semantic_model: name: Logical_Flights_Model description: Logical model for flight data datasets: diff --git a/ontology/ontology.json b/ontology/ontology.json index ad82de3..e890005 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -287,7 +287,7 @@ "type": "string", "description": "Name of the ontology being mapped to" }, - "logical_model": { + "semantic_model": { "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/SemanticModel" }, "concept_mappings": { @@ -298,7 +298,7 @@ "description": "Maps logical model constructs to some concept and its relationships in the ontology" } }, - "required": ["logical_model", "ontology", "concept_mappings"], + "required": ["semantic_model", "ontology", "concept_mappings"], "additionalProperties": false }, "Ontology": { From 33dbd8a44f00402c9e6087f820d17b7dbe6e15a0 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Fri, 22 May 2026 16:32:13 -0400 Subject: [PATCH 15/89] Fixing inconsistency found with markdown and JSON schema --- examples/flights.yaml | 10 +++++----- ontology/ontology.json | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/flights.yaml b/examples/flights.yaml index e0a39be..4162200 100644 --- a/examples/flights.yaml +++ b/examples/flights.yaml @@ -1075,20 +1075,20 @@ ontology_mappings: - object_mapping: concept: Float expression: Example_Runway_example_runway.length - populates: length + relationship: length - object_mapping: concept: String expression: Example_Runway_example_runway.geometry - populates: geometry + relationship: geometry - object_mapping: concept: String expression: Example_Runway_example_runway.designator - populates: designator + relationship: designator - object_mapping: concept: Float expression: Example_Runway_example_runway.shape_length - populates: shape_length + relationship: shape_length - object_mapping: concept: String expression: Example_Runway_example_runway.airport_id - populates: airport_id + relationship: airport_id diff --git a/ontology/ontology.json b/ontology/ontology.json index e890005..f820b01 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -248,7 +248,7 @@ "type": "object", "description": "Mapping from logical schema to the links of relationships in the ontology", "properties": { - "populates": { + "relationship": { "type": "string", "description": "Name of relationship being populated by this mapping node" }, From 156f641df03a433854d53f6bf8370d3749b66f03 Mon Sep 17 00:00:00 2001 From: Baruch Oxman Date: Wed, 27 May 2026 15:42:20 +0300 Subject: [PATCH 16/89] =?UTF-8?q?Add=20bidirectional=20Honeydew=20?= =?UTF-8?q?=E2=86=94=20OSI=20converter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements OSI → Honeydew and Honeydew → OSI conversion with full round-trip fidelity. Relationship names are stored natively on the relation, and relations are always placed on the many side. Co-Authored-By: Claude Sonnet 4.6 --- converters/honeydew/README.md | 68 ++ converters/honeydew/requirements.txt | 2 + .../honeydew/src/honeydew_osi_converter.py | 946 ++++++++++++++++++ .../tests/test_honeydew_osi_converter.py | 888 ++++++++++++++++ 4 files changed, 1904 insertions(+) create mode 100644 converters/honeydew/README.md create mode 100644 converters/honeydew/requirements.txt create mode 100644 converters/honeydew/src/honeydew_osi_converter.py create mode 100644 converters/honeydew/tests/test_honeydew_osi_converter.py diff --git a/converters/honeydew/README.md b/converters/honeydew/README.md new file mode 100644 index 0000000..eb0e54b --- /dev/null +++ b/converters/honeydew/README.md @@ -0,0 +1,68 @@ +# OSI ↔ Honeydew Converter + +Bidirectional converter between [OSI](../../core-spec/spec.md) semantic models and [Honeydew](https://docs.honeydew.ai) workspace YAML. + +## Overview + +| Direction | Input | Output | +|-----------|-------|--------| +| `osi-to-honeydew` | Single OSI YAML file | Honeydew workspace directory | +| `honeydew-to-osi` | Honeydew workspace directory | Single OSI YAML file | + +### OSI → Honeydew mapping + +| OSI concept | Honeydew concept | +|-------------|-----------------| +| `semantic_model.name` | `workspace.yml name` | +| `dataset` | Entity + dataset files under `schema//` | +| `dataset.source` | `dataset.sql` | +| `dataset.primary_key` | `entity.keys` | +| Simple column field | `dataset.attributes` entry | +| Computed field expression | `calculated_attribute` YAML | +| `relationship` (from → to) | `entity.relations` on the "from" entity (`rel_type: many-to-one`) | +| `metric` | `metric` YAML (assigned to entity by expression parse) | + +### Honeydew → OSI mapping + +| Honeydew concept | OSI concept | +|-----------------|-------------| +| `workspace.name` | `semantic_model.name` | +| Entity + primary dataset | `dataset` | +| `entity.keys` | `dataset.primary_key` | +| `dataset.attributes` (columns) | `fields` with `ANSI_SQL` expression = column name | +| `calculated_attribute` SQL | `fields` with `ANSI_SQL` expression + `HONEYDEW` custom extension | +| `entity.relations` (`many-to-one`) | `relationship` with `from` = this entity | +| `entity.relations` (`one-to-many`) | `relationship` with `from` = target entity | +| `metric.sql` | `metric` expression in `ANSI_SQL` dialect | + +## Setup + +```bash +pip install -r requirements.txt +``` + +## Usage + +```bash +# OSI YAML → Honeydew workspace directory +python src/honeydew_osi_converter.py osi-to-honeydew -i input.yaml -o output_dir/ + +# Honeydew workspace directory → OSI YAML +python src/honeydew_osi_converter.py honeydew-to-osi -i workspace_dir/ -o output.yaml +``` + +## Tests + +```bash +python -m pytest tests/ +``` + +## Limitations + +- **One dataset per entity**: The converter maps each OSI dataset to a single Honeydew entity with one source dataset. Multiple datasets per entity are not generated. +- **Datatype inference**: OSI fields have no explicit datatype; the converter infers Honeydew datatypes from the `dimension.is_time` flag (`timestamp`) and the presence/absence of the `dimension` key (`string` vs `number`). +- **Honeydew SQL expressions**: Calculated attributes and metrics use Honeydew's `entity.attribute` reference syntax. These are exported as `ANSI_SQL` dialect expressions in OSI; they remain valid for round-tripping but may not run on other databases without adaptation. +- **Filters**: Honeydew `filter` objects have no OSI equivalent and are not exported. +- **Perspectives and domains**: Not converted (no OSI equivalent). +- **Connection expressions** (`connection_expr`): Preserved in `HONEYDEW` custom extensions on the OSI relationship. +- **`ai_context`**: OSI `ai_context` fields (synonyms, instructions) are dropped during OSI → Honeydew conversion (no native Honeydew equivalent). Honeydew `description` fields are mapped to OSI `description`. diff --git a/converters/honeydew/requirements.txt b/converters/honeydew/requirements.txt new file mode 100644 index 0000000..2f29b5b --- /dev/null +++ b/converters/honeydew/requirements.txt @@ -0,0 +1,2 @@ +PyYAML>=5.0 +pytest>=7.0 diff --git a/converters/honeydew/src/honeydew_osi_converter.py b/converters/honeydew/src/honeydew_osi_converter.py new file mode 100644 index 0000000..0fb9e0a --- /dev/null +++ b/converters/honeydew/src/honeydew_osi_converter.py @@ -0,0 +1,946 @@ +""" +Bidirectional converter between OSI and Honeydew semantic model formats. + +OSI → Honeydew: Converts a single OSI YAML file into a Honeydew workspace + directory (multiple YAML files per entity). + +Honeydew → OSI: Reads a Honeydew workspace directory and produces an OSI YAML. + +Usage: + python honeydew_osi_converter.py osi-to-honeydew -i input.yaml -o output_dir/ + python honeydew_osi_converter.py honeydew-to-osi -i workspace_dir/ -o output.yaml +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import warnings +from typing import Any + +import yaml + +SUPPORTED_OSI_VERSION = "0.2.0.dev0" +HONEYDEW_VENDOR = "HONEYDEW" +_OSI_METADATA_SECTION = "osi" + + +class HoneydewConversionError(Exception): + """Raised when conversion between OSI and Honeydew fails.""" + + +# ───────────────────────────────────────────────────────────────────────────── +# OSI → Honeydew +# ───────────────────────────────────────────────────────────────────────────── + + +def convert_osi_to_honeydew(osi_yaml_str: str) -> dict[str, str]: + """Convert an OSI YAML string to a Honeydew workspace file tree. + + Returns a dict mapping relative file paths to their YAML content strings. + The caller writes these to disk under the desired output directory. + + Honeydew workspace structure produced:: + + workspace.yml + schema// + .yml + datasets/.yml + attributes/.yml (computed fields only) + metrics/.yml + + OSI fields with no direct Honeydew equivalent (``ai_context``, + ``unique_keys``, non-Honeydew ``custom_extensions``, relationship + ``name``) are stored in the Honeydew ``metadata`` section under a section + named ``"osi"`` so they can be recovered on the return trip. + + Args: + osi_yaml_str: OSI YAML document as a string. + + Returns: + Dict of {relative_path: yaml_content}. + + Raises: + HoneydewConversionError: On invalid or unsupported input. + """ + root = yaml.safe_load(osi_yaml_str) + if not isinstance(root, dict): + raise HoneydewConversionError("Invalid OSI YAML: expected a mapping at the root") + + version_str = str(root.get("version", "")) + if version_str != SUPPORTED_OSI_VERSION: + raise HoneydewConversionError( + f"Unsupported OSI version '{version_str}'. Supported: {SUPPORTED_OSI_VERSION}" + ) + + semantic_models = root.get("semantic_model") + if not isinstance(semantic_models, list) or not semantic_models: + raise HoneydewConversionError("'semantic_model' must be a non-empty list") + + if len(semantic_models) > 1: + warnings.warn( + f"OSI YAML contains {len(semantic_models)} semantic models; " + "only the first will be converted" + ) + + return _model_to_files(semantic_models[0]) + + +def _model_to_files(sm: dict[str, Any]) -> dict[str, str]: + name = sm.get("name") + if not name: + raise HoneydewConversionError("Missing 'name' in semantic model") + + files: dict[str, str] = {} + + workspace: dict[str, Any] = {"type": "workspace", "name": name} + if sm.get("description"): + workspace["description"] = sm["description"] + + # Preserve model-level ai_context and non-HONEYDEW custom_extensions + model_ai_ctx = sm.get("ai_context") + model_ext = [e for e in (sm.get("custom_extensions") or []) if e.get("vendor_name") != HONEYDEW_VENDOR] + ws_meta = _build_osi_metadata(ai_context=model_ai_ctx, custom_extensions=model_ext or None) + if ws_meta: + workspace["metadata"] = [ws_meta] + + files["workspace.yml"] = _dump(workspace) + + datasets = sm.get("datasets") or [] + metrics = sm.get("metrics") or [] + relationships = sm.get("relationships") or [] + + entity_names = [ds["name"] for ds in datasets if ds.get("name")] + + # Group OSI relationships by from-entity + rel_by_entity: dict[str, list[dict[str, Any]]] = {} + for rel in relationships: + from_ds = rel.get("from") + if from_ds: + rel_by_entity.setdefault(from_ds, []).append(rel) + + # Assign OSI metrics to entities (honours HONEYDEW entity hint for round-trips) + metric_by_entity = _assign_metrics_to_entities(metrics, entity_names) + + for ds in datasets: + entity_name = ds.get("name") + if not entity_name: + raise HoneydewConversionError("Dataset missing 'name'") + files.update( + _dataset_to_files( + ds, + rel_by_entity.get(entity_name, []), + metric_by_entity.get(entity_name, []), + ) + ) + + return files + + +def _dataset_to_files( + ds: dict[str, Any], + relations: list[dict[str, Any]], + metrics: list[dict[str, Any]], +) -> dict[str, str]: + entity_name = ds["name"] + base = f"schema/{entity_name}" + files: dict[str, str] = {} + + primary_key = ds.get("primary_key") or [] + unique_keys = ds.get("unique_keys") + description = ds.get("description") + ai_context = ds.get("ai_context") + fields = ds.get("fields") or [] + ds_ext = [e for e in (ds.get("custom_extensions") or []) if e.get("vendor_name") != HONEYDEW_VENDOR] + + # ── entity YAML ──────────────────────────────────────────────────────────── + entity_dict: dict[str, Any] = {"type": "entity", "name": entity_name} + if description: + entity_dict["description"] = description + if primary_key: + entity_dict["keys"] = list(primary_key) + entity_dict["key_dataset"] = entity_name + + honeydew_relations = [] + for rel in relations: + hr = _osi_relation_to_honeydew(rel) + if hr is not None: + honeydew_relations.append(hr) + entity_dict["relations"] = honeydew_relations + + # Preserve OSI fields that have no Honeydew native equivalent + entity_meta = _build_osi_metadata( + ai_context=ai_context, + unique_keys=unique_keys, + custom_extensions=ds_ext or None, + ) + if entity_meta: + entity_dict["metadata"] = [entity_meta] + + files[f"{base}/{entity_name}.yml"] = _dump(entity_dict) + + # ── classify fields into dataset attributes vs calculated attributes ──────── + dataset_attrs: list[dict[str, Any]] = [] + calc_attrs: list[dict[str, Any]] = [] + + for field in fields: + field_name = field.get("name") + if not field_name: + raise HoneydewConversionError(f"Field missing 'name' in dataset '{entity_name}'") + + expr = _pick_ansi_expression(field.get("expression"), field_name) + if expr is None: + continue + + datatype = _osi_field_to_honeydew_datatype(field) + field_desc = field.get("description") + field_label = field.get("label") + field_ai_ctx = field.get("ai_context") + field_ext = [e for e in (field.get("custom_extensions") or []) if e.get("vendor_name") != HONEYDEW_VENDOR] + + # Merge ai_context instructions into description; keep full object in metadata + effective_desc = field_desc + if isinstance(field_ai_ctx, str) and field_ai_ctx: + effective_desc = f"{field_desc}\n{field_ai_ctx}" if field_desc else field_ai_ctx + elif isinstance(field_ai_ctx, dict) and field_ai_ctx.get("instructions"): + instr = field_ai_ctx["instructions"] + effective_desc = f"{field_desc}\n{instr}" if field_desc else instr + + # Build labels: OSI label + ai_context synonyms + labels: list[str] = [] + if field_label: + labels.append(field_label) + if isinstance(field_ai_ctx, dict): + for syn in (field_ai_ctx.get("synonyms") or []): + if syn not in labels: + labels.append(syn) + + field_meta = _build_osi_metadata( + ai_context=field_ai_ctx if isinstance(field_ai_ctx, dict) else None, + custom_extensions=field_ext or None, + ) + + if _is_simple_identifier(expr): + attr: dict[str, Any] = {"column": expr, "name": field_name, "datatype": datatype} + if effective_desc: + attr["description"] = effective_desc + if labels: + attr["labels"] = labels + if field_meta: + attr["metadata"] = [field_meta] + dataset_attrs.append(attr) + else: + calc: dict[str, Any] = { + "type": "calculated_attribute", + "entity": entity_name, + "name": field_name, + "datatype": datatype, + "sql": expr, + } + if effective_desc: + calc["description"] = effective_desc + if labels: + calc["labels"] = labels + if field_meta: + calc["metadata"] = [field_meta] + calc_attrs.append(calc) + + # ── dataset YAML ─────────────────────────────────────────────────────────── + source_sql, dataset_type = _parse_osi_source(ds.get("source", "")) + dataset_dict: dict[str, Any] = { + "type": "dataset", + "entity": entity_name, + "name": entity_name, + "sql": source_sql, + "dataset_type": dataset_type, + "attributes": dataset_attrs, + } + if description: + dataset_dict["description"] = description + + files[f"{base}/datasets/{entity_name}.yml"] = _dump(dataset_dict) + + # ── calculated_attribute YAMLs ───────────────────────────────────────────── + for calc in calc_attrs: + files[f"{base}/attributes/{calc['name']}.yml"] = _dump(calc) + + # ── metric YAMLs ──────────────────────────────────────────────────────────── + for metric in metrics: + mname = metric.get("name") + if not mname: + continue + mexpr = _pick_ansi_expression(metric.get("expression"), mname) + if mexpr is None: + continue + + metric_dict: dict[str, Any] = { + "type": "metric", + "entity": entity_name, + "name": mname, + "datatype": "number", + "sql": mexpr, + } + if metric.get("description"): + metric_dict["description"] = metric["description"] + + metric_ai_ctx = metric.get("ai_context") + if isinstance(metric_ai_ctx, str) and metric_ai_ctx: + existing = metric_dict.get("description", "") + metric_dict["description"] = f"{existing}\n{metric_ai_ctx}".strip() if existing else metric_ai_ctx + + metric_ext = [e for e in (metric.get("custom_extensions") or []) if e.get("vendor_name") != HONEYDEW_VENDOR] + metric_meta = _build_osi_metadata( + ai_context=metric_ai_ctx if isinstance(metric_ai_ctx, dict) else None, + custom_extensions=metric_ext or None, + ) + if metric_meta: + metric_dict["metadata"] = [metric_meta] + + files[f"{base}/metrics/{mname}.yml"] = _dump(metric_dict) + + return files + + +def _osi_relation_to_honeydew(rel: dict[str, Any]) -> dict[str, Any] | None: + rel_name = rel.get("name", "") + to_ds = rel.get("to") + if not to_ds: + warnings.warn(f"Relationship '{rel_name}' missing 'to', skipping") + return None + + from_cols = rel.get("from_columns") or [] + to_cols = rel.get("to_columns") or [] + + if len(from_cols) != len(to_cols): + raise HoneydewConversionError( + f"Relationship '{rel_name}': from_columns and to_columns length mismatch " + f"({len(from_cols)} vs {len(to_cols)})" + ) + + honeydew_rel: dict[str, Any] = { + "target_entity": to_ds, + "rel_type": "many-to-one", + } + if rel.get("name"): + honeydew_rel["name"] = rel["name"] + if from_cols: + honeydew_rel["connection"] = [ + {"src_field": fc, "target_field": tc} + for fc, tc in zip(from_cols, to_cols) + ] + return honeydew_rel + + +def _pick_ansi_expression(expression: Any, field_name: str) -> str | None: + """Select the ANSI_SQL expression; fall back to first available dialect.""" + if not isinstance(expression, dict): + return None + dialects = expression.get("dialects") or [] + if not dialects: + return None + + ansi_expr = None + first_expr = None + + for d in dialects: + dialect = (d.get("dialect") or "").upper() + expr = d.get("expression") + if first_expr is None: + first_expr = expr + if dialect == "ANSI_SQL" and ansi_expr is None: + ansi_expr = expr + + if ansi_expr is not None: + return ansi_expr + + if first_expr is not None: + warnings.warn(f"'{field_name}': no ANSI_SQL dialect found; using first available") + return first_expr + + return None + + +def _osi_field_to_honeydew_datatype(field: dict[str, Any]) -> str: + dimension = field.get("dimension") + if isinstance(dimension, dict) and dimension.get("is_time"): + return "timestamp" + if dimension is not None: + return "string" + return "number" + + +def _is_simple_identifier(expr: str) -> bool: + return bool(re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", expr.strip())) + + +def _parse_osi_source(source: str) -> tuple[str, str]: + source = (source or "").strip() + if not source: + return ("", "table") + upper = source.upper() + if upper.startswith(("SELECT ", "SELECT\n", "SELECT\t", "WITH ", "WITH\n", "WITH\t")): + return (source, "sql") + return (source, "table") + + +def _assign_metrics_to_entities( + metrics: list[dict[str, Any]], + entity_names: list[str], +) -> dict[str, list[dict[str, Any]]]: + """Assign each OSI metric to the most appropriate Honeydew entity. + + Priority: + 1. HONEYDEW ``custom_extension`` entity hint (preserves round-trip placement) + 2. First ``entity.column`` pattern in the ANSI_SQL expression + 3. First entity in the model (with a warning) + """ + entity_set = set(entity_names) + result: dict[str, list[dict[str, Any]]] = {} + + for metric in metrics: + mname = metric.get("name", "") + + # Priority 1: HONEYDEW entity hint (set during Honeydew → OSI) + hinted = _get_honeydew_extension(metric).get("entity") + if hinted and hinted in entity_set: + result.setdefault(hinted, []).append(metric) + continue + + # Priority 2: expression scan + expr_dict = metric.get("expression") or {} + dialects = expr_dict.get("dialects") or [] if isinstance(expr_dict, dict) else [] + expr_str = "" + for d in dialects: + if (d.get("dialect") or "").upper() == "ANSI_SQL": + expr_str = d.get("expression") or "" + break + if not expr_str and dialects: + expr_str = dialects[0].get("expression") or "" + + assigned = _find_entity_in_expression(expr_str, entity_set) + + # Priority 3: fallback + if assigned is None: + if entity_names: + assigned = entity_names[0] + warnings.warn( + f"Metric '{mname}': no entity reference found in expression; " + f"assigning to '{assigned}'" + ) + else: + warnings.warn(f"Metric '{mname}': no entities to assign to, skipping") + continue + + result.setdefault(assigned, []).append(metric) + + return result + + +def _find_entity_in_expression(expr: str, entity_names: set[str]) -> str | None: + for match in re.finditer(r"\b([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\b", expr): + if match.group(1) in entity_names: + return match.group(1) + return None + + +# ───────────────────────────────────────────────────────────────────────────── +# Honeydew → OSI +# ───────────────────────────────────────────────────────────────────────────── + + +def convert_honeydew_to_osi(workspace_dir: str) -> str: + """Convert a Honeydew workspace directory to an OSI YAML string. + + Reads workspace.yml and all entity subdirectories under schema/. Honeydew + fields with no OSI equivalent (``owner``, ``display_name``, ``hidden``, + ``format_string``, ``timegrain``, attribute ``labels``) are preserved in a + HONEYDEW ``custom_extension`` so they survive a round-trip back to Honeydew. + + Args: + workspace_dir: Path to the Honeydew workspace root. + + Returns: + OSI YAML document string. + + Raises: + HoneydewConversionError: On missing workspace.yml. + """ + workspace_path = os.path.join(workspace_dir, "workspace.yml") + if not os.path.exists(workspace_path): + raise HoneydewConversionError(f"workspace.yml not found in '{workspace_dir}'") + + with open(workspace_path) as f: + workspace = yaml.safe_load(f) or {} + + model_name = workspace.get("name") or os.path.basename(workspace_dir.rstrip("/\\")) + model_description = workspace.get("description") + ws_osi_meta = _read_osi_metadata(workspace) + + schema_dir = os.path.join(workspace_dir, "schema") + entity_dirs: list[str] = [] + if os.path.isdir(schema_dir): + entity_dirs = sorted( + d for d in os.listdir(schema_dir) + if os.path.isdir(os.path.join(schema_dir, d)) + ) + + osi_datasets: list[dict[str, Any]] = [] + osi_relationships: list[dict[str, Any]] = [] + osi_metrics: list[dict[str, Any]] = [] + seen_relationships: set[tuple] = set() + + for entity_name in entity_dirs: + entity_dir = os.path.join(schema_dir, entity_name) + entity_data = _read_entity_dir(entity_dir, entity_name) + + osi_datasets.append(_entity_to_osi_dataset(entity_data)) + + for rel in entity_data["relations"]: + osi_rel = _honeydew_relation_to_osi( + rel, entity_name, seen_relationships + ) + if osi_rel is not None: + osi_relationships.append(osi_rel) + + for metric in entity_data["metrics"]: + osi_m = _honeydew_metric_to_osi(metric, entity_name) + if osi_m is not None: + osi_metrics.append(osi_m) + + sm: dict[str, Any] = {"name": model_name, "datasets": osi_datasets} + if model_description: + sm["description"] = str(model_description).strip() + if ws_osi_meta.get("ai_context"): + sm["ai_context"] = ws_osi_meta["ai_context"] + + restored_ws_ext = ws_osi_meta.get("custom_extensions") or [] + if restored_ws_ext: + sm["custom_extensions"] = restored_ws_ext + + if osi_relationships: + sm["relationships"] = osi_relationships + if osi_metrics: + sm["metrics"] = osi_metrics + + root: dict[str, Any] = { + "version": SUPPORTED_OSI_VERSION, + "vendors": [HONEYDEW_VENDOR], + "semantic_model": [sm], + } + return _dump(root) + + +def _read_entity_dir(entity_dir: str, entity_name: str) -> dict[str, Any]: + data: dict[str, Any] = { + "name": entity_name, + "description": None, + "keys": [], + "key_dataset": None, + "relations": [], + "primary_dataset": None, + "calculated_attributes": [], + "metrics": [], + "osi_meta": {}, + "honeydew_extra": {}, + } + + entity_yml = os.path.join(entity_dir, f"{entity_name}.yml") + if os.path.exists(entity_yml): + with open(entity_yml) as f: + ey = yaml.safe_load(f) or {} + data["keys"] = _coerce_list(ey.get("keys")) + data["description"] = ey.get("description") + data["key_dataset"] = ey.get("key_dataset") + data["relations"] = ey.get("relations") or [] + data["osi_meta"] = _read_osi_metadata(ey) + data["honeydew_extra"] = { + k: ey[k] for k in ("owner", "display_name", "hidden", "folder", "labels") + if k in ey + } + + datasets_dir = os.path.join(entity_dir, "datasets") + if os.path.isdir(datasets_dir): + all_ds: list[dict[str, Any]] = [] + for fn in sorted(os.listdir(datasets_dir)): + if fn.endswith((".yml", ".yaml")): + with open(os.path.join(datasets_dir, fn)) as f: + all_ds.append(yaml.safe_load(f) or {}) + for ds in all_ds: + if ds.get("name") == data["key_dataset"] or data["primary_dataset"] is None: + data["primary_dataset"] = ds + if ds.get("name") == data["key_dataset"]: + break + + attrs_dir = os.path.join(entity_dir, "attributes") + if os.path.isdir(attrs_dir): + for fn in sorted(os.listdir(attrs_dir)): + if fn.endswith((".yml", ".yaml")): + with open(os.path.join(attrs_dir, fn)) as f: + data["calculated_attributes"].append(yaml.safe_load(f) or {}) + + metrics_dir = os.path.join(entity_dir, "metrics") + if os.path.isdir(metrics_dir): + for fn in sorted(os.listdir(metrics_dir)): + if fn.endswith((".yml", ".yaml")): + with open(os.path.join(metrics_dir, fn)) as f: + data["metrics"].append(yaml.safe_load(f) or {}) + + return data + + +def _entity_to_osi_dataset(entity_data: dict[str, Any]) -> dict[str, Any]: + entity_name = entity_data["name"] + ds: dict[str, Any] = {"name": entity_name} + + if entity_data.get("description"): + ds["description"] = str(entity_data["description"]).strip() + + primary_ds = entity_data.get("primary_dataset") + ds["source"] = (primary_ds.get("sql") or "").strip() if primary_ds else entity_name + + keys = entity_data.get("keys") or [] + if keys: + ds["primary_key"] = list(keys) + + # Restore OSI-only fields preserved in Honeydew metadata + osi_meta = entity_data.get("osi_meta") or {} + if osi_meta.get("ai_context"): + ds["ai_context"] = osi_meta["ai_context"] + if osi_meta.get("unique_keys"): + ds["unique_keys"] = osi_meta["unique_keys"] + + restored_ext = list(osi_meta.get("custom_extensions") or []) + honeydew_extra = entity_data.get("honeydew_extra") or {} + if honeydew_extra: + restored_ext.append({"vendor_name": HONEYDEW_VENDOR, "data": json.dumps(honeydew_extra)}) + if restored_ext: + ds["custom_extensions"] = restored_ext + + # Build fields + fields: list[dict[str, Any]] = [] + seen: set[str] = set() + + if primary_ds: + for attr in primary_ds.get("attributes") or []: + col = attr.get("column") or attr.get("name") or "" + aname = attr.get("name") or col + if not aname or aname in seen: + continue + seen.add(aname) + + field: dict[str, Any] = { + "name": aname, + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": col}]}, + } + datatype = attr.get("datatype") or "string" + dim = _honeydew_datatype_to_osi_dimension(datatype) + if dim is not None: + field["dimension"] = dim + if attr.get("description"): + field["description"] = str(attr["description"]).strip() + + attr_osi_meta = _read_osi_metadata(attr) + attr_labels = attr.get("labels") or [] + + # Restore ai_context (structured form takes priority, else build from labels) + if attr_osi_meta.get("ai_context"): + field["ai_context"] = attr_osi_meta["ai_context"] + elif attr_labels: + field["ai_context"] = {"synonyms": list(attr_labels)} + + # Restore label (first Honeydew label maps to OSI label) + if attr_labels: + field["label"] = attr_labels[0] + + # Honeydew-specific metadata → HONEYDEW custom_extension + attr_honeydew_extra = { + k: attr[k] for k in ("display_name", "hidden", "folder", "format_string", "timegrain") + if k in attr + } + if len(attr_labels) > 1: + attr_honeydew_extra["labels"] = attr_labels + + all_ext = list(attr_osi_meta.get("custom_extensions") or []) + if attr_honeydew_extra: + all_ext.append({"vendor_name": HONEYDEW_VENDOR, "data": json.dumps(attr_honeydew_extra)}) + if all_ext: + field["custom_extensions"] = all_ext + + fields.append(field) + + for calc in entity_data.get("calculated_attributes") or []: + aname = calc.get("name") or "" + if not aname or aname in seen: + continue + seen.add(aname) + + sql = (calc.get("sql") or "").strip() + datatype = calc.get("datatype") or "string" + + field = { + "name": aname, + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": sql}]}, + } + dim = _honeydew_datatype_to_osi_dimension(datatype) + if dim is not None: + field["dimension"] = dim + if calc.get("description"): + cleaned = str(calc["description"]).strip() + if cleaned: + field["description"] = cleaned + + calc_osi_meta = _read_osi_metadata(calc) + calc_labels = calc.get("labels") or [] + + if calc_osi_meta.get("ai_context"): + field["ai_context"] = calc_osi_meta["ai_context"] + elif calc_labels: + field["ai_context"] = {"synonyms": list(calc_labels)} + + if calc_labels: + field["label"] = calc_labels[0] + + calc_honeydew_extra = { + k: calc[k] for k in ("display_name", "hidden", "folder", "format_string", "timegrain") + if k in calc + } + + all_calc_ext = list(calc_osi_meta.get("custom_extensions") or []) + # Always mark as calculated_attribute so OSI → Honeydew routes it correctly + all_calc_ext.append({ + "vendor_name": HONEYDEW_VENDOR, + "data": json.dumps(dict({"type": "calculated_attribute", "entity": entity_name}, **calc_honeydew_extra)), + }) + field["custom_extensions"] = all_calc_ext + + fields.append(field) + + if fields: + ds["fields"] = fields + + return ds + + +def _honeydew_datatype_to_osi_dimension(datatype: str) -> dict[str, Any] | None: + dt = (datatype or "").lower() + if dt in ("date", "timestamp", "time"): + return {"is_time": True} + if dt in ("bool", "string"): + return {"is_time": False} + return None # number / float → OSI fact (no dimension key) + + +def _honeydew_relation_to_osi( + rel: dict[str, Any], + entity_name: str, + seen: set[tuple], +) -> dict[str, Any] | None: + target = rel.get("target_entity") + if not target: + warnings.warn(f"Entity '{entity_name}': relation missing target_entity, skipping") + return None + + rel_type = (rel.get("rel_type") or "many-to-one").lower() + connection = rel.get("connection") or [] + connection_expr = rel.get("connection_expr") + + if rel_type == "many-to-one": + from_entity, to_entity = entity_name, target + from_cols = [c.get("src_field", "") for c in connection] + to_cols = [c.get("target_field", "") for c in connection] + elif rel_type == "one-to-many": + from_entity, to_entity = target, entity_name + from_cols = [c.get("target_field", "") for c in connection] + to_cols = [c.get("src_field", "") for c in connection] + else: + from_entity, to_entity = entity_name, target + from_cols = [c.get("src_field", "") for c in connection] + to_cols = [c.get("target_field", "") for c in connection] + + dedup_key = (from_entity, to_entity, tuple(from_cols), tuple(to_cols)) + if dedup_key in seen: + return None + seen.add(dedup_key) + + rel_name = rel.get("name") or f"{from_entity}_to_{to_entity}" + + osi_rel: dict[str, Any] = {"name": rel_name, "from": from_entity, "to": to_entity} + if from_cols: + osi_rel["from_columns"] = from_cols + osi_rel["to_columns"] = to_cols + + if connection_expr and not connection: + sql_expr = (connection_expr.get("sql") or "") if isinstance(connection_expr, dict) else str(connection_expr) + osi_rel["custom_extensions"] = [ + {"vendor_name": HONEYDEW_VENDOR, "data": json.dumps({"connection_expr": sql_expr})} + ] + + return osi_rel + + +def _honeydew_metric_to_osi(metric: dict[str, Any], entity_name: str) -> dict[str, Any] | None: + mname = metric.get("name") or "" + if not mname: + return None + + sql = (metric.get("sql") or "").strip() + if not sql: + warnings.warn(f"Metric '{mname}' in entity '{entity_name}' has no SQL, skipping") + return None + + osi_m: dict[str, Any] = { + "name": mname, + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": sql}]}, + "custom_extensions": [ + {"vendor_name": HONEYDEW_VENDOR, "data": json.dumps({"entity": entity_name})} + ], + } + + if metric.get("description"): + cleaned = str(metric["description"]).strip() + if cleaned: + osi_m["description"] = cleaned + + metric_osi_meta = _read_osi_metadata(metric) + if metric_osi_meta.get("ai_context"): + osi_m["ai_context"] = metric_osi_meta["ai_context"] + + restored_ext = metric_osi_meta.get("custom_extensions") or [] + if restored_ext: + osi_m["custom_extensions"] = osi_m["custom_extensions"] + list(restored_ext) + + return osi_m + + +# ───────────────────────────────────────────────────────────────────────────── +# OSI metadata helpers — store/restore OSI fields in Honeydew metadata sections +# ───────────────────────────────────────────────────────────────────────────── + + +def _build_osi_metadata( + *, + ai_context: Any = None, + unique_keys: Any = None, + custom_extensions: list | None = None, +) -> dict[str, Any] | None: + """Build a Honeydew metadata entry that stores OSI-only fields for round-tripping.""" + items: list[dict[str, Any]] = [] + + if ai_context is not None: + val = ai_context if isinstance(ai_context, str) else json.dumps(ai_context) + items.append({"name": "ai_context", "value": val}) + if unique_keys: + items.append({"name": "unique_keys", "value": json.dumps(unique_keys)}) + if custom_extensions: + items.append({"name": "custom_extensions", "value": json.dumps(custom_extensions)}) + + if not items: + return None + return {"name": _OSI_METADATA_SECTION, "metadata": items} + + +def _read_osi_metadata(obj: dict[str, Any]) -> dict[str, Any]: + """Read OSI-preserved fields from a Honeydew object's 'osi' metadata section.""" + for section in (obj.get("metadata") or []): + if (section.get("name") or "") != _OSI_METADATA_SECTION: + continue + result: dict[str, Any] = {} + for item in (section.get("metadata") or []): + key = item.get("name") or "" + raw = item.get("value") + if key == "ai_context": + try: + result[key] = json.loads(raw) + except (json.JSONDecodeError, TypeError): + result[key] = raw + elif key in ("unique_keys", "custom_extensions"): + try: + result[key] = json.loads(raw) + except (json.JSONDecodeError, TypeError): + pass + return result + return {} + + +def _get_honeydew_extension(obj: dict[str, Any]) -> dict[str, Any]: + """Extract the HONEYDEW custom_extension data from an OSI object.""" + for ext in (obj.get("custom_extensions") or []): + if ext.get("vendor_name") == HONEYDEW_VENDOR: + try: + return json.loads(ext.get("data") or "{}") + except (json.JSONDecodeError, TypeError): + return {} + return {} + + +# ───────────────────────────────────────────────────────────────────────────── +# Utilities +# ───────────────────────────────────────────────────────────────────────────── + + +def _coerce_list(value: Any) -> list: + if value is None: + return [] + if isinstance(value, list): + return value + return [value] + + +def _dump(data: Any) -> str: + return yaml.dump(data, default_flow_style=False, sort_keys=False, allow_unicode=True) + + +# ───────────────────────────────────────────────────────────────────────────── +# CLI +# ───────────────────────────────────────────────────────────────────────────── + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Bidirectional OSI ↔ Honeydew semantic model converter" + ) + sub = parser.add_subparsers(dest="command", required=True) + + p1 = sub.add_parser("osi-to-honeydew", help="Convert OSI YAML → Honeydew workspace") + p1.add_argument("-i", "--input", required=True, help="OSI YAML input file") + p1.add_argument("-o", "--output", required=True, help="Output directory for Honeydew workspace") + + p2 = sub.add_parser("honeydew-to-osi", help="Convert Honeydew workspace → OSI YAML") + p2.add_argument("-i", "--input", required=True, help="Honeydew workspace directory") + p2.add_argument("-o", "--output", required=True, help="OSI YAML output file") + + args = parser.parse_args() + + if args.command == "osi-to-honeydew": + with open(args.input) as f: + osi_yaml = f.read() + try: + files = convert_osi_to_honeydew(osi_yaml) + except HoneydewConversionError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + for rel_path, content in files.items(): + full_path = os.path.join(args.output, rel_path) + os.makedirs(os.path.dirname(full_path), exist_ok=True) + with open(full_path, "w") as f: + f.write(content) + print(f"Wrote {len(files)} file(s) to {args.output}") + + elif args.command == "honeydew-to-osi": + try: + osi_yaml = convert_honeydew_to_osi(args.input) + except HoneydewConversionError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + with open(args.output, "w") as f: + f.write(osi_yaml) + print(f"Converted {args.input} → {args.output}") + + +if __name__ == "__main__": + main() diff --git a/converters/honeydew/tests/test_honeydew_osi_converter.py b/converters/honeydew/tests/test_honeydew_osi_converter.py new file mode 100644 index 0000000..a4b683d --- /dev/null +++ b/converters/honeydew/tests/test_honeydew_osi_converter.py @@ -0,0 +1,888 @@ +"""Tests for the bidirectional OSI ↔ Honeydew converter.""" + +from __future__ import annotations + +import json +import os +import sys +import warnings +from pathlib import Path + +import pytest +import yaml + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) +from honeydew_osi_converter import ( + HoneydewConversionError, + _assign_metrics_to_entities, + _build_osi_metadata, + _find_entity_in_expression, + _honeydew_datatype_to_osi_dimension, + _is_simple_identifier, + _osi_field_to_honeydew_datatype, + _parse_osi_source, + _pick_ansi_expression, + _read_osi_metadata, + convert_honeydew_to_osi, + convert_osi_to_honeydew, +) + +# ───────────────────────────────────────────────────────────────────────────── +# Helpers +# ───────────────────────────────────────────────────────────────────────────── + +OSI_VERSION = "0.2.0.dev0" + + +def _osi(model_dict): + return yaml.dump( + {"version": OSI_VERSION, "semantic_model": [model_dict]}, + default_flow_style=False, + sort_keys=False, + ) + + +def _minimal_osi_field(name, expr, is_dimension=True, is_time=False): + field = { + "name": name, + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": expr}]}, + } + if is_dimension: + field["dimension"] = {"is_time": is_time} + return field + + +def _minimal_model(): + return { + "name": "test_model", + "datasets": [ + { + "name": "orders", + "source": "db.schema.orders", + "primary_key": ["order_id"], + "fields": [ + _minimal_osi_field("order_id", "order_id"), + _minimal_osi_field("order_date", "order_date", is_time=True), + _minimal_osi_field("total", "total_amount", is_dimension=False), + ], + } + ], + } + + +def _write_workspace(tmp_dir, workspace_name, entities): + """Write a minimal Honeydew workspace to tmp_dir.""" + workspace_path = os.path.join(tmp_dir, "workspace.yml") + with open(workspace_path, "w") as f: + yaml.dump({"type": "workspace", "name": workspace_name}, f) + + for e in entities: + ename = e["name"] + base = os.path.join(tmp_dir, "schema", ename) + os.makedirs(os.path.join(base, "datasets"), exist_ok=True) + os.makedirs(os.path.join(base, "attributes"), exist_ok=True) + os.makedirs(os.path.join(base, "metrics"), exist_ok=True) + + entity_dict = { + "type": "entity", + "name": ename, + "keys": e.get("keys", []), + "key_dataset": e.get("key_dataset", ename), + "relations": e.get("relations", []), + } + with open(os.path.join(base, f"{ename}.yml"), "w") as f: + yaml.dump(entity_dict, f) + + ds_name = e.get("key_dataset", ename) + ds_dict = { + "type": "dataset", + "entity": ename, + "name": ds_name, + "sql": e.get("sql", "DB.SCHEMA." + ename.upper()), + "dataset_type": "table", + "attributes": e.get("dataset_attrs", []), + } + with open(os.path.join(base, "datasets", f"{ds_name}.yml"), "w") as f: + yaml.dump(ds_dict, f) + + for attr in e.get("calc_attrs", []): + with open(os.path.join(base, "attributes", f"{attr['name']}.yml"), "w") as f: + yaml.dump(attr, f) + + for m in e.get("metrics", []): + with open(os.path.join(base, "metrics", f"{m['name']}.yml"), "w") as f: + yaml.dump(m, f) + + +# ───────────────────────────────────────────────────────────────────────────── +# Unit tests – helpers +# ───────────────────────────────────────────────────────────────────────────── + +class TestIsSimpleIdentifier: + def test_plain_name(self): + assert _is_simple_identifier("order_id") is True + + def test_with_spaces(self): + assert _is_simple_identifier("SUM(x)") is False + + def test_with_dot(self): + assert _is_simple_identifier("orders.id") is False + + def test_leading_number(self): + assert _is_simple_identifier("1col") is False + + def test_underscore_prefix(self): + assert _is_simple_identifier("_hidden") is True + + +class TestParseOsiSource: + def test_table_reference(self): + sql, dtype = _parse_osi_source("db.schema.table") + assert sql == "db.schema.table" and dtype == "table" + + def test_select_query(self): + _, dtype = _parse_osi_source("SELECT id FROM foo") + assert dtype == "sql" + + def test_with_query(self): + _, dtype = _parse_osi_source("WITH cte AS (SELECT 1) SELECT * FROM cte") + assert dtype == "sql" + + def test_empty(self): + sql, dtype = _parse_osi_source("") + assert sql == "" and dtype == "table" + + +class TestPickAnsiExpression: + def test_ansi_preferred(self): + expr = {"dialects": [ + {"dialect": "SNOWFLAKE", "expression": "col::VARCHAR"}, + {"dialect": "ANSI_SQL", "expression": "col"}, + ]} + assert _pick_ansi_expression(expr, "f") == "col" + + def test_fallback_to_first(self): + expr = {"dialects": [{"dialect": "SNOWFLAKE", "expression": "col::VARCHAR"}]} + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = _pick_ansi_expression(expr, "f") + assert result == "col::VARCHAR" + assert any("ANSI_SQL" in str(x.message) for x in w) + + def test_none_on_missing(self): + assert _pick_ansi_expression(None, "f") is None + assert _pick_ansi_expression({"dialects": []}, "f") is None + + +class TestOsiFieldDatatypes: + def test_time_dimension(self): + assert _osi_field_to_honeydew_datatype({"dimension": {"is_time": True}}) == "timestamp" + + def test_dimension(self): + assert _osi_field_to_honeydew_datatype({"dimension": {"is_time": False}}) == "string" + + def test_fact(self): + assert _osi_field_to_honeydew_datatype({}) == "number" + + +class TestHoneydewDatatypeToOsiDimension: + def test_date(self): + assert _honeydew_datatype_to_osi_dimension("date") == {"is_time": True} + + def test_timestamp(self): + assert _honeydew_datatype_to_osi_dimension("timestamp") == {"is_time": True} + + def test_string(self): + assert _honeydew_datatype_to_osi_dimension("string") == {"is_time": False} + + def test_bool(self): + assert _honeydew_datatype_to_osi_dimension("bool") == {"is_time": False} + + def test_number(self): + assert _honeydew_datatype_to_osi_dimension("number") is None + + def test_float(self): + assert _honeydew_datatype_to_osi_dimension("float") is None + + +class TestFindEntityInExpression: + def test_finds_entity(self): + assert _find_entity_in_expression("SUM(orders.total)", {"orders", "customers"}) == "orders" + + def test_returns_first_match(self): + result = _find_entity_in_expression("orders.a / customers.b", {"orders", "customers"}) + assert result == "orders" + + def test_no_match(self): + assert _find_entity_in_expression("COUNT(*)", {"orders"}) is None + + def test_ignores_non_entity_prefixes(self): + assert _find_entity_in_expression("SUM(foo.col)", {"orders"}) is None + + +class TestOsiMetadataHelpers: + def test_build_and_read_ai_context_string(self): + section = _build_osi_metadata(ai_context="orders, purchases") + obj = {"metadata": [section]} + result = _read_osi_metadata(obj) + assert result["ai_context"] == "orders, purchases" + + def test_build_and_read_ai_context_dict(self): + ctx = {"instructions": "Use for sales", "synonyms": ["orders", "purchases"]} + section = _build_osi_metadata(ai_context=ctx) + obj = {"metadata": [section]} + result = _read_osi_metadata(obj) + assert result["ai_context"] == ctx + + def test_build_and_read_unique_keys(self): + uks = [["col1", "col2"], ["col3"]] + section = _build_osi_metadata(unique_keys=uks) + obj = {"metadata": [section]} + result = _read_osi_metadata(obj) + assert result["unique_keys"] == uks + + def test_build_and_read_custom_extensions(self): + exts = [{"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "WH"}'}] + section = _build_osi_metadata(custom_extensions=exts) + obj = {"metadata": [section]} + result = _read_osi_metadata(obj) + assert result["custom_extensions"] == exts + + def test_returns_empty_when_no_osi_section(self): + obj = {"metadata": [{"name": "other", "metadata": []}]} + assert _read_osi_metadata(obj) == {} + + def test_returns_empty_when_no_metadata(self): + assert _read_osi_metadata({}) == {} + + def test_build_returns_none_when_nothing_to_store(self): + assert _build_osi_metadata() is None + + +class TestAssignMetricsToEntities: + def test_assigns_by_expression(self): + metrics = [{"name": "total", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}] + result = _assign_metrics_to_entities(metrics, ["orders", "customers"]) + assert "total" in [m["name"] for m in result.get("orders", [])] + + def test_honeydew_hint_takes_priority(self): + metrics = [{ + "name": "cnt", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.x)"}]}, + "custom_extensions": [{"vendor_name": "HONEYDEW", "data": '{"entity": "customers"}'}], + }] + result = _assign_metrics_to_entities(metrics, ["orders", "customers"]) + # hint says customers even though expression references orders + assert "cnt" in [m["name"] for m in result.get("customers", [])] + assert "orders" not in result + + def test_falls_back_to_first_entity(self): + metrics = [{"name": "cnt", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "COUNT(*)"}]}}] + with warnings.catch_warnings(record=True): + result = _assign_metrics_to_entities(metrics, ["orders"]) + assert "cnt" in [m["name"] for m in result.get("orders", [])] + + def test_no_entities(self): + metrics = [{"name": "m", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "COUNT(*)"}]}}] + with warnings.catch_warnings(record=True): + result = _assign_metrics_to_entities(metrics, []) + assert result == {} + + +# ───────────────────────────────────────────────────────────────────────────── +# OSI → Honeydew integration tests +# ───────────────────────────────────────────────────────────────────────────── + +class TestOsiToHoneydew: + def test_workspace_yml_created(self): + files = convert_osi_to_honeydew(_osi(_minimal_model())) + ws = yaml.safe_load(files["workspace.yml"]) + assert ws["name"] == "test_model" and ws["type"] == "workspace" + + def test_entity_yml_created(self): + files = convert_osi_to_honeydew(_osi(_minimal_model())) + entity = yaml.safe_load(files["schema/orders/orders.yml"]) + assert entity["name"] == "orders" + assert entity["keys"] == ["order_id"] + assert entity["key_dataset"] == "orders" + + def test_dataset_yml_created(self): + files = convert_osi_to_honeydew(_osi(_minimal_model())) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + assert ds["sql"] == "db.schema.orders" + assert ds["dataset_type"] == "table" + + def test_simple_fields_become_dataset_attributes(self): + files = convert_osi_to_honeydew(_osi(_minimal_model())) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + names = [a["name"] for a in ds["attributes"]] + assert "order_id" in names and "order_date" in names and "total" in names + + def test_time_field_gets_timestamp_datatype(self): + files = convert_osi_to_honeydew(_osi(_minimal_model())) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + attrs = {a["name"]: a for a in ds["attributes"]} + assert attrs["order_date"]["datatype"] == "timestamp" + + def test_fact_field_gets_number_datatype(self): + files = convert_osi_to_honeydew(_osi(_minimal_model())) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + attrs = {a["name"]: a for a in ds["attributes"]} + assert attrs["total"]["datatype"] == "number" + + def test_complex_expression_becomes_calculated_attribute(self): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "disc_price", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "price * (1 - discount)"}]}, + "dimension": {"is_time": False}, + }]}]} + files = convert_osi_to_honeydew(_osi(model)) + assert "schema/orders/attributes/disc_price.yml" in files + calc = yaml.safe_load(files["schema/orders/attributes/disc_price.yml"]) + assert calc["type"] == "calculated_attribute" + assert calc["sql"] == "price * (1 - discount)" + + def test_label_mapped_to_honeydew_labels(self): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "status", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, + "dimension": {"is_time": False}, + "label": "sales", + }]}]} + files = convert_osi_to_honeydew(_osi(model)) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + attrs = {a["name"]: a for a in ds["attributes"]} + assert "sales" in attrs["status"]["labels"] + + def test_ai_context_string_merged_into_description(self): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "total", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, + "description": "Base desc", + "ai_context": "revenue, earnings", + }]}]} + files = convert_osi_to_honeydew(_osi(model)) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + attrs = {a["name"]: a for a in ds["attributes"]} + assert "revenue, earnings" in attrs["total"]["description"] + + def test_ai_context_dict_instructions_merged_into_description(self): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "total", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, + "ai_context": {"instructions": "Use for revenue", "synonyms": ["rev", "earnings"]}, + }]}]} + files = convert_osi_to_honeydew(_osi(model)) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + attrs = {a["name"]: a for a in ds["attributes"]} + assert "Use for revenue" in attrs["total"]["description"] + assert "rev" in attrs["total"]["labels"] + + def test_ai_context_dict_stored_in_metadata(self): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "total", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, + "ai_context": {"instructions": "Use for revenue", "synonyms": ["rev"]}, + }]}]} + files = convert_osi_to_honeydew(_osi(model)) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + attr = next(a for a in ds["attributes"] if a["name"] == "total") + # Should be in the osi metadata section + osi_section = next((s for s in attr.get("metadata", []) if s["name"] == "osi"), None) + assert osi_section is not None + ai_item = next((i for i in osi_section["metadata"] if i["name"] == "ai_context"), None) + assert ai_item is not None + + def test_unique_keys_stored_in_entity_metadata(self): + model = {"name": "m", "datasets": [{"name": "items", "source": "db.s.items", + "primary_key": ["item_id"], + "unique_keys": [["sku"], ["item_id", "variant"]], + "fields": []}]} + files = convert_osi_to_honeydew(_osi(model)) + entity = yaml.safe_load(files["schema/items/items.yml"]) + osi_section = next((s for s in entity.get("metadata", []) if s["name"] == "osi"), None) + assert osi_section is not None + uk_item = next((i for i in osi_section["metadata"] if i["name"] == "unique_keys"), None) + assert uk_item is not None + assert json.loads(uk_item["value"]) == [["sku"], ["item_id", "variant"]] + + def test_non_honeydew_custom_extensions_stored_in_metadata(self): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "custom_extensions": [{"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "WH"}'}], + "fields": []}]} + files = convert_osi_to_honeydew(_osi(model)) + entity = yaml.safe_load(files["schema/orders/orders.yml"]) + osi_section = next((s for s in entity.get("metadata", []) if s["name"] == "osi"), None) + assert osi_section is not None + ext_item = next((i for i in osi_section["metadata"] if i["name"] == "custom_extensions"), None) + assert ext_item is not None + exts = json.loads(ext_item["value"]) + assert any(e["vendor_name"] == "SNOWFLAKE" for e in exts) + + def test_relationship_name_stored_in_relation(self): + model = {"name": "m", "datasets": [ + {"name": "orders", "source": "db.s.orders", "fields": []}, + {"name": "customers", "source": "db.s.customers", "fields": []}, + ], "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", + "from_columns": ["cid"], "to_columns": ["id"]}]} + files = convert_osi_to_honeydew(_osi(model)) + entity = yaml.safe_load(files["schema/orders/orders.yml"]) + assert entity["relations"][0]["name"] == "orders_to_customers" + + def test_model_ai_context_stored_in_workspace_metadata(self): + model = {"name": "m", "datasets": [], + "ai_context": {"instructions": "Use for retail analytics", "synonyms": ["store"]}} + files = convert_osi_to_honeydew(_osi(model)) + ws = yaml.safe_load(files["workspace.yml"]) + osi_section = next((s for s in ws.get("metadata", []) if s["name"] == "osi"), None) + assert osi_section is not None + + def test_relationship_added_to_entity(self): + model = {"name": "m", "datasets": [ + {"name": "orders", "source": "db.s.orders", "fields": []}, + {"name": "customers", "source": "db.s.customers", "fields": []}, + ], "relationships": [{"name": "r", "from": "orders", "to": "customers", + "from_columns": ["cid"], "to_columns": ["id"]}]} + files = convert_osi_to_honeydew(_osi(model)) + entity = yaml.safe_load(files["schema/orders/orders.yml"]) + assert len(entity["relations"]) == 1 + rel = entity["relations"][0] + assert rel["target_entity"] == "customers" + assert rel["rel_type"] == "many-to-one" + assert rel["connection"] == [{"src_field": "cid", "target_field": "id"}] + + def test_to_entity_has_no_relation(self): + model = {"name": "m", "datasets": [ + {"name": "orders", "source": "db.s.orders", "fields": []}, + {"name": "customers", "source": "db.s.customers", "fields": []}, + ], "relationships": [{"name": "r", "from": "orders", "to": "customers", + "from_columns": ["cid"], "to_columns": ["id"]}]} + files = convert_osi_to_honeydew(_osi(model)) + entity = yaml.safe_load(files["schema/customers/customers.yml"]) + assert entity["relations"] == [] + + def test_metric_assigned_by_expression_entity(self): + model = {"name": "m", + "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], + "metrics": [{"name": "total", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}]} + files = convert_osi_to_honeydew(_osi(model)) + assert "schema/orders/metrics/total.yml" in files + + def test_metric_entity_hint_overrides_expression(self): + model = {"name": "m", + "datasets": [ + {"name": "orders", "source": "db.s.orders", "fields": []}, + {"name": "customers", "source": "db.s.customers", "fields": []}, + ], + "metrics": [{ + "name": "cnt", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.x)"}]}, + "custom_extensions": [{"vendor_name": "HONEYDEW", "data": '{"entity": "customers"}'}], + }]} + files = convert_osi_to_honeydew(_osi(model)) + assert "schema/customers/metrics/cnt.yml" in files + assert "schema/orders/metrics/cnt.yml" not in files + + def test_invalid_version_raises(self): + with pytest.raises(HoneydewConversionError, match="Unsupported"): + convert_osi_to_honeydew("version: '9.9.9'\nsemantic_model:\n - name: m\n") + + def test_missing_semantic_model_raises(self): + with pytest.raises(HoneydewConversionError): + convert_osi_to_honeydew(f"version: '{OSI_VERSION}'\n") + + def test_subquery_source_uses_sql_type(self): + model = {"name": "m", "datasets": [{"name": "orders", + "source": "SELECT * FROM raw.orders WHERE active = true", "fields": []}]} + files = convert_osi_to_honeydew(_osi(model)) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + assert ds["dataset_type"] == "sql" + + def test_composite_primary_key(self): + model = {"name": "m", "datasets": [{"name": "li", "source": "db.s.li", + "primary_key": ["order_id", "line_number"], "fields": []}]} + files = convert_osi_to_honeydew(_osi(model)) + entity = yaml.safe_load(files["schema/li/li.yml"]) + assert entity["keys"] == ["order_id", "line_number"] + + def test_multiple_semantic_models_warns(self): + doc = yaml.dump({"version": OSI_VERSION, "semantic_model": [ + {"name": "m1", "datasets": []}, + {"name": "m2", "datasets": []}, + ]}) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + files = convert_osi_to_honeydew(doc) + assert any("only the first" in str(x.message) for x in w) + assert yaml.safe_load(files["workspace.yml"])["name"] == "m1" + + +# ───────────────────────────────────────────────────────────────────────────── +# Honeydew → OSI integration tests +# ───────────────────────────────────────────────────────────────────────────── + +class TestHoneydewToOsi: + def test_basic_conversion(self, tmp_path): + _write_workspace(str(tmp_path), "tpch", [{ + "name": "orders", "keys": ["orderkey"], "key_dataset": "tpch_orders", + "sql": "SNOWFLAKE_SAMPLE_DATA.TPCH_SF1.ORDERS", + "dataset_attrs": [ + {"column": "o_orderkey", "name": "orderkey", "datatype": "number"}, + {"column": "o_orderdate", "name": "orderdate", "datatype": "date"}, + ], + }]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + sm = result["semantic_model"][0] + assert sm["name"] == "tpch" + ds = sm["datasets"][0] + assert ds["source"] == "SNOWFLAKE_SAMPLE_DATA.TPCH_SF1.ORDERS" + assert ds["primary_key"] == ["orderkey"] + + def test_field_types_from_datatypes(self, tmp_path): + _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], + "key_dataset": "orders", "sql": "db.s.orders", + "dataset_attrs": [ + {"column": "id", "name": "id", "datatype": "number"}, + {"column": "status", "name": "status", "datatype": "string"}, + {"column": "created_at", "name": "created_at", "datatype": "timestamp"}, + ]}]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + fields = {f["name"]: f for f in result["semantic_model"][0]["datasets"][0]["fields"]} + assert fields["id"].get("dimension") is None + assert fields["status"]["dimension"] == {"is_time": False} + assert fields["created_at"]["dimension"] == {"is_time": True} + + def test_labels_become_osi_label_and_ai_context(self, tmp_path): + _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], + "key_dataset": "orders", "sql": "db.s.orders", + "dataset_attrs": [ + {"column": "status", "name": "status", "datatype": "string", + "labels": ["sales", "reporting"]}, + ]}]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + f = next(f for f in result["semantic_model"][0]["datasets"][0]["fields"] if f["name"] == "status") + assert f["label"] == "sales" + assert "sales" in (f.get("ai_context") or {}).get("synonyms", []) + + def test_many_to_one_relation_to_osi(self, tmp_path): + _write_workspace(str(tmp_path), "ws", [ + {"name": "orders", "keys": ["order_id"], "key_dataset": "orders", "sql": "db.s.orders", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "connection": [{"src_field": "customer_id", "target_field": "id"}]}], + "dataset_attrs": []}, + {"name": "customers", "keys": ["id"], "key_dataset": "customers", "sql": "db.s.customers", "dataset_attrs": []}, + ]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + rels = result["semantic_model"][0]["relationships"] + assert len(rels) == 1 + assert rels[0]["from"] == "orders" and rels[0]["to"] == "customers" + + def test_one_to_many_direction_flipped(self, tmp_path): + _write_workspace(str(tmp_path), "ws", [ + {"name": "customers", "keys": ["id"], "key_dataset": "customers", "sql": "db.s.customers", + "relations": [{"target_entity": "orders", "rel_type": "one-to-many", + "connection": [{"src_field": "id", "target_field": "customer_id"}]}], + "dataset_attrs": []}, + {"name": "orders", "keys": ["order_id"], "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": []}, + ]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + rel = result["semantic_model"][0]["relationships"][0] + assert rel["from"] == "orders" and rel["to"] == "customers" + + def test_duplicate_relations_deduplicated(self, tmp_path): + _write_workspace(str(tmp_path), "ws", [ + {"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "db.s.orders", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "connection": [{"src_field": "cid", "target_field": "id"}]}], + "dataset_attrs": []}, + {"name": "customers", "keys": ["id"], "key_dataset": "customers", "sql": "db.s.customers", + "relations": [{"target_entity": "orders", "rel_type": "one-to-many", + "connection": [{"src_field": "id", "target_field": "cid"}]}], + "dataset_attrs": []}, + ]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + assert len(result["semantic_model"][0].get("relationships", [])) == 1 + + def test_metrics_converted(self, tmp_path): + _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], + "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], + "metrics": [{"type": "metric", "entity": "orders", "name": "count", + "datatype": "number", "sql": "COUNT(*)"}]}]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + m = result["semantic_model"][0]["metrics"][0] + assert m["name"] == "count" + assert m["expression"]["dialects"][0]["expression"] == "COUNT(*)" + + def test_metric_entity_preserved_in_custom_extension(self, tmp_path): + _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], + "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], + "metrics": [{"type": "metric", "entity": "orders", "name": "cnt", + "datatype": "number", "sql": "COUNT(*)"}]}]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + m = result["semantic_model"][0]["metrics"][0] + ext = m["custom_extensions"][0] + assert ext["vendor_name"] == "HONEYDEW" + assert json.loads(ext["data"])["entity"] == "orders" + + def test_calculated_attribute_as_field(self, tmp_path): + _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], + "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], + "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", + "name": "discounted", "datatype": "number", + "sql": "orders.price * (1 - orders.discount)"}]}]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + fields = {f["name"]: f for f in result["semantic_model"][0]["datasets"][0]["fields"]} + assert "discounted" in fields + assert "orders.price" in fields["discounted"]["expression"]["dialects"][0]["expression"] + + def test_missing_workspace_yml_raises(self, tmp_path): + with pytest.raises(HoneydewConversionError, match="workspace.yml"): + convert_honeydew_to_osi(str(tmp_path)) + + def test_missing_schema_dir_produces_empty_model(self, tmp_path): + (tmp_path / "workspace.yml").write_text(yaml.dump({"type": "workspace", "name": "ws"})) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + assert result["semantic_model"][0]["datasets"] == [] + + def test_vendors_includes_honeydew(self, tmp_path): + (tmp_path / "workspace.yml").write_text(yaml.dump({"type": "workspace", "name": "ws"})) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + assert "HONEYDEW" in result.get("vendors", []) + + def test_empty_metrics_skipped(self, tmp_path): + _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], + "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], + "metrics": [{"type": "metric", "entity": "orders", "name": "bad", + "datatype": "number", "sql": ""}]}]) + with warnings.catch_warnings(record=True): + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + assert "metrics" not in result["semantic_model"][0] + + +# ───────────────────────────────────────────────────────────────────────────── +# Round-trip tests (idempotency) +# ───────────────────────────────────────────────────────────────────────────── + +class TestOsiToHoneydewToOsiRoundTrip: + """OSI → Honeydew → OSI: verify key fields survive both legs.""" + + def _roundtrip(self, model_dict, tmp_path): + files = convert_osi_to_honeydew(_osi(model_dict)) + for rel_path, content in files.items(): + p = tmp_path / rel_path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + return yaml.safe_load(convert_honeydew_to_osi(str(tmp_path)))["semantic_model"][0] + + def test_name_and_description_preserved(self, tmp_path): + model = {"name": "retail", "description": "Retail model", "datasets": []} + sm = self._roundtrip(model, tmp_path) + assert sm["name"] == "retail" + assert sm["description"] == "Retail model" + + def test_primary_key_preserved(self, tmp_path): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "primary_key": ["order_id"], "fields": []}]} + sm = self._roundtrip(model, tmp_path) + assert sm["datasets"][0]["primary_key"] == ["order_id"] + + def test_composite_primary_key_preserved(self, tmp_path): + model = {"name": "m", "datasets": [{"name": "li", "source": "db.s.li", + "primary_key": ["order_id", "line_no"], "fields": []}]} + sm = self._roundtrip(model, tmp_path) + assert sm["datasets"][0]["primary_key"] == ["order_id", "line_no"] + + def test_unique_keys_preserved(self, tmp_path): + model = {"name": "m", "datasets": [{"name": "items", "source": "db.s.items", + "primary_key": ["id"], + "unique_keys": [["sku"], ["id", "variant"]], + "fields": []}]} + sm = self._roundtrip(model, tmp_path) + assert sm["datasets"][0]["unique_keys"] == [["sku"], ["id", "variant"]] + + def test_field_label_preserved(self, tmp_path): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "fields": [{"name": "status", "label": "sales", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, + "dimension": {"is_time": False}}]}]} + sm = self._roundtrip(model, tmp_path) + f = next(f for f in sm["datasets"][0]["fields"] if f["name"] == "status") + assert f["label"] == "sales" + + def test_ai_context_string_preserved(self, tmp_path): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "fields": [{"name": "status", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, + "ai_context": "order status, order state", + "dimension": {"is_time": False}}]}]} + sm = self._roundtrip(model, tmp_path) + f = next(f for f in sm["datasets"][0]["fields"] if f["name"] == "status") + # String ai_context is merged into description; exact form may vary + assert f.get("description") or f.get("ai_context") + + def test_ai_context_dict_preserved(self, tmp_path): + ctx = {"instructions": "Use for revenue analysis", "synonyms": ["revenue", "sales"]} + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "fields": [{"name": "total", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, + "ai_context": ctx}]}]} + sm = self._roundtrip(model, tmp_path) + f = next(f for f in sm["datasets"][0]["fields"] if f["name"] == "total") + assert f.get("ai_context") == ctx + + def test_model_ai_context_preserved(self, tmp_path): + ctx = {"instructions": "Retail analytics", "synonyms": ["store"]} + model = {"name": "m", "ai_context": ctx, "datasets": []} + sm = self._roundtrip(model, tmp_path) + assert sm.get("ai_context") == ctx + + def test_non_honeydew_custom_extensions_preserved(self, tmp_path): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "custom_extensions": [{"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "WH"}'}], + "fields": []}]} + sm = self._roundtrip(model, tmp_path) + exts = sm["datasets"][0].get("custom_extensions") or [] + assert any(e["vendor_name"] == "SNOWFLAKE" for e in exts) + + def test_relationship_name_preserved(self, tmp_path): + model = {"name": "m", "datasets": [ + {"name": "orders", "source": "db.s.orders", "fields": []}, + {"name": "customers", "source": "db.s.customers", "fields": []}, + ], "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", + "from_columns": ["cid"], "to_columns": ["id"]}]} + sm = self._roundtrip(model, tmp_path) + assert sm["relationships"][0]["name"] == "orders_to_customers" + + def test_relationship_columns_preserved(self, tmp_path): + model = {"name": "m", "datasets": [ + {"name": "orders", "source": "db.s.orders", "fields": []}, + {"name": "customers", "source": "db.s.customers", "fields": []}, + ], "relationships": [{"name": "r", "from": "orders", "to": "customers", + "from_columns": ["cid"], "to_columns": ["id"]}]} + sm = self._roundtrip(model, tmp_path) + rel = sm["relationships"][0] + assert rel["from_columns"] == ["cid"] and rel["to_columns"] == ["id"] + + def test_metric_name_and_expression_preserved(self, tmp_path): + model = {"name": "m", + "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], + "metrics": [{"name": "total_revenue", "description": "Sum of sales", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}]} + sm = self._roundtrip(model, tmp_path) + m = sm["metrics"][0] + assert m["name"] == "total_revenue" + assert m["expression"]["dialects"][0]["expression"] == "SUM(orders.total)" + assert m["description"] == "Sum of sales" + + def test_tpcds_example_roundtrip(self, tmp_path): + tpcds_path = ( + Path(__file__).resolve().parent.parent.parent.parent + / "examples" / "tpcds_semantic_model.yaml" + ) + if not tpcds_path.exists(): + pytest.skip("TPC-DS example not found") + osi_yaml = tpcds_path.read_text() + files = convert_osi_to_honeydew(osi_yaml) + for rel_path, content in files.items(): + p = tmp_path / rel_path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + sm = result["semantic_model"][0] + assert sm["name"] == "tpcds_retail_model" + ds_names = {ds["name"] for ds in sm["datasets"]} + assert "store_sales" in ds_names and "customer" in ds_names + + +class TestHoneydewToOsiToHoneydewRoundTrip: + """Honeydew → OSI → Honeydew: verify Honeydew-specific fields survive.""" + + def _roundtrip(self, entities, tmp_path): + _write_workspace(str(tmp_path), "ws", entities) + osi_yaml = convert_honeydew_to_osi(str(tmp_path)) + files = convert_osi_to_honeydew(osi_yaml) + # Write to a second directory + out_dir = tmp_path / "out" + for rel_path, content in files.items(): + p = out_dir / rel_path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + return out_dir + + def test_entity_name_and_keys_preserved(self, tmp_path): + out_dir = self._roundtrip([{ + "name": "orders", "keys": ["order_id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", "dataset_attrs": [], + }], tmp_path) + entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) + assert entity["name"] == "orders" + assert entity["keys"] == ["order_id"] + + def test_source_preserved(self, tmp_path): + out_dir = self._roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.SCHEMA.ORDERS", "dataset_attrs": [], + }], tmp_path) + ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) + assert ds["sql"] == "DB.SCHEMA.ORDERS" + + def test_column_attributes_preserved(self, tmp_path): + out_dir = self._roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", + "dataset_attrs": [ + {"column": "o_id", "name": "id", "datatype": "number"}, + {"column": "o_status", "name": "status", "datatype": "string"}, + ], + }], tmp_path) + ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) + attrs = {a["name"]: a for a in ds["attributes"]} + assert attrs["id"]["column"] == "o_id" + assert attrs["status"]["datatype"] == "string" + + def test_labels_preserved_on_column(self, tmp_path): + out_dir = self._roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", + "dataset_attrs": [ + {"column": "status", "name": "status", "datatype": "string", "labels": ["sales"]}, + ], + }], tmp_path) + ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) + attrs = {a["name"]: a for a in ds["attributes"]} + assert "sales" in attrs["status"].get("labels", []) + + def test_calculated_attribute_sql_preserved(self, tmp_path): + out_dir = self._roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", "dataset_attrs": [], + "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", + "name": "disc", "datatype": "number", + "sql": "orders.price * (1 - orders.discount)"}], + }], tmp_path) + calc = yaml.safe_load((out_dir / "schema/orders/attributes/disc.yml").read_text()) + assert calc["sql"] == "orders.price * (1 - orders.discount)" + + def test_metric_entity_assignment_preserved(self, tmp_path): + out_dir = self._roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", "dataset_attrs": [], + "metrics": [{"type": "metric", "entity": "orders", "name": "cnt", + "datatype": "number", "sql": "COUNT(*)"}], + }], tmp_path) + m = yaml.safe_load((out_dir / "schema/orders/metrics/cnt.yml").read_text()) + assert m["entity"] == "orders" + assert m["sql"] == "COUNT(*)" + + def test_relation_preserved(self, tmp_path): + out_dir = self._roundtrip([ + {"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "connection": [{"src_field": "cid", "target_field": "id"}]}], + "dataset_attrs": []}, + {"name": "customers", "keys": ["id"], "key_dataset": "customers", + "sql": "DB.S.CUSTOMERS", "dataset_attrs": []}, + ], tmp_path) + entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) + assert entity["relations"][0]["target_entity"] == "customers" + assert entity["relations"][0]["connection"][0]["src_field"] == "cid" From ee2e0e93d784222f5f16b78e58e4750500f38476 Mon Sep 17 00:00:00 2001 From: Baruch Oxman Date: Wed, 27 May 2026 15:42:36 +0300 Subject: [PATCH 17/89] Add .gitignore for honeydew converter Co-Authored-By: Claude Sonnet 4.6 --- converters/honeydew/.gitignore | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 converters/honeydew/.gitignore diff --git a/converters/honeydew/.gitignore b/converters/honeydew/.gitignore new file mode 100644 index 0000000..b5fed1f --- /dev/null +++ b/converters/honeydew/.gitignore @@ -0,0 +1,8 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +*.egg-info/ +dist/ +build/ +.venv/ +venv/ From f5f4a717013cdf571654fd049f03d44ccfa0909a Mon Sep 17 00:00:00 2001 From: Baruch Oxman Date: Wed, 27 May 2026 16:12:32 +0300 Subject: [PATCH 18/89] Fix round-trip bugs and update README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix empty-string expression bypassing None guard - Restore calculated_attribute routing via HONEYDEW type hint - Preserve bool datatype through OSI round-trip - Store string metric ai_context in osi metadata for recovery - Warn on duplicate metric names instead of silently overwriting - Restore connection_expr from HONEYDEW custom_extension on OSI→Honeydew - Warn on malformed JSON in _read_osi_metadata instead of silent drop - Remove filter limitation from README (not applicable) - Update Honeydew docs link to honeydew.ai/docs Co-Authored-By: Claude Sonnet 4.6 --- converters/honeydew/README.md | 7 +- .../honeydew/src/honeydew_osi_converter.py | 29 ++++- .../tests/test_honeydew_osi_converter.py | 107 ++++++++++++++++++ 3 files changed, 134 insertions(+), 9 deletions(-) diff --git a/converters/honeydew/README.md b/converters/honeydew/README.md index eb0e54b..8f8a25e 100644 --- a/converters/honeydew/README.md +++ b/converters/honeydew/README.md @@ -1,6 +1,6 @@ # OSI ↔ Honeydew Converter -Bidirectional converter between [OSI](../../core-spec/spec.md) semantic models and [Honeydew](https://docs.honeydew.ai) workspace YAML. +Bidirectional converter between [OSI](../../core-spec/spec.md) semantic models and [Honeydew](https://honeydew.ai/docs) workspace YAML. ## Overview @@ -62,7 +62,6 @@ python -m pytest tests/ - **One dataset per entity**: The converter maps each OSI dataset to a single Honeydew entity with one source dataset. Multiple datasets per entity are not generated. - **Datatype inference**: OSI fields have no explicit datatype; the converter infers Honeydew datatypes from the `dimension.is_time` flag (`timestamp`) and the presence/absence of the `dimension` key (`string` vs `number`). - **Honeydew SQL expressions**: Calculated attributes and metrics use Honeydew's `entity.attribute` reference syntax. These are exported as `ANSI_SQL` dialect expressions in OSI; they remain valid for round-tripping but may not run on other databases without adaptation. -- **Filters**: Honeydew `filter` objects have no OSI equivalent and are not exported. - **Perspectives and domains**: Not converted (no OSI equivalent). -- **Connection expressions** (`connection_expr`): Preserved in `HONEYDEW` custom extensions on the OSI relationship. -- **`ai_context`**: OSI `ai_context` fields (synonyms, instructions) are dropped during OSI → Honeydew conversion (no native Honeydew equivalent). Honeydew `description` fields are mapped to OSI `description`. +- **Connection expressions** (`connection_expr`): Preserved in `HONEYDEW` custom extensions on the OSI relationship and restored on the return trip. +- **`ai_context`**: OSI `ai_context` fields (synonyms, instructions) are stored in Honeydew `metadata` for round-trip recovery. Instructions are also merged into `description` for human readability. diff --git a/converters/honeydew/src/honeydew_osi_converter.py b/converters/honeydew/src/honeydew_osi_converter.py index 0fb9e0a..30c8dcd 100644 --- a/converters/honeydew/src/honeydew_osi_converter.py +++ b/converters/honeydew/src/honeydew_osi_converter.py @@ -192,7 +192,7 @@ def _dataset_to_files( raise HoneydewConversionError(f"Field missing 'name' in dataset '{entity_name}'") expr = _pick_ansi_expression(field.get("expression"), field_name) - if expr is None: + if not expr: continue datatype = _osi_field_to_honeydew_datatype(field) @@ -223,7 +223,10 @@ def _dataset_to_files( custom_extensions=field_ext or None, ) - if _is_simple_identifier(expr): + hd_hint = _get_honeydew_extension(field) + force_calc = hd_hint.get("type") == "calculated_attribute" + + if _is_simple_identifier(expr) and not force_calc: attr: dict[str, Any] = {"column": expr, "name": field_name, "datatype": datatype} if effective_desc: attr["description"] = effective_desc @@ -293,13 +296,18 @@ def _dataset_to_files( metric_ext = [e for e in (metric.get("custom_extensions") or []) if e.get("vendor_name") != HONEYDEW_VENDOR] metric_meta = _build_osi_metadata( - ai_context=metric_ai_ctx if isinstance(metric_ai_ctx, dict) else None, + ai_context=metric_ai_ctx, custom_extensions=metric_ext or None, ) if metric_meta: metric_dict["metadata"] = [metric_meta] - files[f"{base}/metrics/{mname}.yml"] = _dump(metric_dict) + metric_path = f"{base}/metrics/{mname}.yml" + if metric_path in files: + warnings.warn( + f"Metric '{mname}' in entity '{entity_name}' is defined more than once; later definition wins" + ) + files[metric_path] = _dump(metric_dict) return files @@ -331,6 +339,10 @@ def _osi_relation_to_honeydew(rel: dict[str, Any]) -> dict[str, Any] | None: {"src_field": fc, "target_field": tc} for fc, tc in zip(from_cols, to_cols) ] + elif not from_cols: + hd_ext = _get_honeydew_extension(rel) + if hd_ext.get("connection_expr"): + honeydew_rel["connection_expr"] = {"sql": hd_ext["connection_expr"]} return honeydew_rel @@ -364,6 +376,9 @@ def _pick_ansi_expression(expression: Any, field_name: str) -> str | None: def _osi_field_to_honeydew_datatype(field: dict[str, Any]) -> str: + hd_ext = _get_honeydew_extension(field) + if hd_ext.get("datatype"): + return hd_ext["datatype"] dimension = field.get("dimension") if isinstance(dimension, dict) and dimension.get("is_time"): return "timestamp" @@ -660,6 +675,8 @@ def _entity_to_osi_dataset(entity_data: dict[str, Any]) -> dict[str, Any]: k: attr[k] for k in ("display_name", "hidden", "folder", "format_string", "timegrain") if k in attr } + if datatype == "bool": + attr_honeydew_extra["datatype"] = datatype if len(attr_labels) > 1: attr_honeydew_extra["labels"] = attr_labels @@ -707,6 +724,8 @@ def _entity_to_osi_dataset(entity_data: dict[str, Any]) -> dict[str, Any]: k: calc[k] for k in ("display_name", "hidden", "folder", "format_string", "timegrain") if k in calc } + if datatype == "bool": + calc_honeydew_extra["datatype"] = datatype all_calc_ext = list(calc_osi_meta.get("custom_extensions") or []) # Always mark as calculated_attribute so OSI → Honeydew routes it correctly @@ -860,7 +879,7 @@ def _read_osi_metadata(obj: dict[str, Any]) -> dict[str, Any]: try: result[key] = json.loads(raw) except (json.JSONDecodeError, TypeError): - pass + warnings.warn(f"Could not parse OSI metadata field '{key}': {raw!r}") return result return {} diff --git a/converters/honeydew/tests/test_honeydew_osi_converter.py b/converters/honeydew/tests/test_honeydew_osi_converter.py index a4b683d..81a449d 100644 --- a/converters/honeydew/tests/test_honeydew_osi_converter.py +++ b/converters/honeydew/tests/test_honeydew_osi_converter.py @@ -886,3 +886,110 @@ def test_relation_preserved(self, tmp_path): entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) assert entity["relations"][0]["target_entity"] == "customers" assert entity["relations"][0]["connection"][0]["src_field"] == "cid" + + def test_bool_datatype_preserved(self, tmp_path): + out_dir = self._roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", + "dataset_attrs": [ + {"column": "is_active", "name": "is_active", "datatype": "bool"}, + ], + }], tmp_path) + ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) + attrs = {a["name"]: a for a in ds["attributes"]} + assert attrs["is_active"]["datatype"] == "bool" + + def test_connection_expr_preserved(self, tmp_path): + out_dir = self._roundtrip([ + {"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "connection_expr": {"sql": "orders.cid = customers.id AND orders.region = customers.region"}}], + "dataset_attrs": []}, + {"name": "customers", "keys": ["id"], "key_dataset": "customers", + "sql": "DB.S.CUSTOMERS", "dataset_attrs": []}, + ], tmp_path) + entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) + rel = entity["relations"][0] + assert rel.get("connection_expr", {}).get("sql") == "orders.cid = customers.id AND orders.region = customers.region" + + def test_calc_attr_with_simple_identifier_sql_preserved(self, tmp_path): + out_dir = self._roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", "dataset_attrs": [], + "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", + "name": "revenue", "datatype": "number", "sql": "revenue"}], + }], tmp_path) + # sql='revenue' is a simple identifier — must still come back as calculated_attribute + calc_path = out_dir / "schema/orders/attributes/revenue.yml" + assert calc_path.exists(), "calculated_attribute with simple-id sql should not become a dataset column" + calc = yaml.safe_load(calc_path.read_text()) + assert calc["sql"] == "revenue" + + +# ───────────────────────────────────────────────────────────────────────────── +# Bug-fix regression tests +# ───────────────────────────────────────────────────────────────────────────── + +class TestBugFixes: + def test_empty_string_expression_skipped(self): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "bad", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": ""}]}, + "dimension": {"is_time": False}, + }]}]} + files = convert_osi_to_honeydew(_osi(model)) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + names = [a["name"] for a in ds["attributes"]] + assert "bad" not in names + assert "schema/orders/attributes/bad.yml" not in files + + def test_duplicate_metric_name_warns(self): + model = {"name": "m", + "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], + "metrics": [ + {"name": "total", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.a)"}]}}, + {"name": "total", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.b)"}]}}, + ]} + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + files = convert_osi_to_honeydew(_osi(model)) + assert any("total" in str(x.message) for x in w) + # Last definition wins + m = yaml.safe_load(files["schema/orders/metrics/total.yml"]) + assert "orders.b" in m["sql"] + + def test_metric_string_ai_context_preserved_in_roundtrip(self, tmp_path): + model = {"name": "m", + "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], + "metrics": [{"name": "rev", "ai_context": "Use for revenue analysis", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}]} + files = convert_osi_to_honeydew(_osi(model)) + for rel_path, content in files.items(): + p = tmp_path / rel_path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + m = result["semantic_model"][0]["metrics"][0] + assert m.get("ai_context") == "Use for revenue analysis" + + def test_malformed_osi_metadata_json_warns(self, tmp_path): + ws_path = tmp_path / "workspace.yml" + ws_path.write_text(yaml.dump({"type": "workspace", "name": "ws"})) + base = tmp_path / "schema" / "orders" + (base / "datasets").mkdir(parents=True) + entity = { + "type": "entity", "name": "orders", "keys": ["id"], "key_dataset": "orders", + "relations": [], + "metadata": [{"name": "osi", "metadata": [ + {"name": "unique_keys", "value": "[broken json"}, + ]}], + } + (base / "orders.yml").write_text(yaml.dump(entity)) + (base / "datasets" / "orders.yml").write_text(yaml.dump( + {"type": "dataset", "entity": "orders", "name": "orders", + "sql": "DB.S.ORDERS", "dataset_type": "table", "attributes": []} + )) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + convert_honeydew_to_osi(str(tmp_path)) + assert any("unique_keys" in str(x.message) for x in w) From 7cc9ea3a11ff9d07f1da7c386291dd0d12b05d65 Mon Sep 17 00:00:00 2001 From: Baruch Oxman Date: Wed, 27 May 2026 16:29:58 +0300 Subject: [PATCH 19/89] Fix code review findings: round-trip data loss, path traversal, expression guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Restore Honeydew-specific fields (display_name, hidden, format_string, timegrain, owner, folder) in OSI→Honeydew direction by reading them back from the HONEYDEW custom_extension on both entity and attribute objects - Add path traversal guard in main() using os.path.normpath + startswith check - Guard against whitespace-only expressions (not expr or not expr.strip()) - Warn when field expression is a non-dict value instead of silently dropping it - Simplify elif not from_cols: to else: in _osi_relation_to_honeydew - Strengthen test_ai_context_string_preserved to assert the string value is recoverable in description - Add 5 new tests: display_name/format round-trip, calc attr Honeydew fields, entity owner, whitespace expression skipped, non-dict expression warns Co-Authored-By: Claude Sonnet 4.6 --- .../honeydew/src/honeydew_osi_converter.py | 31 ++++++- .../tests/test_honeydew_osi_converter.py | 86 ++++++++++++++++++- 2 files changed, 110 insertions(+), 7 deletions(-) diff --git a/converters/honeydew/src/honeydew_osi_converter.py b/converters/honeydew/src/honeydew_osi_converter.py index 30c8dcd..25d9ab0 100644 --- a/converters/honeydew/src/honeydew_osi_converter.py +++ b/converters/honeydew/src/honeydew_osi_converter.py @@ -164,6 +164,14 @@ def _dataset_to_files( entity_dict["keys"] = list(primary_key) entity_dict["key_dataset"] = entity_name + # Restore Honeydew-specific entity fields from HONEYDEW custom_extension + entity_hd_hint = _get_honeydew_extension(ds) + for key in ("owner", "display_name", "hidden", "folder"): + if key in entity_hd_hint: + entity_dict[key] = entity_hd_hint[key] + if "labels" in entity_hd_hint: + entity_dict["labels"] = entity_hd_hint["labels"] + honeydew_relations = [] for rel in relations: hr = _osi_relation_to_honeydew(rel) @@ -192,7 +200,7 @@ def _dataset_to_files( raise HoneydewConversionError(f"Field missing 'name' in dataset '{entity_name}'") expr = _pick_ansi_expression(field.get("expression"), field_name) - if not expr: + if not expr or not expr.strip(): continue datatype = _osi_field_to_honeydew_datatype(field) @@ -226,6 +234,8 @@ def _dataset_to_files( hd_hint = _get_honeydew_extension(field) force_calc = hd_hint.get("type") == "calculated_attribute" + _hd_attr_keys = ("display_name", "hidden", "folder", "format_string", "timegrain") + if _is_simple_identifier(expr) and not force_calc: attr: dict[str, Any] = {"column": expr, "name": field_name, "datatype": datatype} if effective_desc: @@ -234,6 +244,9 @@ def _dataset_to_files( attr["labels"] = labels if field_meta: attr["metadata"] = [field_meta] + for _k in _hd_attr_keys: + if _k in hd_hint: + attr[_k] = hd_hint[_k] dataset_attrs.append(attr) else: calc: dict[str, Any] = { @@ -249,6 +262,9 @@ def _dataset_to_files( calc["labels"] = labels if field_meta: calc["metadata"] = [field_meta] + for _k in _hd_attr_keys: + if _k in hd_hint: + calc[_k] = hd_hint[_k] calc_attrs.append(calc) # ── dataset YAML ─────────────────────────────────────────────────────────── @@ -276,7 +292,7 @@ def _dataset_to_files( if not mname: continue mexpr = _pick_ansi_expression(metric.get("expression"), mname) - if mexpr is None: + if not mexpr or not mexpr.strip(): continue metric_dict: dict[str, Any] = { @@ -339,7 +355,7 @@ def _osi_relation_to_honeydew(rel: dict[str, Any]) -> dict[str, Any] | None: {"src_field": fc, "target_field": tc} for fc, tc in zip(from_cols, to_cols) ] - elif not from_cols: + else: hd_ext = _get_honeydew_extension(rel) if hd_ext.get("connection_expr"): honeydew_rel["connection_expr"] = {"sql": hd_ext["connection_expr"]} @@ -348,7 +364,10 @@ def _osi_relation_to_honeydew(rel: dict[str, Any]) -> dict[str, Any] | None: def _pick_ansi_expression(expression: Any, field_name: str) -> str | None: """Select the ANSI_SQL expression; fall back to first available dialect.""" + if expression is None: + return None if not isinstance(expression, dict): + warnings.warn(f"'{field_name}': 'expression' must be a mapping; field will be skipped") return None dialects = expression.get("dialects") or [] if not dialects: @@ -942,8 +961,12 @@ def main() -> None: print(f"Error: {e}", file=sys.stderr) sys.exit(1) + output_abs = os.path.abspath(args.output) for rel_path, content in files.items(): - full_path = os.path.join(args.output, rel_path) + full_path = os.path.normpath(os.path.join(output_abs, rel_path)) + if not full_path.startswith(output_abs + os.sep): + print(f"Error: refusing to write outside output directory: {rel_path}", file=sys.stderr) + sys.exit(1) os.makedirs(os.path.dirname(full_path), exist_ok=True) with open(full_path, "w") as f: f.write(content) diff --git a/converters/honeydew/tests/test_honeydew_osi_converter.py b/converters/honeydew/tests/test_honeydew_osi_converter.py index 81a449d..9115e7c 100644 --- a/converters/honeydew/tests/test_honeydew_osi_converter.py +++ b/converters/honeydew/tests/test_honeydew_osi_converter.py @@ -710,15 +710,16 @@ def test_field_label_preserved(self, tmp_path): assert f["label"] == "sales" def test_ai_context_string_preserved(self, tmp_path): + ai_ctx_value = "order status, order state" model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{"name": "status", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, - "ai_context": "order status, order state", + "ai_context": ai_ctx_value, "dimension": {"is_time": False}}]}]} sm = self._roundtrip(model, tmp_path) f = next(f for f in sm["datasets"][0]["fields"] if f["name"] == "status") - # String ai_context is merged into description; exact form may vary - assert f.get("description") or f.get("ai_context") + # String ai_context is merged into description on OSI→Honeydew; value must be recoverable + assert ai_ctx_value in (f.get("description") or "") or f.get("ai_context") == ai_ctx_value def test_ai_context_dict_preserved(self, tmp_path): ctx = {"instructions": "Use for revenue analysis", "synonyms": ["revenue", "sales"]} @@ -912,6 +913,60 @@ def test_connection_expr_preserved(self, tmp_path): rel = entity["relations"][0] assert rel.get("connection_expr", {}).get("sql") == "orders.cid = customers.id AND orders.region = customers.region" + def test_dataset_attr_display_name_and_format_preserved(self, tmp_path): + out_dir = self._roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", + "dataset_attrs": [ + {"column": "status", "name": "status", "datatype": "string", + "display_name": "Order Status", "hidden": True, "format_string": "##,###"}, + ], + }], tmp_path) + ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) + attrs = {a["name"]: a for a in ds["attributes"]} + assert attrs["status"]["display_name"] == "Order Status" + assert attrs["status"]["hidden"] is True + assert attrs["status"]["format_string"] == "##,###" + + def test_calc_attr_honeydew_fields_preserved(self, tmp_path): + out_dir = self._roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", "dataset_attrs": [], + "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", + "name": "disc", "datatype": "number", + "sql": "orders.price * 0.9", + "display_name": "Discounted Price", + "timegrain": "day"}], + }], tmp_path) + calc = yaml.safe_load((out_dir / "schema/orders/attributes/disc.yml").read_text()) + assert calc["display_name"] == "Discounted Price" + assert calc["timegrain"] == "day" + + def test_entity_owner_and_display_name_preserved(self, tmp_path): + ws_path = tmp_path / "workspace.yml" + ws_path.write_text(yaml.dump({"type": "workspace", "name": "ws"})) + base = tmp_path / "schema" / "orders" + (base / "datasets").mkdir(parents=True) + (base / "orders.yml").write_text(yaml.dump({ + "type": "entity", "name": "orders", "keys": ["id"], + "key_dataset": "orders", "relations": [], + "owner": "analytics_team", "display_name": "Orders Table", + })) + (base / "datasets" / "orders.yml").write_text(yaml.dump({ + "type": "dataset", "entity": "orders", "name": "orders", + "sql": "DB.S.ORDERS", "dataset_type": "table", "attributes": [], + })) + osi_yaml = convert_honeydew_to_osi(str(tmp_path)) + files = convert_osi_to_honeydew(osi_yaml) + out_dir = tmp_path / "out" + for rel_path, content in files.items(): + p = out_dir / rel_path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) + assert entity.get("owner") == "analytics_team" + assert entity.get("display_name") == "Orders Table" + def test_calc_attr_with_simple_identifier_sql_preserved(self, tmp_path): out_dir = self._roundtrip([{ "name": "orders", "keys": ["id"], "key_dataset": "orders", @@ -972,6 +1027,31 @@ def test_metric_string_ai_context_preserved_in_roundtrip(self, tmp_path): m = result["semantic_model"][0]["metrics"][0] assert m.get("ai_context") == "Use for revenue analysis" + def test_whitespace_expression_skipped(self): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "bad", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": " "}]}, + "dimension": {"is_time": False}, + }]}]} + files = convert_osi_to_honeydew(_osi(model)) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + names = [a["name"] for a in ds["attributes"]] + assert "bad" not in names + assert "schema/orders/attributes/bad.yml" not in files + + def test_non_dict_expression_warns(self): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "bad", + "expression": "just_a_string", + "dimension": {"is_time": False}, + }]}]} + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + files = convert_osi_to_honeydew(_osi(model)) + assert any("must be a mapping" in str(x.message) for x in w) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + assert all(a["name"] != "bad" for a in ds["attributes"]) + def test_malformed_osi_metadata_json_warns(self, tmp_path): ws_path = tmp_path / "workspace.yml" ws_path.write_text(yaml.dump({"type": "workspace", "name": "ws"})) From b5e565cd45cf3a53846276864ab578184f475719 Mon Sep 17 00:00:00 2001 From: Baruch Oxman Date: Wed, 27 May 2026 17:11:30 +0300 Subject: [PATCH 20/89] Refactor tests to pytest functions + parametrize; require Python 3.12 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rewrite entire test file from classes to module-level pytest functions; parametrize repeated cases (is_simple_identifier, parse_osi_source, field datatypes, entity honeydew fields, dataset/calc attr fields, empty/whitespace expressions, path traversal guard, check_safe_path) - Extract _check_safe_path into a named helper in the converter (was inlined in main()) so path traversal logic is independently testable - Add parametrized test_check_safe_path covering ../evil.yml and ../../etc/passwd rejection and legitimate nested paths - Add test_empty_or_whitespace_metric_expression_skipped to cover the OSI→Honeydew whitespace guard on metrics (was previously untested) - Parametrize entity/attr/calc Honeydew field round-trip tests (owner, display_name, hidden, folder each verified independently) - Update _write_workspace helper to pass through entity-level fields (owner, display_name, hidden, folder) so round-trip tests use the standard _honeydew_roundtrip() helper instead of manual workspace setup - Promote _HD_ATTR_KEYS tuple to module-level constant (was defined inside the field loop on every iteration) - Drop from __future__ import annotations — requires Python 3.12+ - Add pyproject.toml with requires-python = ">=3.12" - Add Python 3.12+ requirement section to README Co-Authored-By: Claude Sonnet 4.6 --- converters/honeydew/README.md | 5 + converters/honeydew/pyproject.toml | 25 + .../honeydew/src/honeydew_osi_converter.py | 27 +- .../tests/test_honeydew_osi_converter.py | 1813 +++++++++-------- 4 files changed, 968 insertions(+), 902 deletions(-) create mode 100644 converters/honeydew/pyproject.toml diff --git a/converters/honeydew/README.md b/converters/honeydew/README.md index 8f8a25e..103d19a 100644 --- a/converters/honeydew/README.md +++ b/converters/honeydew/README.md @@ -35,6 +35,11 @@ Bidirectional converter between [OSI](../../core-spec/spec.md) semantic models a | `entity.relations` (`one-to-many`) | `relationship` with `from` = target entity | | `metric.sql` | `metric` expression in `ANSI_SQL` dialect | +## Requirements + +- Python 3.12+ +- PyYAML 6.0+ + ## Setup ```bash diff --git a/converters/honeydew/pyproject.toml b/converters/honeydew/pyproject.toml new file mode 100644 index 0000000..82df220 --- /dev/null +++ b/converters/honeydew/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "honeydew-osi" +version = "0.2.0.dev0" +description = "Bidirectional converter between Honeydew workspace YAML and OSI semantic model" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "pyyaml>=6.0", +] + +[dependency-groups] +dev = [ + "pytest>=8.0", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build] +include = ["src/**/*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/converters/honeydew/src/honeydew_osi_converter.py b/converters/honeydew/src/honeydew_osi_converter.py index 25d9ab0..e082360 100644 --- a/converters/honeydew/src/honeydew_osi_converter.py +++ b/converters/honeydew/src/honeydew_osi_converter.py @@ -11,8 +11,6 @@ python honeydew_osi_converter.py honeydew-to-osi -i workspace_dir/ -o output.yaml """ -from __future__ import annotations - import argparse import json import os @@ -26,6 +24,7 @@ SUPPORTED_OSI_VERSION = "0.2.0.dev0" HONEYDEW_VENDOR = "HONEYDEW" _OSI_METADATA_SECTION = "osi" +_HD_ATTR_KEYS = ("display_name", "hidden", "folder", "format_string", "timegrain") class HoneydewConversionError(Exception): @@ -234,8 +233,6 @@ def _dataset_to_files( hd_hint = _get_honeydew_extension(field) force_calc = hd_hint.get("type") == "calculated_attribute" - _hd_attr_keys = ("display_name", "hidden", "folder", "format_string", "timegrain") - if _is_simple_identifier(expr) and not force_calc: attr: dict[str, Any] = {"column": expr, "name": field_name, "datatype": datatype} if effective_desc: @@ -244,9 +241,9 @@ def _dataset_to_files( attr["labels"] = labels if field_meta: attr["metadata"] = [field_meta] - for _k in _hd_attr_keys: - if _k in hd_hint: - attr[_k] = hd_hint[_k] + for k in _HD_ATTR_KEYS: + if k in hd_hint: + attr[k] = hd_hint[k] dataset_attrs.append(attr) else: calc: dict[str, Any] = { @@ -262,9 +259,9 @@ def _dataset_to_files( calc["labels"] = labels if field_meta: calc["metadata"] = [field_meta] - for _k in _hd_attr_keys: - if _k in hd_hint: - calc[_k] = hd_hint[_k] + for k in _HD_ATTR_KEYS: + if k in hd_hint: + calc[k] = hd_hint[k] calc_attrs.append(calc) # ── dataset YAML ─────────────────────────────────────────────────────────── @@ -931,6 +928,12 @@ def _dump(data: Any) -> str: return yaml.dump(data, default_flow_style=False, sort_keys=False, allow_unicode=True) +def _check_safe_path(output_abs: str, rel_path: str) -> bool: + """Return True iff the resolved path stays inside output_abs.""" + full = os.path.normpath(os.path.join(output_abs, rel_path)) + return full.startswith(output_abs + os.sep) + + # ───────────────────────────────────────────────────────────────────────────── # CLI # ───────────────────────────────────────────────────────────────────────────── @@ -963,10 +966,10 @@ def main() -> None: output_abs = os.path.abspath(args.output) for rel_path, content in files.items(): - full_path = os.path.normpath(os.path.join(output_abs, rel_path)) - if not full_path.startswith(output_abs + os.sep): + if not _check_safe_path(output_abs, rel_path): print(f"Error: refusing to write outside output directory: {rel_path}", file=sys.stderr) sys.exit(1) + full_path = os.path.normpath(os.path.join(output_abs, rel_path)) os.makedirs(os.path.dirname(full_path), exist_ok=True) with open(full_path, "w") as f: f.write(content) diff --git a/converters/honeydew/tests/test_honeydew_osi_converter.py b/converters/honeydew/tests/test_honeydew_osi_converter.py index 9115e7c..810661e 100644 --- a/converters/honeydew/tests/test_honeydew_osi_converter.py +++ b/converters/honeydew/tests/test_honeydew_osi_converter.py @@ -1,7 +1,5 @@ """Tests for the bidirectional OSI ↔ Honeydew converter.""" -from __future__ import annotations - import json import os import sys @@ -16,6 +14,7 @@ HoneydewConversionError, _assign_metrics_to_entities, _build_osi_metadata, + _check_safe_path, _find_entity_in_expression, _honeydew_datatype_to_osi_dimension, _is_simple_identifier, @@ -90,6 +89,9 @@ def _write_workspace(tmp_dir, workspace_name, entities): "key_dataset": e.get("key_dataset", ename), "relations": e.get("relations", []), } + for k in ("owner", "display_name", "hidden", "folder"): + if k in e: + entity_dict[k] = e[k] with open(os.path.join(base, f"{ename}.yml"), "w") as f: yaml.dump(entity_dict, f) @@ -114,962 +116,993 @@ def _write_workspace(tmp_dir, workspace_name, entities): yaml.dump(m, f) -# ───────────────────────────────────────────────────────────────────────────── -# Unit tests – helpers -# ───────────────────────────────────────────────────────────────────────────── - -class TestIsSimpleIdentifier: - def test_plain_name(self): - assert _is_simple_identifier("order_id") is True - - def test_with_spaces(self): - assert _is_simple_identifier("SUM(x)") is False - - def test_with_dot(self): - assert _is_simple_identifier("orders.id") is False - - def test_leading_number(self): - assert _is_simple_identifier("1col") is False - - def test_underscore_prefix(self): - assert _is_simple_identifier("_hidden") is True - - -class TestParseOsiSource: - def test_table_reference(self): - sql, dtype = _parse_osi_source("db.schema.table") - assert sql == "db.schema.table" and dtype == "table" +def _osi_roundtrip(model_dict, tmp_path): + """OSI → Honeydew → OSI; returns the semantic model dict.""" + files = convert_osi_to_honeydew(_osi(model_dict)) + for rel_path, content in files.items(): + p = tmp_path / rel_path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + return yaml.safe_load(convert_honeydew_to_osi(str(tmp_path)))["semantic_model"][0] - def test_select_query(self): - _, dtype = _parse_osi_source("SELECT id FROM foo") - assert dtype == "sql" - def test_with_query(self): - _, dtype = _parse_osi_source("WITH cte AS (SELECT 1) SELECT * FROM cte") - assert dtype == "sql" +def _honeydew_roundtrip(entities, tmp_path): + """Honeydew → OSI → Honeydew; returns Path to the output workspace directory.""" + _write_workspace(str(tmp_path), "ws", entities) + osi_yaml = convert_honeydew_to_osi(str(tmp_path)) + files = convert_osi_to_honeydew(osi_yaml) + out_dir = tmp_path / "out" + for rel_path, content in files.items(): + p = out_dir / rel_path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + return out_dir - def test_empty(self): - sql, dtype = _parse_osi_source("") - assert sql == "" and dtype == "table" - - -class TestPickAnsiExpression: - def test_ansi_preferred(self): - expr = {"dialects": [ - {"dialect": "SNOWFLAKE", "expression": "col::VARCHAR"}, - {"dialect": "ANSI_SQL", "expression": "col"}, - ]} - assert _pick_ansi_expression(expr, "f") == "col" - - def test_fallback_to_first(self): - expr = {"dialects": [{"dialect": "SNOWFLAKE", "expression": "col::VARCHAR"}]} - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - result = _pick_ansi_expression(expr, "f") - assert result == "col::VARCHAR" - assert any("ANSI_SQL" in str(x.message) for x in w) - - def test_none_on_missing(self): - assert _pick_ansi_expression(None, "f") is None - assert _pick_ansi_expression({"dialects": []}, "f") is None +# ───────────────────────────────────────────────────────────────────────────── +# Unit tests – helpers +# ───────────────────────────────────────────────────────────────────────────── -class TestOsiFieldDatatypes: - def test_time_dimension(self): - assert _osi_field_to_honeydew_datatype({"dimension": {"is_time": True}}) == "timestamp" +@pytest.mark.parametrize("expr,expected", [ + ("order_id", True), + ("SUM(x)", False), + ("orders.id", False), + ("1col", False), + ("_hidden", True), +]) +def test_is_simple_identifier(expr, expected): + assert _is_simple_identifier(expr) is expected + + +@pytest.mark.parametrize("source,expected_sql,expected_type", [ + ("db.schema.table", "db.schema.table", "table"), + ("SELECT id FROM foo", "SELECT id FROM foo", "sql"), + ("WITH cte AS (SELECT 1) SELECT * FROM cte", "WITH cte AS (SELECT 1) SELECT * FROM cte", "sql"), + ("", "", "table"), +]) +def test_parse_osi_source(source, expected_sql, expected_type): + sql, dtype = _parse_osi_source(source) + assert sql == expected_sql and dtype == expected_type + + +@pytest.mark.parametrize("field,expected_dt", [ + ({"dimension": {"is_time": True}}, "timestamp"), + ({"dimension": {"is_time": False}}, "string"), + ({}, "number"), +]) +def test_osi_field_to_honeydew_datatype(field, expected_dt): + assert _osi_field_to_honeydew_datatype(field) == expected_dt + + +@pytest.mark.parametrize("datatype,expected_dim", [ + ("date", {"is_time": True}), + ("timestamp", {"is_time": True}), + ("string", {"is_time": False}), + ("bool", {"is_time": False}), + ("number", None), + ("float", None), +]) +def test_honeydew_datatype_to_osi_dimension(datatype, expected_dim): + assert _honeydew_datatype_to_osi_dimension(datatype) == expected_dim + + +@pytest.mark.parametrize("expr,entities,expected", [ + ("SUM(orders.total)", {"orders", "customers"}, "orders"), + ("orders.a / customers.b", {"orders", "customers"}, "orders"), + ("COUNT(*)", {"orders"}, None), + ("SUM(foo.col)", {"orders"}, None), +]) +def test_find_entity_in_expression(expr, entities, expected): + assert _find_entity_in_expression(expr, entities) == expected + + +def test_pick_ansi_expression_ansi_preferred(): + expr = {"dialects": [ + {"dialect": "SNOWFLAKE", "expression": "col::VARCHAR"}, + {"dialect": "ANSI_SQL", "expression": "col"}, + ]} + assert _pick_ansi_expression(expr, "f") == "col" + + +def test_pick_ansi_expression_fallback_warns(): + expr = {"dialects": [{"dialect": "SNOWFLAKE", "expression": "col::VARCHAR"}]} + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = _pick_ansi_expression(expr, "f") + assert result == "col::VARCHAR" + assert any("ANSI_SQL" in str(x.message) for x in w) + + +@pytest.mark.parametrize("expression", [None, {"dialects": []}]) +def test_pick_ansi_expression_returns_none(expression): + assert _pick_ansi_expression(expression, "f") is None + + +def test_pick_ansi_expression_non_dict_warns(): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = _pick_ansi_expression("just_a_string", "f") + assert result is None + assert any("must be a mapping" in str(x.message) for x in w) - def test_dimension(self): - assert _osi_field_to_honeydew_datatype({"dimension": {"is_time": False}}) == "string" - def test_fact(self): - assert _osi_field_to_honeydew_datatype({}) == "number" +# ───────────────────────────────────────────────────────────────────────────── +# OSI metadata helpers +# ───────────────────────────────────────────────────────────────────────────── +def test_build_and_read_ai_context_string(): + section = _build_osi_metadata(ai_context="orders, purchases") + result = _read_osi_metadata({"metadata": [section]}) + assert result["ai_context"] == "orders, purchases" -class TestHoneydewDatatypeToOsiDimension: - def test_date(self): - assert _honeydew_datatype_to_osi_dimension("date") == {"is_time": True} - def test_timestamp(self): - assert _honeydew_datatype_to_osi_dimension("timestamp") == {"is_time": True} +def test_build_and_read_ai_context_dict(): + ctx = {"instructions": "Use for sales", "synonyms": ["orders", "purchases"]} + section = _build_osi_metadata(ai_context=ctx) + result = _read_osi_metadata({"metadata": [section]}) + assert result["ai_context"] == ctx - def test_string(self): - assert _honeydew_datatype_to_osi_dimension("string") == {"is_time": False} - def test_bool(self): - assert _honeydew_datatype_to_osi_dimension("bool") == {"is_time": False} +def test_build_and_read_unique_keys(): + uks = [["col1", "col2"], ["col3"]] + section = _build_osi_metadata(unique_keys=uks) + result = _read_osi_metadata({"metadata": [section]}) + assert result["unique_keys"] == uks - def test_number(self): - assert _honeydew_datatype_to_osi_dimension("number") is None - def test_float(self): - assert _honeydew_datatype_to_osi_dimension("float") is None +def test_build_and_read_custom_extensions(): + exts = [{"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "WH"}'}] + section = _build_osi_metadata(custom_extensions=exts) + result = _read_osi_metadata({"metadata": [section]}) + assert result["custom_extensions"] == exts -class TestFindEntityInExpression: - def test_finds_entity(self): - assert _find_entity_in_expression("SUM(orders.total)", {"orders", "customers"}) == "orders" +def test_read_osi_metadata_no_osi_section(): + assert _read_osi_metadata({"metadata": [{"name": "other", "metadata": []}]}) == {} - def test_returns_first_match(self): - result = _find_entity_in_expression("orders.a / customers.b", {"orders", "customers"}) - assert result == "orders" - def test_no_match(self): - assert _find_entity_in_expression("COUNT(*)", {"orders"}) is None +def test_read_osi_metadata_no_metadata(): + assert _read_osi_metadata({}) == {} - def test_ignores_non_entity_prefixes(self): - assert _find_entity_in_expression("SUM(foo.col)", {"orders"}) is None +def test_build_osi_metadata_nothing_to_store(): + assert _build_osi_metadata() is None -class TestOsiMetadataHelpers: - def test_build_and_read_ai_context_string(self): - section = _build_osi_metadata(ai_context="orders, purchases") - obj = {"metadata": [section]} - result = _read_osi_metadata(obj) - assert result["ai_context"] == "orders, purchases" - def test_build_and_read_ai_context_dict(self): - ctx = {"instructions": "Use for sales", "synonyms": ["orders", "purchases"]} - section = _build_osi_metadata(ai_context=ctx) - obj = {"metadata": [section]} - result = _read_osi_metadata(obj) - assert result["ai_context"] == ctx +# ───────────────────────────────────────────────────────────────────────────── +# Assign metrics to entities +# ───────────────────────────────────────────────────────────────────────────── - def test_build_and_read_unique_keys(self): - uks = [["col1", "col2"], ["col3"]] - section = _build_osi_metadata(unique_keys=uks) - obj = {"metadata": [section]} - result = _read_osi_metadata(obj) - assert result["unique_keys"] == uks +def test_assign_metrics_by_expression(): + metrics = [{"name": "total", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}] + result = _assign_metrics_to_entities(metrics, ["orders", "customers"]) + assert "total" in [m["name"] for m in result.get("orders", [])] - def test_build_and_read_custom_extensions(self): - exts = [{"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "WH"}'}] - section = _build_osi_metadata(custom_extensions=exts) - obj = {"metadata": [section]} - result = _read_osi_metadata(obj) - assert result["custom_extensions"] == exts - def test_returns_empty_when_no_osi_section(self): - obj = {"metadata": [{"name": "other", "metadata": []}]} - assert _read_osi_metadata(obj) == {} +def test_assign_metrics_honeydew_hint_takes_priority(): + metrics = [{ + "name": "cnt", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.x)"}]}, + "custom_extensions": [{"vendor_name": "HONEYDEW", "data": '{"entity": "customers"}'}], + }] + result = _assign_metrics_to_entities(metrics, ["orders", "customers"]) + assert "cnt" in [m["name"] for m in result.get("customers", [])] + assert "orders" not in result - def test_returns_empty_when_no_metadata(self): - assert _read_osi_metadata({}) == {} - def test_build_returns_none_when_nothing_to_store(self): - assert _build_osi_metadata() is None +def test_assign_metrics_fallback_to_first_entity(): + metrics = [{"name": "cnt", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "COUNT(*)"}]}}] + with warnings.catch_warnings(record=True): + result = _assign_metrics_to_entities(metrics, ["orders"]) + assert "cnt" in [m["name"] for m in result.get("orders", [])] -class TestAssignMetricsToEntities: - def test_assigns_by_expression(self): - metrics = [{"name": "total", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}] - result = _assign_metrics_to_entities(metrics, ["orders", "customers"]) - assert "total" in [m["name"] for m in result.get("orders", [])] +def test_assign_metrics_no_entities(): + metrics = [{"name": "m", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "COUNT(*)"}]}}] + with warnings.catch_warnings(record=True): + result = _assign_metrics_to_entities(metrics, []) + assert result == {} - def test_honeydew_hint_takes_priority(self): - metrics = [{ - "name": "cnt", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.x)"}]}, - "custom_extensions": [{"vendor_name": "HONEYDEW", "data": '{"entity": "customers"}'}], - }] - result = _assign_metrics_to_entities(metrics, ["orders", "customers"]) - # hint says customers even though expression references orders - assert "cnt" in [m["name"] for m in result.get("customers", [])] - assert "orders" not in result - def test_falls_back_to_first_entity(self): - metrics = [{"name": "cnt", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "COUNT(*)"}]}}] - with warnings.catch_warnings(record=True): - result = _assign_metrics_to_entities(metrics, ["orders"]) - assert "cnt" in [m["name"] for m in result.get("orders", [])] +# ───────────────────────────────────────────────────────────────────────────── +# Path traversal guard +# ───────────────────────────────────────────────────────────────────────────── - def test_no_entities(self): - metrics = [{"name": "m", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "COUNT(*)"}]}}] - with warnings.catch_warnings(record=True): - result = _assign_metrics_to_entities(metrics, []) - assert result == {} +@pytest.mark.parametrize("rel_path,expected", [ + ("workspace.yml", True), + ("schema/orders/orders.yml", True), + ("schema/orders/datasets/orders.yml", True), + ("../evil.yml", False), + ("../../etc/passwd", False), + ("schema/../../../evil", False), +]) +def test_check_safe_path(rel_path, expected): + output_abs = os.path.abspath("/tmp/test_output") + assert _check_safe_path(output_abs, rel_path) is expected # ───────────────────────────────────────────────────────────────────────────── # OSI → Honeydew integration tests # ───────────────────────────────────────────────────────────────────────────── -class TestOsiToHoneydew: - def test_workspace_yml_created(self): - files = convert_osi_to_honeydew(_osi(_minimal_model())) - ws = yaml.safe_load(files["workspace.yml"]) - assert ws["name"] == "test_model" and ws["type"] == "workspace" - - def test_entity_yml_created(self): - files = convert_osi_to_honeydew(_osi(_minimal_model())) - entity = yaml.safe_load(files["schema/orders/orders.yml"]) - assert entity["name"] == "orders" - assert entity["keys"] == ["order_id"] - assert entity["key_dataset"] == "orders" - - def test_dataset_yml_created(self): - files = convert_osi_to_honeydew(_osi(_minimal_model())) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - assert ds["sql"] == "db.schema.orders" - assert ds["dataset_type"] == "table" - - def test_simple_fields_become_dataset_attributes(self): - files = convert_osi_to_honeydew(_osi(_minimal_model())) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - names = [a["name"] for a in ds["attributes"]] - assert "order_id" in names and "order_date" in names and "total" in names - - def test_time_field_gets_timestamp_datatype(self): - files = convert_osi_to_honeydew(_osi(_minimal_model())) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - attrs = {a["name"]: a for a in ds["attributes"]} - assert attrs["order_date"]["datatype"] == "timestamp" - - def test_fact_field_gets_number_datatype(self): - files = convert_osi_to_honeydew(_osi(_minimal_model())) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - attrs = {a["name"]: a for a in ds["attributes"]} - assert attrs["total"]["datatype"] == "number" - - def test_complex_expression_becomes_calculated_attribute(self): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ - "name": "disc_price", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "price * (1 - discount)"}]}, - "dimension": {"is_time": False}, - }]}]} - files = convert_osi_to_honeydew(_osi(model)) - assert "schema/orders/attributes/disc_price.yml" in files - calc = yaml.safe_load(files["schema/orders/attributes/disc_price.yml"]) - assert calc["type"] == "calculated_attribute" - assert calc["sql"] == "price * (1 - discount)" - - def test_label_mapped_to_honeydew_labels(self): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ - "name": "status", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, - "dimension": {"is_time": False}, - "label": "sales", - }]}]} - files = convert_osi_to_honeydew(_osi(model)) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - attrs = {a["name"]: a for a in ds["attributes"]} - assert "sales" in attrs["status"]["labels"] - - def test_ai_context_string_merged_into_description(self): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ - "name": "total", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, - "description": "Base desc", - "ai_context": "revenue, earnings", - }]}]} - files = convert_osi_to_honeydew(_osi(model)) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - attrs = {a["name"]: a for a in ds["attributes"]} - assert "revenue, earnings" in attrs["total"]["description"] - - def test_ai_context_dict_instructions_merged_into_description(self): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ - "name": "total", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, - "ai_context": {"instructions": "Use for revenue", "synonyms": ["rev", "earnings"]}, - }]}]} - files = convert_osi_to_honeydew(_osi(model)) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - attrs = {a["name"]: a for a in ds["attributes"]} - assert "Use for revenue" in attrs["total"]["description"] - assert "rev" in attrs["total"]["labels"] - - def test_ai_context_dict_stored_in_metadata(self): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ - "name": "total", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, - "ai_context": {"instructions": "Use for revenue", "synonyms": ["rev"]}, - }]}]} - files = convert_osi_to_honeydew(_osi(model)) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - attr = next(a for a in ds["attributes"] if a["name"] == "total") - # Should be in the osi metadata section - osi_section = next((s for s in attr.get("metadata", []) if s["name"] == "osi"), None) - assert osi_section is not None - ai_item = next((i for i in osi_section["metadata"] if i["name"] == "ai_context"), None) - assert ai_item is not None - - def test_unique_keys_stored_in_entity_metadata(self): - model = {"name": "m", "datasets": [{"name": "items", "source": "db.s.items", - "primary_key": ["item_id"], - "unique_keys": [["sku"], ["item_id", "variant"]], - "fields": []}]} - files = convert_osi_to_honeydew(_osi(model)) - entity = yaml.safe_load(files["schema/items/items.yml"]) - osi_section = next((s for s in entity.get("metadata", []) if s["name"] == "osi"), None) - assert osi_section is not None - uk_item = next((i for i in osi_section["metadata"] if i["name"] == "unique_keys"), None) - assert uk_item is not None - assert json.loads(uk_item["value"]) == [["sku"], ["item_id", "variant"]] - - def test_non_honeydew_custom_extensions_stored_in_metadata(self): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", - "custom_extensions": [{"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "WH"}'}], - "fields": []}]} - files = convert_osi_to_honeydew(_osi(model)) - entity = yaml.safe_load(files["schema/orders/orders.yml"]) - osi_section = next((s for s in entity.get("metadata", []) if s["name"] == "osi"), None) - assert osi_section is not None - ext_item = next((i for i in osi_section["metadata"] if i["name"] == "custom_extensions"), None) - assert ext_item is not None - exts = json.loads(ext_item["value"]) - assert any(e["vendor_name"] == "SNOWFLAKE" for e in exts) - - def test_relationship_name_stored_in_relation(self): - model = {"name": "m", "datasets": [ +def test_osi_to_honeydew_workspace_yml(): + files = convert_osi_to_honeydew(_osi(_minimal_model())) + ws = yaml.safe_load(files["workspace.yml"]) + assert ws["name"] == "test_model" and ws["type"] == "workspace" + + +def test_osi_to_honeydew_entity_yml(): + files = convert_osi_to_honeydew(_osi(_minimal_model())) + entity = yaml.safe_load(files["schema/orders/orders.yml"]) + assert entity["name"] == "orders" + assert entity["keys"] == ["order_id"] + assert entity["key_dataset"] == "orders" + + +def test_osi_to_honeydew_dataset_yml(): + files = convert_osi_to_honeydew(_osi(_minimal_model())) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + assert ds["sql"] == "db.schema.orders" + assert ds["dataset_type"] == "table" + + +def test_osi_to_honeydew_simple_fields_become_dataset_attributes(): + files = convert_osi_to_honeydew(_osi(_minimal_model())) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + names = [a["name"] for a in ds["attributes"]] + assert "order_id" in names and "order_date" in names and "total" in names + + +@pytest.mark.parametrize("field_name,expected_dt", [ + ("order_date", "timestamp"), + ("total", "number"), +]) +def test_osi_to_honeydew_field_datatypes(field_name, expected_dt): + files = convert_osi_to_honeydew(_osi(_minimal_model())) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + attrs = {a["name"]: a for a in ds["attributes"]} + assert attrs[field_name]["datatype"] == expected_dt + + +def test_osi_to_honeydew_complex_expression_becomes_calculated_attribute(): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "disc_price", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "price * (1 - discount)"}]}, + "dimension": {"is_time": False}, + }]}]} + files = convert_osi_to_honeydew(_osi(model)) + assert "schema/orders/attributes/disc_price.yml" in files + calc = yaml.safe_load(files["schema/orders/attributes/disc_price.yml"]) + assert calc["type"] == "calculated_attribute" + assert calc["sql"] == "price * (1 - discount)" + + +def test_osi_to_honeydew_label_mapped_to_honeydew_labels(): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "status", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, + "dimension": {"is_time": False}, + "label": "sales", + }]}]} + files = convert_osi_to_honeydew(_osi(model)) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + attrs = {a["name"]: a for a in ds["attributes"]} + assert "sales" in attrs["status"]["labels"] + + +def test_osi_to_honeydew_ai_context_string_merged_into_description(): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "total", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, + "description": "Base desc", + "ai_context": "revenue, earnings", + }]}]} + files = convert_osi_to_honeydew(_osi(model)) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + attrs = {a["name"]: a for a in ds["attributes"]} + assert "revenue, earnings" in attrs["total"]["description"] + + +def test_osi_to_honeydew_ai_context_dict_instructions_merged_into_description(): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "total", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, + "ai_context": {"instructions": "Use for revenue", "synonyms": ["rev", "earnings"]}, + }]}]} + files = convert_osi_to_honeydew(_osi(model)) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + attrs = {a["name"]: a for a in ds["attributes"]} + assert "Use for revenue" in attrs["total"]["description"] + assert "rev" in attrs["total"]["labels"] + + +def test_osi_to_honeydew_ai_context_dict_stored_in_metadata(): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "total", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, + "ai_context": {"instructions": "Use for revenue", "synonyms": ["rev"]}, + }]}]} + files = convert_osi_to_honeydew(_osi(model)) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + attr = next(a for a in ds["attributes"] if a["name"] == "total") + osi_section = next((s for s in attr.get("metadata", []) if s["name"] == "osi"), None) + assert osi_section is not None + assert any(i["name"] == "ai_context" for i in osi_section["metadata"]) + + +def test_osi_to_honeydew_unique_keys_stored_in_entity_metadata(): + model = {"name": "m", "datasets": [{"name": "items", "source": "db.s.items", + "primary_key": ["item_id"], + "unique_keys": [["sku"], ["item_id", "variant"]], + "fields": []}]} + files = convert_osi_to_honeydew(_osi(model)) + entity = yaml.safe_load(files["schema/items/items.yml"]) + osi_section = next((s for s in entity.get("metadata", []) if s["name"] == "osi"), None) + assert osi_section is not None + uk_item = next((i for i in osi_section["metadata"] if i["name"] == "unique_keys"), None) + assert uk_item is not None + assert json.loads(uk_item["value"]) == [["sku"], ["item_id", "variant"]] + + +def test_osi_to_honeydew_non_honeydew_extensions_stored_in_metadata(): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "custom_extensions": [{"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "WH"}'}], + "fields": []}]} + files = convert_osi_to_honeydew(_osi(model)) + entity = yaml.safe_load(files["schema/orders/orders.yml"]) + osi_section = next((s for s in entity.get("metadata", []) if s["name"] == "osi"), None) + assert osi_section is not None + ext_item = next((i for i in osi_section["metadata"] if i["name"] == "custom_extensions"), None) + assert ext_item is not None + exts = json.loads(ext_item["value"]) + assert any(e["vendor_name"] == "SNOWFLAKE" for e in exts) + + +def test_osi_to_honeydew_relationship_name_stored_in_relation(): + model = {"name": "m", "datasets": [ + {"name": "orders", "source": "db.s.orders", "fields": []}, + {"name": "customers", "source": "db.s.customers", "fields": []}, + ], "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", + "from_columns": ["cid"], "to_columns": ["id"]}]} + files = convert_osi_to_honeydew(_osi(model)) + entity = yaml.safe_load(files["schema/orders/orders.yml"]) + assert entity["relations"][0]["name"] == "orders_to_customers" + + +def test_osi_to_honeydew_model_ai_context_stored_in_workspace_metadata(): + model = {"name": "m", "datasets": [], + "ai_context": {"instructions": "Use for retail analytics", "synonyms": ["store"]}} + files = convert_osi_to_honeydew(_osi(model)) + ws = yaml.safe_load(files["workspace.yml"]) + assert any(s["name"] == "osi" for s in ws.get("metadata", [])) + + +def test_osi_to_honeydew_relationship_on_from_entity_only(): + model = {"name": "m", "datasets": [ + {"name": "orders", "source": "db.s.orders", "fields": []}, + {"name": "customers", "source": "db.s.customers", "fields": []}, + ], "relationships": [{"name": "r", "from": "orders", "to": "customers", + "from_columns": ["cid"], "to_columns": ["id"]}]} + files = convert_osi_to_honeydew(_osi(model)) + orders = yaml.safe_load(files["schema/orders/orders.yml"]) + customers = yaml.safe_load(files["schema/customers/customers.yml"]) + assert len(orders["relations"]) == 1 + assert customers["relations"] == [] + rel = orders["relations"][0] + assert rel["target_entity"] == "customers" and rel["rel_type"] == "many-to-one" + assert rel["connection"] == [{"src_field": "cid", "target_field": "id"}] + + +def test_osi_to_honeydew_metric_assigned_by_expression_entity(): + model = {"name": "m", + "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], + "metrics": [{"name": "total", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}]} + files = convert_osi_to_honeydew(_osi(model)) + assert "schema/orders/metrics/total.yml" in files + + +def test_osi_to_honeydew_metric_entity_hint_overrides_expression(): + model = {"name": "m", + "datasets": [ {"name": "orders", "source": "db.s.orders", "fields": []}, {"name": "customers", "source": "db.s.customers", "fields": []}, - ], "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", - "from_columns": ["cid"], "to_columns": ["id"]}]} - files = convert_osi_to_honeydew(_osi(model)) - entity = yaml.safe_load(files["schema/orders/orders.yml"]) - assert entity["relations"][0]["name"] == "orders_to_customers" + ], + "metrics": [{ + "name": "cnt", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.x)"}]}, + "custom_extensions": [{"vendor_name": "HONEYDEW", "data": '{"entity": "customers"}'}], + }]} + files = convert_osi_to_honeydew(_osi(model)) + assert "schema/customers/metrics/cnt.yml" in files + assert "schema/orders/metrics/cnt.yml" not in files - def test_model_ai_context_stored_in_workspace_metadata(self): - model = {"name": "m", "datasets": [], - "ai_context": {"instructions": "Use for retail analytics", "synonyms": ["store"]}} - files = convert_osi_to_honeydew(_osi(model)) - ws = yaml.safe_load(files["workspace.yml"]) - osi_section = next((s for s in ws.get("metadata", []) if s["name"] == "osi"), None) - assert osi_section is not None - def test_relationship_added_to_entity(self): - model = {"name": "m", "datasets": [ - {"name": "orders", "source": "db.s.orders", "fields": []}, - {"name": "customers", "source": "db.s.customers", "fields": []}, - ], "relationships": [{"name": "r", "from": "orders", "to": "customers", - "from_columns": ["cid"], "to_columns": ["id"]}]} - files = convert_osi_to_honeydew(_osi(model)) - entity = yaml.safe_load(files["schema/orders/orders.yml"]) - assert len(entity["relations"]) == 1 - rel = entity["relations"][0] - assert rel["target_entity"] == "customers" - assert rel["rel_type"] == "many-to-one" - assert rel["connection"] == [{"src_field": "cid", "target_field": "id"}] - - def test_to_entity_has_no_relation(self): - model = {"name": "m", "datasets": [ - {"name": "orders", "source": "db.s.orders", "fields": []}, - {"name": "customers", "source": "db.s.customers", "fields": []}, - ], "relationships": [{"name": "r", "from": "orders", "to": "customers", - "from_columns": ["cid"], "to_columns": ["id"]}]} - files = convert_osi_to_honeydew(_osi(model)) - entity = yaml.safe_load(files["schema/customers/customers.yml"]) - assert entity["relations"] == [] +def test_osi_to_honeydew_invalid_version_raises(): + with pytest.raises(HoneydewConversionError, match="Unsupported"): + convert_osi_to_honeydew("version: '9.9.9'\nsemantic_model:\n - name: m\n") - def test_metric_assigned_by_expression_entity(self): - model = {"name": "m", - "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], - "metrics": [{"name": "total", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}]} - files = convert_osi_to_honeydew(_osi(model)) - assert "schema/orders/metrics/total.yml" in files - - def test_metric_entity_hint_overrides_expression(self): - model = {"name": "m", - "datasets": [ - {"name": "orders", "source": "db.s.orders", "fields": []}, - {"name": "customers", "source": "db.s.customers", "fields": []}, - ], - "metrics": [{ - "name": "cnt", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.x)"}]}, - "custom_extensions": [{"vendor_name": "HONEYDEW", "data": '{"entity": "customers"}'}], - }]} - files = convert_osi_to_honeydew(_osi(model)) - assert "schema/customers/metrics/cnt.yml" in files - assert "schema/orders/metrics/cnt.yml" not in files - def test_invalid_version_raises(self): - with pytest.raises(HoneydewConversionError, match="Unsupported"): - convert_osi_to_honeydew("version: '9.9.9'\nsemantic_model:\n - name: m\n") +def test_osi_to_honeydew_missing_semantic_model_raises(): + with pytest.raises(HoneydewConversionError): + convert_osi_to_honeydew(f"version: '{OSI_VERSION}'\n") - def test_missing_semantic_model_raises(self): - with pytest.raises(HoneydewConversionError): - convert_osi_to_honeydew(f"version: '{OSI_VERSION}'\n") - def test_subquery_source_uses_sql_type(self): - model = {"name": "m", "datasets": [{"name": "orders", - "source": "SELECT * FROM raw.orders WHERE active = true", "fields": []}]} - files = convert_osi_to_honeydew(_osi(model)) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - assert ds["dataset_type"] == "sql" +def test_osi_to_honeydew_subquery_source_uses_sql_type(): + model = {"name": "m", "datasets": [{"name": "orders", + "source": "SELECT * FROM raw.orders WHERE active = true", "fields": []}]} + files = convert_osi_to_honeydew(_osi(model)) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + assert ds["dataset_type"] == "sql" - def test_composite_primary_key(self): - model = {"name": "m", "datasets": [{"name": "li", "source": "db.s.li", - "primary_key": ["order_id", "line_number"], "fields": []}]} - files = convert_osi_to_honeydew(_osi(model)) - entity = yaml.safe_load(files["schema/li/li.yml"]) - assert entity["keys"] == ["order_id", "line_number"] - def test_multiple_semantic_models_warns(self): - doc = yaml.dump({"version": OSI_VERSION, "semantic_model": [ - {"name": "m1", "datasets": []}, - {"name": "m2", "datasets": []}, - ]}) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - files = convert_osi_to_honeydew(doc) - assert any("only the first" in str(x.message) for x in w) - assert yaml.safe_load(files["workspace.yml"])["name"] == "m1" +def test_osi_to_honeydew_composite_primary_key(): + model = {"name": "m", "datasets": [{"name": "li", "source": "db.s.li", + "primary_key": ["order_id", "line_number"], "fields": []}]} + files = convert_osi_to_honeydew(_osi(model)) + entity = yaml.safe_load(files["schema/li/li.yml"]) + assert entity["keys"] == ["order_id", "line_number"] + + +def test_osi_to_honeydew_multiple_models_warns(): + doc = yaml.dump({"version": OSI_VERSION, "semantic_model": [ + {"name": "m1", "datasets": []}, + {"name": "m2", "datasets": []}, + ]}) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + files = convert_osi_to_honeydew(doc) + assert any("only the first" in str(x.message) for x in w) + assert yaml.safe_load(files["workspace.yml"])["name"] == "m1" # ───────────────────────────────────────────────────────────────────────────── # Honeydew → OSI integration tests # ───────────────────────────────────────────────────────────────────────────── -class TestHoneydewToOsi: - def test_basic_conversion(self, tmp_path): - _write_workspace(str(tmp_path), "tpch", [{ - "name": "orders", "keys": ["orderkey"], "key_dataset": "tpch_orders", - "sql": "SNOWFLAKE_SAMPLE_DATA.TPCH_SF1.ORDERS", - "dataset_attrs": [ - {"column": "o_orderkey", "name": "orderkey", "datatype": "number"}, - {"column": "o_orderdate", "name": "orderdate", "datatype": "date"}, - ], - }]) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - sm = result["semantic_model"][0] - assert sm["name"] == "tpch" - ds = sm["datasets"][0] - assert ds["source"] == "SNOWFLAKE_SAMPLE_DATA.TPCH_SF1.ORDERS" - assert ds["primary_key"] == ["orderkey"] - - def test_field_types_from_datatypes(self, tmp_path): - _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], - "key_dataset": "orders", "sql": "db.s.orders", - "dataset_attrs": [ - {"column": "id", "name": "id", "datatype": "number"}, - {"column": "status", "name": "status", "datatype": "string"}, - {"column": "created_at", "name": "created_at", "datatype": "timestamp"}, - ]}]) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - fields = {f["name"]: f for f in result["semantic_model"][0]["datasets"][0]["fields"]} - assert fields["id"].get("dimension") is None - assert fields["status"]["dimension"] == {"is_time": False} - assert fields["created_at"]["dimension"] == {"is_time": True} - - def test_labels_become_osi_label_and_ai_context(self, tmp_path): - _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], - "key_dataset": "orders", "sql": "db.s.orders", - "dataset_attrs": [ - {"column": "status", "name": "status", "datatype": "string", - "labels": ["sales", "reporting"]}, - ]}]) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - f = next(f for f in result["semantic_model"][0]["datasets"][0]["fields"] if f["name"] == "status") - assert f["label"] == "sales" - assert "sales" in (f.get("ai_context") or {}).get("synonyms", []) - - def test_many_to_one_relation_to_osi(self, tmp_path): - _write_workspace(str(tmp_path), "ws", [ - {"name": "orders", "keys": ["order_id"], "key_dataset": "orders", "sql": "db.s.orders", - "relations": [{"target_entity": "customers", "rel_type": "many-to-one", - "connection": [{"src_field": "customer_id", "target_field": "id"}]}], - "dataset_attrs": []}, - {"name": "customers", "keys": ["id"], "key_dataset": "customers", "sql": "db.s.customers", "dataset_attrs": []}, - ]) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - rels = result["semantic_model"][0]["relationships"] - assert len(rels) == 1 - assert rels[0]["from"] == "orders" and rels[0]["to"] == "customers" - - def test_one_to_many_direction_flipped(self, tmp_path): - _write_workspace(str(tmp_path), "ws", [ - {"name": "customers", "keys": ["id"], "key_dataset": "customers", "sql": "db.s.customers", - "relations": [{"target_entity": "orders", "rel_type": "one-to-many", - "connection": [{"src_field": "id", "target_field": "customer_id"}]}], - "dataset_attrs": []}, - {"name": "orders", "keys": ["order_id"], "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": []}, - ]) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - rel = result["semantic_model"][0]["relationships"][0] - assert rel["from"] == "orders" and rel["to"] == "customers" - - def test_duplicate_relations_deduplicated(self, tmp_path): - _write_workspace(str(tmp_path), "ws", [ - {"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "db.s.orders", - "relations": [{"target_entity": "customers", "rel_type": "many-to-one", - "connection": [{"src_field": "cid", "target_field": "id"}]}], - "dataset_attrs": []}, - {"name": "customers", "keys": ["id"], "key_dataset": "customers", "sql": "db.s.customers", - "relations": [{"target_entity": "orders", "rel_type": "one-to-many", - "connection": [{"src_field": "id", "target_field": "cid"}]}], - "dataset_attrs": []}, - ]) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - assert len(result["semantic_model"][0].get("relationships", [])) == 1 - - def test_metrics_converted(self, tmp_path): - _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], - "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], - "metrics": [{"type": "metric", "entity": "orders", "name": "count", - "datatype": "number", "sql": "COUNT(*)"}]}]) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - m = result["semantic_model"][0]["metrics"][0] - assert m["name"] == "count" - assert m["expression"]["dialects"][0]["expression"] == "COUNT(*)" - - def test_metric_entity_preserved_in_custom_extension(self, tmp_path): - _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], - "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], - "metrics": [{"type": "metric", "entity": "orders", "name": "cnt", - "datatype": "number", "sql": "COUNT(*)"}]}]) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - m = result["semantic_model"][0]["metrics"][0] - ext = m["custom_extensions"][0] - assert ext["vendor_name"] == "HONEYDEW" - assert json.loads(ext["data"])["entity"] == "orders" - - def test_calculated_attribute_as_field(self, tmp_path): - _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], - "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], - "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", - "name": "discounted", "datatype": "number", - "sql": "orders.price * (1 - orders.discount)"}]}]) +def test_honeydew_to_osi_basic(tmp_path): + _write_workspace(str(tmp_path), "tpch", [{ + "name": "orders", "keys": ["orderkey"], "key_dataset": "tpch_orders", + "sql": "SNOWFLAKE_SAMPLE_DATA.TPCH_SF1.ORDERS", + "dataset_attrs": [ + {"column": "o_orderkey", "name": "orderkey", "datatype": "number"}, + {"column": "o_orderdate", "name": "orderdate", "datatype": "date"}, + ], + }]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + sm = result["semantic_model"][0] + assert sm["name"] == "tpch" + ds = sm["datasets"][0] + assert ds["source"] == "SNOWFLAKE_SAMPLE_DATA.TPCH_SF1.ORDERS" + assert ds["primary_key"] == ["orderkey"] + + +@pytest.mark.parametrize("col_name,datatype,expected_dim", [ + ("id", "number", None), + ("status", "string", {"is_time": False}), + ("created_at", "timestamp", {"is_time": True}), +]) +def test_honeydew_to_osi_field_types(tmp_path, col_name, datatype, expected_dim): + _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], + "key_dataset": "orders", "sql": "db.s.orders", + "dataset_attrs": [{"column": col_name, "name": col_name, "datatype": datatype}]}]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + fields = {f["name"]: f for f in result["semantic_model"][0]["datasets"][0]["fields"]} + assert fields[col_name].get("dimension") == expected_dim + + +def test_honeydew_to_osi_labels_become_label_and_ai_context(tmp_path): + _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], + "key_dataset": "orders", "sql": "db.s.orders", + "dataset_attrs": [ + {"column": "status", "name": "status", "datatype": "string", + "labels": ["sales", "reporting"]}, + ]}]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + f = next(f for f in result["semantic_model"][0]["datasets"][0]["fields"] if f["name"] == "status") + assert f["label"] == "sales" + assert "sales" in (f.get("ai_context") or {}).get("synonyms", []) + + +def test_honeydew_to_osi_many_to_one_relation(tmp_path): + _write_workspace(str(tmp_path), "ws", [ + {"name": "orders", "keys": ["order_id"], "key_dataset": "orders", "sql": "db.s.orders", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "connection": [{"src_field": "customer_id", "target_field": "id"}]}], + "dataset_attrs": []}, + {"name": "customers", "keys": ["id"], "key_dataset": "customers", "sql": "db.s.customers", "dataset_attrs": []}, + ]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + rels = result["semantic_model"][0]["relationships"] + assert len(rels) == 1 + assert rels[0]["from"] == "orders" and rels[0]["to"] == "customers" + + +def test_honeydew_to_osi_one_to_many_direction_flipped(tmp_path): + _write_workspace(str(tmp_path), "ws", [ + {"name": "customers", "keys": ["id"], "key_dataset": "customers", "sql": "db.s.customers", + "relations": [{"target_entity": "orders", "rel_type": "one-to-many", + "connection": [{"src_field": "id", "target_field": "customer_id"}]}], + "dataset_attrs": []}, + {"name": "orders", "keys": ["order_id"], "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": []}, + ]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + rel = result["semantic_model"][0]["relationships"][0] + assert rel["from"] == "orders" and rel["to"] == "customers" + + +def test_honeydew_to_osi_duplicate_relations_deduplicated(tmp_path): + _write_workspace(str(tmp_path), "ws", [ + {"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "db.s.orders", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "connection": [{"src_field": "cid", "target_field": "id"}]}], + "dataset_attrs": []}, + {"name": "customers", "keys": ["id"], "key_dataset": "customers", "sql": "db.s.customers", + "relations": [{"target_entity": "orders", "rel_type": "one-to-many", + "connection": [{"src_field": "id", "target_field": "cid"}]}], + "dataset_attrs": []}, + ]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + assert len(result["semantic_model"][0].get("relationships", [])) == 1 + + +def test_honeydew_to_osi_metric_converted(tmp_path): + _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], + "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], + "metrics": [{"type": "metric", "entity": "orders", "name": "count", + "datatype": "number", "sql": "COUNT(*)"}]}]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + m = result["semantic_model"][0]["metrics"][0] + assert m["name"] == "count" + assert m["expression"]["dialects"][0]["expression"] == "COUNT(*)" + + +def test_honeydew_to_osi_metric_entity_preserved_in_extension(tmp_path): + _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], + "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], + "metrics": [{"type": "metric", "entity": "orders", "name": "cnt", + "datatype": "number", "sql": "COUNT(*)"}]}]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + m = result["semantic_model"][0]["metrics"][0] + ext = m["custom_extensions"][0] + assert ext["vendor_name"] == "HONEYDEW" + assert json.loads(ext["data"])["entity"] == "orders" + + +def test_honeydew_to_osi_calculated_attribute_as_field(tmp_path): + _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], + "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], + "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", + "name": "discounted", "datatype": "number", + "sql": "orders.price * (1 - orders.discount)"}]}]) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + fields = {f["name"]: f for f in result["semantic_model"][0]["datasets"][0]["fields"]} + assert "discounted" in fields + assert "orders.price" in fields["discounted"]["expression"]["dialects"][0]["expression"] + + +def test_honeydew_to_osi_missing_workspace_raises(tmp_path): + with pytest.raises(HoneydewConversionError, match="workspace.yml"): + convert_honeydew_to_osi(str(tmp_path)) + + +def test_honeydew_to_osi_missing_schema_dir_empty_model(tmp_path): + (tmp_path / "workspace.yml").write_text(yaml.dump({"type": "workspace", "name": "ws"})) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + assert result["semantic_model"][0]["datasets"] == [] + + +def test_honeydew_to_osi_vendors_includes_honeydew(tmp_path): + (tmp_path / "workspace.yml").write_text(yaml.dump({"type": "workspace", "name": "ws"})) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + assert "HONEYDEW" in result.get("vendors", []) + + +def test_honeydew_to_osi_empty_metric_sql_skipped(tmp_path): + _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], + "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], + "metrics": [{"type": "metric", "entity": "orders", "name": "bad", + "datatype": "number", "sql": ""}]}]) + with warnings.catch_warnings(record=True): result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - fields = {f["name"]: f for f in result["semantic_model"][0]["datasets"][0]["fields"]} - assert "discounted" in fields - assert "orders.price" in fields["discounted"]["expression"]["dialects"][0]["expression"] - - def test_missing_workspace_yml_raises(self, tmp_path): - with pytest.raises(HoneydewConversionError, match="workspace.yml"): - convert_honeydew_to_osi(str(tmp_path)) + assert "metrics" not in result["semantic_model"][0] - def test_missing_schema_dir_produces_empty_model(self, tmp_path): - (tmp_path / "workspace.yml").write_text(yaml.dump({"type": "workspace", "name": "ws"})) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - assert result["semantic_model"][0]["datasets"] == [] - def test_vendors_includes_honeydew(self, tmp_path): - (tmp_path / "workspace.yml").write_text(yaml.dump({"type": "workspace", "name": "ws"})) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - assert "HONEYDEW" in result.get("vendors", []) +# ───────────────────────────────────────────────────────────────────────────── +# OSI → Honeydew → OSI round-trip tests +# ───────────────────────────────────────────────────────────────────────────── - def test_empty_metrics_skipped(self, tmp_path): - _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], - "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], - "metrics": [{"type": "metric", "entity": "orders", "name": "bad", - "datatype": "number", "sql": ""}]}]) - with warnings.catch_warnings(record=True): - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - assert "metrics" not in result["semantic_model"][0] +def test_osi_roundtrip_name_and_description(tmp_path): + model = {"name": "retail", "description": "Retail model", "datasets": []} + sm = _osi_roundtrip(model, tmp_path) + assert sm["name"] == "retail" and sm["description"] == "Retail model" + + +@pytest.mark.parametrize("primary_key", [ + ["order_id"], + ["order_id", "line_no"], +]) +def test_osi_roundtrip_primary_key(tmp_path, primary_key): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "primary_key": primary_key, "fields": []}]} + sm = _osi_roundtrip(model, tmp_path) + assert sm["datasets"][0]["primary_key"] == primary_key + + +def test_osi_roundtrip_unique_keys(tmp_path): + model = {"name": "m", "datasets": [{"name": "items", "source": "db.s.items", + "primary_key": ["id"], + "unique_keys": [["sku"], ["id", "variant"]], + "fields": []}]} + sm = _osi_roundtrip(model, tmp_path) + assert sm["datasets"][0]["unique_keys"] == [["sku"], ["id", "variant"]] + + +def test_osi_roundtrip_field_label(tmp_path): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "fields": [{"name": "status", "label": "sales", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, + "dimension": {"is_time": False}}]}]} + sm = _osi_roundtrip(model, tmp_path) + f = next(f for f in sm["datasets"][0]["fields"] if f["name"] == "status") + assert f["label"] == "sales" + + +def test_osi_roundtrip_ai_context_string(tmp_path): + ai_ctx_value = "order status, order state" + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "fields": [{"name": "status", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, + "ai_context": ai_ctx_value, + "dimension": {"is_time": False}}]}]} + sm = _osi_roundtrip(model, tmp_path) + f = next(f for f in sm["datasets"][0]["fields"] if f["name"] == "status") + # String ai_context is merged into description on OSI→Honeydew; value must be recoverable + assert ai_ctx_value in (f.get("description") or "") or f.get("ai_context") == ai_ctx_value + + +def test_osi_roundtrip_ai_context_dict(tmp_path): + ctx = {"instructions": "Use for revenue analysis", "synonyms": ["revenue", "sales"]} + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "fields": [{"name": "total", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, + "ai_context": ctx}]}]} + sm = _osi_roundtrip(model, tmp_path) + f = next(f for f in sm["datasets"][0]["fields"] if f["name"] == "total") + assert f.get("ai_context") == ctx + + +def test_osi_roundtrip_model_ai_context(tmp_path): + ctx = {"instructions": "Retail analytics", "synonyms": ["store"]} + model = {"name": "m", "ai_context": ctx, "datasets": []} + sm = _osi_roundtrip(model, tmp_path) + assert sm.get("ai_context") == ctx + + +def test_osi_roundtrip_non_honeydew_custom_extensions(tmp_path): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "custom_extensions": [{"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "WH"}'}], + "fields": []}]} + sm = _osi_roundtrip(model, tmp_path) + exts = sm["datasets"][0].get("custom_extensions") or [] + assert any(e["vendor_name"] == "SNOWFLAKE" for e in exts) + + +def test_osi_roundtrip_relationship_name(tmp_path): + model = {"name": "m", "datasets": [ + {"name": "orders", "source": "db.s.orders", "fields": []}, + {"name": "customers", "source": "db.s.customers", "fields": []}, + ], "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", + "from_columns": ["cid"], "to_columns": ["id"]}]} + sm = _osi_roundtrip(model, tmp_path) + assert sm["relationships"][0]["name"] == "orders_to_customers" + + +def test_osi_roundtrip_relationship_columns(tmp_path): + model = {"name": "m", "datasets": [ + {"name": "orders", "source": "db.s.orders", "fields": []}, + {"name": "customers", "source": "db.s.customers", "fields": []}, + ], "relationships": [{"name": "r", "from": "orders", "to": "customers", + "from_columns": ["cid"], "to_columns": ["id"]}]} + sm = _osi_roundtrip(model, tmp_path) + rel = sm["relationships"][0] + assert rel["from_columns"] == ["cid"] and rel["to_columns"] == ["id"] + + +def test_osi_roundtrip_metric(tmp_path): + model = {"name": "m", + "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], + "metrics": [{"name": "total_revenue", "description": "Sum of sales", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}]} + sm = _osi_roundtrip(model, tmp_path) + m = sm["metrics"][0] + assert m["name"] == "total_revenue" + assert m["expression"]["dialects"][0]["expression"] == "SUM(orders.total)" + assert m["description"] == "Sum of sales" + + +def test_osi_roundtrip_tpcds_example(tmp_path): + tpcds_path = ( + Path(__file__).resolve().parent.parent.parent.parent + / "examples" / "tpcds_semantic_model.yaml" + ) + if not tpcds_path.exists(): + pytest.skip("TPC-DS example not found") + osi_yaml = tpcds_path.read_text() + files = convert_osi_to_honeydew(osi_yaml) + for rel_path, content in files.items(): + p = tmp_path / rel_path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + sm = result["semantic_model"][0] + assert sm["name"] == "tpcds_retail_model" + ds_names = {ds["name"] for ds in sm["datasets"]} + assert "store_sales" in ds_names and "customer" in ds_names # ───────────────────────────────────────────────────────────────────────────── -# Round-trip tests (idempotency) +# Honeydew → OSI → Honeydew round-trip tests # ───────────────────────────────────────────────────────────────────────────── -class TestOsiToHoneydewToOsiRoundTrip: - """OSI → Honeydew → OSI: verify key fields survive both legs.""" - - def _roundtrip(self, model_dict, tmp_path): - files = convert_osi_to_honeydew(_osi(model_dict)) - for rel_path, content in files.items(): - p = tmp_path / rel_path - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(content) - return yaml.safe_load(convert_honeydew_to_osi(str(tmp_path)))["semantic_model"][0] - - def test_name_and_description_preserved(self, tmp_path): - model = {"name": "retail", "description": "Retail model", "datasets": []} - sm = self._roundtrip(model, tmp_path) - assert sm["name"] == "retail" - assert sm["description"] == "Retail model" - - def test_primary_key_preserved(self, tmp_path): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", - "primary_key": ["order_id"], "fields": []}]} - sm = self._roundtrip(model, tmp_path) - assert sm["datasets"][0]["primary_key"] == ["order_id"] - - def test_composite_primary_key_preserved(self, tmp_path): - model = {"name": "m", "datasets": [{"name": "li", "source": "db.s.li", - "primary_key": ["order_id", "line_no"], "fields": []}]} - sm = self._roundtrip(model, tmp_path) - assert sm["datasets"][0]["primary_key"] == ["order_id", "line_no"] - - def test_unique_keys_preserved(self, tmp_path): - model = {"name": "m", "datasets": [{"name": "items", "source": "db.s.items", - "primary_key": ["id"], - "unique_keys": [["sku"], ["id", "variant"]], - "fields": []}]} - sm = self._roundtrip(model, tmp_path) - assert sm["datasets"][0]["unique_keys"] == [["sku"], ["id", "variant"]] - - def test_field_label_preserved(self, tmp_path): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", - "fields": [{"name": "status", "label": "sales", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, - "dimension": {"is_time": False}}]}]} - sm = self._roundtrip(model, tmp_path) - f = next(f for f in sm["datasets"][0]["fields"] if f["name"] == "status") - assert f["label"] == "sales" - - def test_ai_context_string_preserved(self, tmp_path): - ai_ctx_value = "order status, order state" - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", - "fields": [{"name": "status", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, - "ai_context": ai_ctx_value, - "dimension": {"is_time": False}}]}]} - sm = self._roundtrip(model, tmp_path) - f = next(f for f in sm["datasets"][0]["fields"] if f["name"] == "status") - # String ai_context is merged into description on OSI→Honeydew; value must be recoverable - assert ai_ctx_value in (f.get("description") or "") or f.get("ai_context") == ai_ctx_value - - def test_ai_context_dict_preserved(self, tmp_path): - ctx = {"instructions": "Use for revenue analysis", "synonyms": ["revenue", "sales"]} - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", - "fields": [{"name": "total", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, - "ai_context": ctx}]}]} - sm = self._roundtrip(model, tmp_path) - f = next(f for f in sm["datasets"][0]["fields"] if f["name"] == "total") - assert f.get("ai_context") == ctx - - def test_model_ai_context_preserved(self, tmp_path): - ctx = {"instructions": "Retail analytics", "synonyms": ["store"]} - model = {"name": "m", "ai_context": ctx, "datasets": []} - sm = self._roundtrip(model, tmp_path) - assert sm.get("ai_context") == ctx - - def test_non_honeydew_custom_extensions_preserved(self, tmp_path): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", - "custom_extensions": [{"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "WH"}'}], - "fields": []}]} - sm = self._roundtrip(model, tmp_path) - exts = sm["datasets"][0].get("custom_extensions") or [] - assert any(e["vendor_name"] == "SNOWFLAKE" for e in exts) - - def test_relationship_name_preserved(self, tmp_path): - model = {"name": "m", "datasets": [ - {"name": "orders", "source": "db.s.orders", "fields": []}, - {"name": "customers", "source": "db.s.customers", "fields": []}, - ], "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", - "from_columns": ["cid"], "to_columns": ["id"]}]} - sm = self._roundtrip(model, tmp_path) - assert sm["relationships"][0]["name"] == "orders_to_customers" - - def test_relationship_columns_preserved(self, tmp_path): - model = {"name": "m", "datasets": [ - {"name": "orders", "source": "db.s.orders", "fields": []}, - {"name": "customers", "source": "db.s.customers", "fields": []}, - ], "relationships": [{"name": "r", "from": "orders", "to": "customers", - "from_columns": ["cid"], "to_columns": ["id"]}]} - sm = self._roundtrip(model, tmp_path) - rel = sm["relationships"][0] - assert rel["from_columns"] == ["cid"] and rel["to_columns"] == ["id"] - - def test_metric_name_and_expression_preserved(self, tmp_path): - model = {"name": "m", - "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], - "metrics": [{"name": "total_revenue", "description": "Sum of sales", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}]} - sm = self._roundtrip(model, tmp_path) - m = sm["metrics"][0] - assert m["name"] == "total_revenue" - assert m["expression"]["dialects"][0]["expression"] == "SUM(orders.total)" - assert m["description"] == "Sum of sales" - - def test_tpcds_example_roundtrip(self, tmp_path): - tpcds_path = ( - Path(__file__).resolve().parent.parent.parent.parent - / "examples" / "tpcds_semantic_model.yaml" - ) - if not tpcds_path.exists(): - pytest.skip("TPC-DS example not found") - osi_yaml = tpcds_path.read_text() - files = convert_osi_to_honeydew(osi_yaml) - for rel_path, content in files.items(): - p = tmp_path / rel_path - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(content) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - sm = result["semantic_model"][0] - assert sm["name"] == "tpcds_retail_model" - ds_names = {ds["name"] for ds in sm["datasets"]} - assert "store_sales" in ds_names and "customer" in ds_names - - -class TestHoneydewToOsiToHoneydewRoundTrip: - """Honeydew → OSI → Honeydew: verify Honeydew-specific fields survive.""" - - def _roundtrip(self, entities, tmp_path): - _write_workspace(str(tmp_path), "ws", entities) - osi_yaml = convert_honeydew_to_osi(str(tmp_path)) - files = convert_osi_to_honeydew(osi_yaml) - # Write to a second directory - out_dir = tmp_path / "out" - for rel_path, content in files.items(): - p = out_dir / rel_path - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(content) - return out_dir - - def test_entity_name_and_keys_preserved(self, tmp_path): - out_dir = self._roundtrip([{ - "name": "orders", "keys": ["order_id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", "dataset_attrs": [], - }], tmp_path) - entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) - assert entity["name"] == "orders" - assert entity["keys"] == ["order_id"] - - def test_source_preserved(self, tmp_path): - out_dir = self._roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.SCHEMA.ORDERS", "dataset_attrs": [], - }], tmp_path) - ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) - assert ds["sql"] == "DB.SCHEMA.ORDERS" - - def test_column_attributes_preserved(self, tmp_path): - out_dir = self._roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", - "dataset_attrs": [ - {"column": "o_id", "name": "id", "datatype": "number"}, - {"column": "o_status", "name": "status", "datatype": "string"}, - ], - }], tmp_path) - ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) - attrs = {a["name"]: a for a in ds["attributes"]} - assert attrs["id"]["column"] == "o_id" - assert attrs["status"]["datatype"] == "string" - - def test_labels_preserved_on_column(self, tmp_path): - out_dir = self._roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", - "dataset_attrs": [ - {"column": "status", "name": "status", "datatype": "string", "labels": ["sales"]}, - ], - }], tmp_path) - ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) - attrs = {a["name"]: a for a in ds["attributes"]} - assert "sales" in attrs["status"].get("labels", []) - - def test_calculated_attribute_sql_preserved(self, tmp_path): - out_dir = self._roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", "dataset_attrs": [], - "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", - "name": "disc", "datatype": "number", - "sql": "orders.price * (1 - orders.discount)"}], - }], tmp_path) - calc = yaml.safe_load((out_dir / "schema/orders/attributes/disc.yml").read_text()) - assert calc["sql"] == "orders.price * (1 - orders.discount)" - - def test_metric_entity_assignment_preserved(self, tmp_path): - out_dir = self._roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", "dataset_attrs": [], - "metrics": [{"type": "metric", "entity": "orders", "name": "cnt", - "datatype": "number", "sql": "COUNT(*)"}], - }], tmp_path) - m = yaml.safe_load((out_dir / "schema/orders/metrics/cnt.yml").read_text()) - assert m["entity"] == "orders" - assert m["sql"] == "COUNT(*)" - - def test_relation_preserved(self, tmp_path): - out_dir = self._roundtrip([ - {"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", - "relations": [{"target_entity": "customers", "rel_type": "many-to-one", - "connection": [{"src_field": "cid", "target_field": "id"}]}], - "dataset_attrs": []}, - {"name": "customers", "keys": ["id"], "key_dataset": "customers", - "sql": "DB.S.CUSTOMERS", "dataset_attrs": []}, - ], tmp_path) - entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) - assert entity["relations"][0]["target_entity"] == "customers" - assert entity["relations"][0]["connection"][0]["src_field"] == "cid" - - def test_bool_datatype_preserved(self, tmp_path): - out_dir = self._roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", - "dataset_attrs": [ - {"column": "is_active", "name": "is_active", "datatype": "bool"}, - ], - }], tmp_path) - ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) - attrs = {a["name"]: a for a in ds["attributes"]} - assert attrs["is_active"]["datatype"] == "bool" - - def test_connection_expr_preserved(self, tmp_path): - out_dir = self._roundtrip([ - {"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", - "relations": [{"target_entity": "customers", "rel_type": "many-to-one", - "connection_expr": {"sql": "orders.cid = customers.id AND orders.region = customers.region"}}], - "dataset_attrs": []}, - {"name": "customers", "keys": ["id"], "key_dataset": "customers", - "sql": "DB.S.CUSTOMERS", "dataset_attrs": []}, - ], tmp_path) - entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) - rel = entity["relations"][0] - assert rel.get("connection_expr", {}).get("sql") == "orders.cid = customers.id AND orders.region = customers.region" - - def test_dataset_attr_display_name_and_format_preserved(self, tmp_path): - out_dir = self._roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", - "dataset_attrs": [ - {"column": "status", "name": "status", "datatype": "string", - "display_name": "Order Status", "hidden": True, "format_string": "##,###"}, - ], - }], tmp_path) - ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) - attrs = {a["name"]: a for a in ds["attributes"]} - assert attrs["status"]["display_name"] == "Order Status" - assert attrs["status"]["hidden"] is True - assert attrs["status"]["format_string"] == "##,###" - - def test_calc_attr_honeydew_fields_preserved(self, tmp_path): - out_dir = self._roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", "dataset_attrs": [], - "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", - "name": "disc", "datatype": "number", - "sql": "orders.price * 0.9", - "display_name": "Discounted Price", - "timegrain": "day"}], - }], tmp_path) - calc = yaml.safe_load((out_dir / "schema/orders/attributes/disc.yml").read_text()) - assert calc["display_name"] == "Discounted Price" - assert calc["timegrain"] == "day" - - def test_entity_owner_and_display_name_preserved(self, tmp_path): - ws_path = tmp_path / "workspace.yml" - ws_path.write_text(yaml.dump({"type": "workspace", "name": "ws"})) - base = tmp_path / "schema" / "orders" - (base / "datasets").mkdir(parents=True) - (base / "orders.yml").write_text(yaml.dump({ - "type": "entity", "name": "orders", "keys": ["id"], - "key_dataset": "orders", "relations": [], - "owner": "analytics_team", "display_name": "Orders Table", - })) - (base / "datasets" / "orders.yml").write_text(yaml.dump({ - "type": "dataset", "entity": "orders", "name": "orders", - "sql": "DB.S.ORDERS", "dataset_type": "table", "attributes": [], - })) - osi_yaml = convert_honeydew_to_osi(str(tmp_path)) - files = convert_osi_to_honeydew(osi_yaml) - out_dir = tmp_path / "out" - for rel_path, content in files.items(): - p = out_dir / rel_path - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(content) - entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) - assert entity.get("owner") == "analytics_team" - assert entity.get("display_name") == "Orders Table" - - def test_calc_attr_with_simple_identifier_sql_preserved(self, tmp_path): - out_dir = self._roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", "dataset_attrs": [], - "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", - "name": "revenue", "datatype": "number", "sql": "revenue"}], - }], tmp_path) - # sql='revenue' is a simple identifier — must still come back as calculated_attribute - calc_path = out_dir / "schema/orders/attributes/revenue.yml" - assert calc_path.exists(), "calculated_attribute with simple-id sql should not become a dataset column" - calc = yaml.safe_load(calc_path.read_text()) - assert calc["sql"] == "revenue" +def test_honeydew_roundtrip_entity_name_and_keys(tmp_path): + out_dir = _honeydew_roundtrip([{ + "name": "orders", "keys": ["order_id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", "dataset_attrs": [], + }], tmp_path) + entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) + assert entity["name"] == "orders" and entity["keys"] == ["order_id"] + + +def test_honeydew_roundtrip_source(tmp_path): + out_dir = _honeydew_roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.SCHEMA.ORDERS", "dataset_attrs": [], + }], tmp_path) + ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) + assert ds["sql"] == "DB.SCHEMA.ORDERS" + + +def test_honeydew_roundtrip_column_attributes(tmp_path): + out_dir = _honeydew_roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", + "dataset_attrs": [ + {"column": "o_id", "name": "id", "datatype": "number"}, + {"column": "o_status", "name": "status", "datatype": "string"}, + ], + }], tmp_path) + ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) + attrs = {a["name"]: a for a in ds["attributes"]} + assert attrs["id"]["column"] == "o_id" + assert attrs["status"]["datatype"] == "string" + + +def test_honeydew_roundtrip_labels_on_column(tmp_path): + out_dir = _honeydew_roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", + "dataset_attrs": [{"column": "status", "name": "status", "datatype": "string", "labels": ["sales"]}], + }], tmp_path) + ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) + attrs = {a["name"]: a for a in ds["attributes"]} + assert "sales" in attrs["status"].get("labels", []) + + +def test_honeydew_roundtrip_calculated_attribute_sql(tmp_path): + out_dir = _honeydew_roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", "dataset_attrs": [], + "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", + "name": "disc", "datatype": "number", + "sql": "orders.price * (1 - orders.discount)"}], + }], tmp_path) + calc = yaml.safe_load((out_dir / "schema/orders/attributes/disc.yml").read_text()) + assert calc["sql"] == "orders.price * (1 - orders.discount)" + + +def test_honeydew_roundtrip_metric_entity_assignment(tmp_path): + out_dir = _honeydew_roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", "dataset_attrs": [], + "metrics": [{"type": "metric", "entity": "orders", "name": "cnt", + "datatype": "number", "sql": "COUNT(*)"}], + }], tmp_path) + m = yaml.safe_load((out_dir / "schema/orders/metrics/cnt.yml").read_text()) + assert m["entity"] == "orders" and m["sql"] == "COUNT(*)" + + +def test_honeydew_roundtrip_relation(tmp_path): + out_dir = _honeydew_roundtrip([ + {"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "connection": [{"src_field": "cid", "target_field": "id"}]}], + "dataset_attrs": []}, + {"name": "customers", "keys": ["id"], "key_dataset": "customers", + "sql": "DB.S.CUSTOMERS", "dataset_attrs": []}, + ], tmp_path) + entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) + assert entity["relations"][0]["target_entity"] == "customers" + assert entity["relations"][0]["connection"][0]["src_field"] == "cid" + + +def test_honeydew_roundtrip_bool_datatype(tmp_path): + out_dir = _honeydew_roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", + "dataset_attrs": [{"column": "is_active", "name": "is_active", "datatype": "bool"}], + }], tmp_path) + ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) + attrs = {a["name"]: a for a in ds["attributes"]} + assert attrs["is_active"]["datatype"] == "bool" + + +def test_honeydew_roundtrip_connection_expr(tmp_path): + out_dir = _honeydew_roundtrip([ + {"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "connection_expr": {"sql": "orders.cid = customers.id AND orders.region = customers.region"}}], + "dataset_attrs": []}, + {"name": "customers", "keys": ["id"], "key_dataset": "customers", + "sql": "DB.S.CUSTOMERS", "dataset_attrs": []}, + ], tmp_path) + entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) + rel = entity["relations"][0] + assert rel.get("connection_expr", {}).get("sql") == "orders.cid = customers.id AND orders.region = customers.region" + + +@pytest.mark.parametrize("attr_extra,check_key,check_val", [ + ({"display_name": "Order Status"}, "display_name", "Order Status"), + ({"hidden": True}, "hidden", True), + ({"format_string": "##,###"}, "format_string", "##,###"), +]) +def test_honeydew_roundtrip_dataset_attr_honeydew_field(tmp_path, attr_extra, check_key, check_val): + attr = {"column": "status", "name": "status", "datatype": "string", **attr_extra} + out_dir = _honeydew_roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", "dataset_attrs": [attr], + }], tmp_path) + ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) + attrs = {a["name"]: a for a in ds["attributes"]} + assert attrs["status"][check_key] == check_val + + +@pytest.mark.parametrize("calc_extra,check_key,check_val", [ + ({"display_name": "Discounted Price"}, "display_name", "Discounted Price"), + ({"timegrain": "day"}, "timegrain", "day"), +]) +def test_honeydew_roundtrip_calc_attr_honeydew_field(tmp_path, calc_extra, check_key, check_val): + calc = {"type": "calculated_attribute", "entity": "orders", + "name": "disc", "datatype": "number", "sql": "orders.price * 0.9", **calc_extra} + out_dir = _honeydew_roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", "dataset_attrs": [], "calc_attrs": [calc], + }], tmp_path) + result = yaml.safe_load((out_dir / "schema/orders/attributes/disc.yml").read_text()) + assert result[check_key] == check_val + + +@pytest.mark.parametrize("entity_extra,check_key,check_val", [ + ({"owner": "analytics_team"}, "owner", "analytics_team"), + ({"display_name": "Orders Table"}, "display_name", "Orders Table"), + ({"hidden": True}, "hidden", True), + ({"folder": "finance"}, "folder", "finance"), +]) +def test_honeydew_roundtrip_entity_honeydew_field(tmp_path, entity_extra, check_key, check_val): + out_dir = _honeydew_roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", "dataset_attrs": [], **entity_extra, + }], tmp_path) + entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) + assert entity.get(check_key) == check_val + + +def test_honeydew_roundtrip_calc_attr_simple_identifier_stays_calc(tmp_path): + out_dir = _honeydew_roundtrip([{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", "dataset_attrs": [], + "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", + "name": "revenue", "datatype": "number", "sql": "revenue"}], + }], tmp_path) + calc_path = out_dir / "schema/orders/attributes/revenue.yml" + assert calc_path.exists(), "calculated_attribute with simple-id sql should not become a dataset column" + calc = yaml.safe_load(calc_path.read_text()) + assert calc["sql"] == "revenue" # ───────────────────────────────────────────────────────────────────────────── # Bug-fix regression tests # ───────────────────────────────────────────────────────────────────────────── -class TestBugFixes: - def test_empty_string_expression_skipped(self): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ - "name": "bad", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": ""}]}, - "dimension": {"is_time": False}, - }]}]} +@pytest.mark.parametrize("expression", [ + {"dialects": [{"dialect": "ANSI_SQL", "expression": ""}]}, + {"dialects": [{"dialect": "ANSI_SQL", "expression": " "}]}, +]) +def test_empty_or_whitespace_field_expression_skipped(expression): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "bad", + "expression": expression, + "dimension": {"is_time": False}, + }]}]} + files = convert_osi_to_honeydew(_osi(model)) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + assert all(a["name"] != "bad" for a in ds["attributes"]) + assert "schema/orders/attributes/bad.yml" not in files + + +@pytest.mark.parametrize("expression", [ + {"dialects": [{"dialect": "ANSI_SQL", "expression": ""}]}, + {"dialects": [{"dialect": "ANSI_SQL", "expression": " "}]}, +]) +def test_empty_or_whitespace_metric_expression_skipped(expression): + model = {"name": "m", + "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], + "metrics": [{"name": "bad_m", "expression": expression}]} + files = convert_osi_to_honeydew(_osi(model)) + assert "schema/orders/metrics/bad_m.yml" not in files + + +def test_non_dict_expression_warns(): + model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "bad", + "expression": "just_a_string", + "dimension": {"is_time": False}, + }]}]} + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") files = convert_osi_to_honeydew(_osi(model)) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - names = [a["name"] for a in ds["attributes"]] - assert "bad" not in names - assert "schema/orders/attributes/bad.yml" not in files - - def test_duplicate_metric_name_warns(self): - model = {"name": "m", - "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], - "metrics": [ - {"name": "total", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.a)"}]}}, - {"name": "total", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.b)"}]}}, - ]} - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - files = convert_osi_to_honeydew(_osi(model)) - assert any("total" in str(x.message) for x in w) - # Last definition wins - m = yaml.safe_load(files["schema/orders/metrics/total.yml"]) - assert "orders.b" in m["sql"] - - def test_metric_string_ai_context_preserved_in_roundtrip(self, tmp_path): - model = {"name": "m", - "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], - "metrics": [{"name": "rev", "ai_context": "Use for revenue analysis", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}]} - files = convert_osi_to_honeydew(_osi(model)) - for rel_path, content in files.items(): - p = tmp_path / rel_path - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(content) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - m = result["semantic_model"][0]["metrics"][0] - assert m.get("ai_context") == "Use for revenue analysis" - - def test_whitespace_expression_skipped(self): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ - "name": "bad", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": " "}]}, - "dimension": {"is_time": False}, - }]}]} + assert any("must be a mapping" in str(x.message) for x in w) + ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) + assert all(a["name"] != "bad" for a in ds["attributes"]) + + +def test_duplicate_metric_name_warns(): + model = {"name": "m", + "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], + "metrics": [ + {"name": "total", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.a)"}]}}, + {"name": "total", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.b)"}]}}, + ]} + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") files = convert_osi_to_honeydew(_osi(model)) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - names = [a["name"] for a in ds["attributes"]] - assert "bad" not in names - assert "schema/orders/attributes/bad.yml" not in files - - def test_non_dict_expression_warns(self): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ - "name": "bad", - "expression": "just_a_string", - "dimension": {"is_time": False}, - }]}]} - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - files = convert_osi_to_honeydew(_osi(model)) - assert any("must be a mapping" in str(x.message) for x in w) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - assert all(a["name"] != "bad" for a in ds["attributes"]) - - def test_malformed_osi_metadata_json_warns(self, tmp_path): - ws_path = tmp_path / "workspace.yml" - ws_path.write_text(yaml.dump({"type": "workspace", "name": "ws"})) - base = tmp_path / "schema" / "orders" - (base / "datasets").mkdir(parents=True) - entity = { - "type": "entity", "name": "orders", "keys": ["id"], "key_dataset": "orders", - "relations": [], - "metadata": [{"name": "osi", "metadata": [ - {"name": "unique_keys", "value": "[broken json"}, - ]}], - } - (base / "orders.yml").write_text(yaml.dump(entity)) - (base / "datasets" / "orders.yml").write_text(yaml.dump( - {"type": "dataset", "entity": "orders", "name": "orders", - "sql": "DB.S.ORDERS", "dataset_type": "table", "attributes": []} - )) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - convert_honeydew_to_osi(str(tmp_path)) - assert any("unique_keys" in str(x.message) for x in w) + assert any("total" in str(x.message) for x in w) + m = yaml.safe_load(files["schema/orders/metrics/total.yml"]) + assert "orders.b" in m["sql"] + + +def test_metric_string_ai_context_preserved_in_roundtrip(tmp_path): + model = {"name": "m", + "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], + "metrics": [{"name": "rev", "ai_context": "Use for revenue analysis", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}]} + files = convert_osi_to_honeydew(_osi(model)) + for rel_path, content in files.items(): + p = tmp_path / rel_path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + m = result["semantic_model"][0]["metrics"][0] + assert m.get("ai_context") == "Use for revenue analysis" + + +def test_malformed_osi_metadata_json_warns(tmp_path): + ws_path = tmp_path / "workspace.yml" + ws_path.write_text(yaml.dump({"type": "workspace", "name": "ws"})) + base = tmp_path / "schema" / "orders" + (base / "datasets").mkdir(parents=True) + entity = { + "type": "entity", "name": "orders", "keys": ["id"], "key_dataset": "orders", + "relations": [], + "metadata": [{"name": "osi", "metadata": [ + {"name": "unique_keys", "value": "[broken json"}, + ]}], + } + (base / "orders.yml").write_text(yaml.dump(entity)) + (base / "datasets" / "orders.yml").write_text(yaml.dump( + {"type": "dataset", "entity": "orders", "name": "orders", + "sql": "DB.S.ORDERS", "dataset_type": "table", "attributes": []} + )) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + convert_honeydew_to_osi(str(tmp_path)) + assert any("unique_keys" in str(x.message) for x in w) From 35b1f1b83d3714a75a75b3033572afebed7d97f5 Mon Sep 17 00:00:00 2001 From: Baruch Oxman Date: Wed, 27 May 2026 18:34:19 +0300 Subject: [PATCH 21/89] Five code quality improvements to the Honeydew converter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove requirements.txt (superseded by pyproject.toml) - Extract _fields_to_honeydew() from _dataset_to_files(): field classification is now a named, independently testable function; four direct unit tests added - Warn when a relationship has neither from_columns nor connection_expr so callers discover the incomplete join before it reaches Honeydew - Round-trip the vendors list: non-HONEYDEW vendors are stored in the workspace osi metadata on OSI→Honeydew and merged back on the return trip (HONEYDEW always appears first) - Add main() CLI smoke tests via subprocess: osi-to-honeydew writes the expected workspace.yml, honeydew-to-osi writes a parseable OSI YAML, and path traversal in an entity name is rejected with exit code 1 - Update README setup instructions to use pip install . / pip install -e . Co-Authored-By: Claude Sonnet 4.6 --- converters/honeydew/README.md | 4 +- converters/honeydew/requirements.txt | 2 - .../honeydew/src/honeydew_osi_converter.py | 139 ++++++++++------- .../tests/test_honeydew_osi_converter.py | 147 ++++++++++++++++++ 4 files changed, 232 insertions(+), 60 deletions(-) delete mode 100644 converters/honeydew/requirements.txt diff --git a/converters/honeydew/README.md b/converters/honeydew/README.md index 103d19a..62e1e8b 100644 --- a/converters/honeydew/README.md +++ b/converters/honeydew/README.md @@ -43,7 +43,9 @@ Bidirectional converter between [OSI](../../core-spec/spec.md) semantic models a ## Setup ```bash -pip install -r requirements.txt +pip install . +# or in editable mode for development: +pip install -e . ``` ## Usage diff --git a/converters/honeydew/requirements.txt b/converters/honeydew/requirements.txt deleted file mode 100644 index 2f29b5b..0000000 --- a/converters/honeydew/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -PyYAML>=5.0 -pytest>=7.0 diff --git a/converters/honeydew/src/honeydew_osi_converter.py b/converters/honeydew/src/honeydew_osi_converter.py index e082360..3857bb7 100644 --- a/converters/honeydew/src/honeydew_osi_converter.py +++ b/converters/honeydew/src/honeydew_osi_converter.py @@ -85,10 +85,11 @@ def convert_osi_to_honeydew(osi_yaml_str: str) -> dict[str, str]: "only the first will be converted" ) - return _model_to_files(semantic_models[0]) + vendors = [v for v in (root.get("vendors") or []) if v != HONEYDEW_VENDOR] + return _model_to_files(semantic_models[0], extra_vendors=vendors) -def _model_to_files(sm: dict[str, Any]) -> dict[str, str]: +def _model_to_files(sm: dict[str, Any], *, extra_vendors: list[str] | None = None) -> dict[str, str]: name = sm.get("name") if not name: raise HoneydewConversionError("Missing 'name' in semantic model") @@ -99,10 +100,14 @@ def _model_to_files(sm: dict[str, Any]) -> dict[str, str]: if sm.get("description"): workspace["description"] = sm["description"] - # Preserve model-level ai_context and non-HONEYDEW custom_extensions + # Preserve model-level ai_context, non-HONEYDEW custom_extensions, and extra vendors model_ai_ctx = sm.get("ai_context") model_ext = [e for e in (sm.get("custom_extensions") or []) if e.get("vendor_name") != HONEYDEW_VENDOR] - ws_meta = _build_osi_metadata(ai_context=model_ai_ctx, custom_extensions=model_ext or None) + ws_meta = _build_osi_metadata( + ai_context=model_ai_ctx, + custom_extensions=model_ext or None, + extra_vendors=extra_vendors or None, + ) if ws_meta: workspace["metadata"] = [ws_meta] @@ -139,57 +144,11 @@ def _model_to_files(sm: dict[str, Any]) -> dict[str, str]: return files -def _dataset_to_files( - ds: dict[str, Any], - relations: list[dict[str, Any]], - metrics: list[dict[str, Any]], -) -> dict[str, str]: - entity_name = ds["name"] - base = f"schema/{entity_name}" - files: dict[str, str] = {} - - primary_key = ds.get("primary_key") or [] - unique_keys = ds.get("unique_keys") - description = ds.get("description") - ai_context = ds.get("ai_context") - fields = ds.get("fields") or [] - ds_ext = [e for e in (ds.get("custom_extensions") or []) if e.get("vendor_name") != HONEYDEW_VENDOR] - - # ── entity YAML ──────────────────────────────────────────────────────────── - entity_dict: dict[str, Any] = {"type": "entity", "name": entity_name} - if description: - entity_dict["description"] = description - if primary_key: - entity_dict["keys"] = list(primary_key) - entity_dict["key_dataset"] = entity_name - - # Restore Honeydew-specific entity fields from HONEYDEW custom_extension - entity_hd_hint = _get_honeydew_extension(ds) - for key in ("owner", "display_name", "hidden", "folder"): - if key in entity_hd_hint: - entity_dict[key] = entity_hd_hint[key] - if "labels" in entity_hd_hint: - entity_dict["labels"] = entity_hd_hint["labels"] - - honeydew_relations = [] - for rel in relations: - hr = _osi_relation_to_honeydew(rel) - if hr is not None: - honeydew_relations.append(hr) - entity_dict["relations"] = honeydew_relations - - # Preserve OSI fields that have no Honeydew native equivalent - entity_meta = _build_osi_metadata( - ai_context=ai_context, - unique_keys=unique_keys, - custom_extensions=ds_ext or None, - ) - if entity_meta: - entity_dict["metadata"] = [entity_meta] - - files[f"{base}/{entity_name}.yml"] = _dump(entity_dict) - - # ── classify fields into dataset attributes vs calculated attributes ──────── +def _fields_to_honeydew( + fields: list[dict[str, Any]], + entity_name: str, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Classify OSI fields into Honeydew dataset attributes and calculated attributes.""" dataset_attrs: list[dict[str, Any]] = [] calc_attrs: list[dict[str, Any]] = [] @@ -264,6 +223,62 @@ def _dataset_to_files( calc[k] = hd_hint[k] calc_attrs.append(calc) + return dataset_attrs, calc_attrs + + +def _dataset_to_files( + ds: dict[str, Any], + relations: list[dict[str, Any]], + metrics: list[dict[str, Any]], +) -> dict[str, str]: + entity_name = ds["name"] + base = f"schema/{entity_name}" + files: dict[str, str] = {} + + primary_key = ds.get("primary_key") or [] + unique_keys = ds.get("unique_keys") + description = ds.get("description") + ai_context = ds.get("ai_context") + fields = ds.get("fields") or [] + ds_ext = [e for e in (ds.get("custom_extensions") or []) if e.get("vendor_name") != HONEYDEW_VENDOR] + + # ── entity YAML ──────────────────────────────────────────────────────────── + entity_dict: dict[str, Any] = {"type": "entity", "name": entity_name} + if description: + entity_dict["description"] = description + if primary_key: + entity_dict["keys"] = list(primary_key) + entity_dict["key_dataset"] = entity_name + + # Restore Honeydew-specific entity fields from HONEYDEW custom_extension + entity_hd_hint = _get_honeydew_extension(ds) + for key in ("owner", "display_name", "hidden", "folder"): + if key in entity_hd_hint: + entity_dict[key] = entity_hd_hint[key] + if "labels" in entity_hd_hint: + entity_dict["labels"] = entity_hd_hint["labels"] + + honeydew_relations = [] + for rel in relations: + hr = _osi_relation_to_honeydew(rel) + if hr is not None: + honeydew_relations.append(hr) + entity_dict["relations"] = honeydew_relations + + # Preserve OSI fields that have no Honeydew native equivalent + entity_meta = _build_osi_metadata( + ai_context=ai_context, + unique_keys=unique_keys, + custom_extensions=ds_ext or None, + ) + if entity_meta: + entity_dict["metadata"] = [entity_meta] + + files[f"{base}/{entity_name}.yml"] = _dump(entity_dict) + + # ── classify fields into dataset attributes vs calculated attributes ──────── + dataset_attrs, calc_attrs = _fields_to_honeydew(fields, entity_name) + # ── dataset YAML ─────────────────────────────────────────────────────────── source_sql, dataset_type = _parse_osi_source(ds.get("source", "")) dataset_dict: dict[str, Any] = { @@ -356,6 +371,11 @@ def _osi_relation_to_honeydew(rel: dict[str, Any]) -> dict[str, Any] | None: hd_ext = _get_honeydew_extension(rel) if hd_ext.get("connection_expr"): honeydew_rel["connection_expr"] = {"sql": hd_ext["connection_expr"]} + else: + warnings.warn( + f"Relationship '{rel_name}' has no from_columns and no connection_expr; " + "Honeydew will not be able to resolve the join" + ) return honeydew_rel @@ -556,9 +576,11 @@ def convert_honeydew_to_osi(workspace_dir: str) -> str: if osi_metrics: sm["metrics"] = osi_metrics + extra_vendors = ws_osi_meta.get("vendors") or [] + vendors = [HONEYDEW_VENDOR] + [v for v in extra_vendors if v != HONEYDEW_VENDOR] root: dict[str, Any] = { "version": SUPPORTED_OSI_VERSION, - "vendors": [HONEYDEW_VENDOR], + "vendors": vendors, "semantic_model": [sm], } return _dump(root) @@ -860,6 +882,7 @@ def _build_osi_metadata( ai_context: Any = None, unique_keys: Any = None, custom_extensions: list | None = None, + extra_vendors: list[str] | None = None, ) -> dict[str, Any] | None: """Build a Honeydew metadata entry that stores OSI-only fields for round-tripping.""" items: list[dict[str, Any]] = [] @@ -871,6 +894,8 @@ def _build_osi_metadata( items.append({"name": "unique_keys", "value": json.dumps(unique_keys)}) if custom_extensions: items.append({"name": "custom_extensions", "value": json.dumps(custom_extensions)}) + if extra_vendors: + items.append({"name": "vendors", "value": json.dumps(extra_vendors)}) if not items: return None @@ -891,7 +916,7 @@ def _read_osi_metadata(obj: dict[str, Any]) -> dict[str, Any]: result[key] = json.loads(raw) except (json.JSONDecodeError, TypeError): result[key] = raw - elif key in ("unique_keys", "custom_extensions"): + elif key in ("unique_keys", "custom_extensions", "vendors"): try: result[key] = json.loads(raw) except (json.JSONDecodeError, TypeError): diff --git a/converters/honeydew/tests/test_honeydew_osi_converter.py b/converters/honeydew/tests/test_honeydew_osi_converter.py index 810661e..f46fb2f 100644 --- a/converters/honeydew/tests/test_honeydew_osi_converter.py +++ b/converters/honeydew/tests/test_honeydew_osi_converter.py @@ -15,6 +15,7 @@ _assign_metrics_to_entities, _build_osi_metadata, _check_safe_path, + _fields_to_honeydew, _find_entity_in_expression, _honeydew_datatype_to_osi_dimension, _is_simple_identifier, @@ -1106,3 +1107,149 @@ def test_malformed_osi_metadata_json_warns(tmp_path): warnings.simplefilter("always") convert_honeydew_to_osi(str(tmp_path)) assert any("unique_keys" in str(x.message) for x in w) + + +# ───────────────────────────────────────────────────────────────────────────── +# _fields_to_honeydew unit tests +# ───────────────────────────────────────────────────────────────────────────── + +def test_fields_to_honeydew_simple_identifier_goes_to_dataset(): + fields = [{"name": "status", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, + "dimension": {"is_time": False}}] + dataset_attrs, calc_attrs = _fields_to_honeydew(fields, "orders") + assert len(dataset_attrs) == 1 and len(calc_attrs) == 0 + assert dataset_attrs[0]["column"] == "status" + + +def test_fields_to_honeydew_complex_sql_goes_to_calc(): + fields = [{"name": "disc", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "price * 0.9"}]}}] + dataset_attrs, calc_attrs = _fields_to_honeydew(fields, "orders") + assert len(dataset_attrs) == 0 and len(calc_attrs) == 1 + assert calc_attrs[0]["sql"] == "price * 0.9" + + +def test_fields_to_honeydew_missing_name_raises(): + fields = [{"expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "col"}]}}] + with pytest.raises(HoneydewConversionError, match="missing 'name'"): + _fields_to_honeydew(fields, "orders") + + +def test_fields_to_honeydew_empty_expression_skipped(): + fields = [{"name": "bad", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": ""}]}}] + dataset_attrs, calc_attrs = _fields_to_honeydew(fields, "orders") + assert dataset_attrs == [] and calc_attrs == [] + + +# ───────────────────────────────────────────────────────────────────────────── +# Connectionless relation warning +# ───────────────────────────────────────────────────────────────────────────── + +def test_connectionless_relation_warns(): + model = {"name": "m", "datasets": [ + {"name": "orders", "source": "db.s.orders", "fields": []}, + {"name": "customers", "source": "db.s.customers", "fields": []}, + ], "relationships": [{"name": "r", "from": "orders", "to": "customers"}]} + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + files = convert_osi_to_honeydew(_osi(model)) + assert any("resolve the join" in str(x.message) for x in w) + entity = yaml.safe_load(files["schema/orders/orders.yml"]) + assert entity["relations"][0]["target_entity"] == "customers" + + +# ───────────────────────────────────────────────────────────────────────────── +# vendors round-trip +# ───────────────────────────────────────────────────────────────────────────── + +def test_vendors_roundtrip_preserves_non_honeydew(tmp_path): + doc = yaml.dump({ + "version": OSI_VERSION, + "vendors": ["SNOWFLAKE", "HONEYDEW"], + "semantic_model": [{"name": "m", "datasets": []}], + }) + files = convert_osi_to_honeydew(doc) + for rel_path, content in files.items(): + p = tmp_path / rel_path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + assert "SNOWFLAKE" in result["vendors"] + assert "HONEYDEW" in result["vendors"] + + +def test_vendors_always_includes_honeydew(tmp_path): + doc = yaml.dump({ + "version": OSI_VERSION, + "vendors": ["SNOWFLAKE"], + "semantic_model": [{"name": "m", "datasets": []}], + }) + files = convert_osi_to_honeydew(doc) + for rel_path, content in files.items(): + p = tmp_path / rel_path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + assert result["vendors"][0] == "HONEYDEW" + + +# ───────────────────────────────────────────────────────────────────────────── +# main() CLI smoke tests +# ───────────────────────────────────────────────────────────────────────────── + +def test_main_osi_to_honeydew(tmp_path): + import subprocess, sys + input_file = tmp_path / "model.yaml" + input_file.write_text(yaml.dump({ + "version": OSI_VERSION, + "semantic_model": [{"name": "m", "datasets": [ + {"name": "orders", "source": "db.s.orders", "fields": []} + ]}], + })) + output_dir = tmp_path / "out" + result = subprocess.run( + [sys.executable, str(Path(__file__).resolve().parent.parent / "src" / "honeydew_osi_converter.py"), + "osi-to-honeydew", "-i", str(input_file), "-o", str(output_dir)], + capture_output=True, text=True, + ) + assert result.returncode == 0 + assert (output_dir / "workspace.yml").exists() + ws = yaml.safe_load((output_dir / "workspace.yml").read_text()) + assert ws["name"] == "m" + + +def test_main_honeydew_to_osi(tmp_path): + import subprocess, sys + _write_workspace(str(tmp_path), "ws", [{ + "name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", "dataset_attrs": [], + }]) + output_file = tmp_path / "output.yaml" + result = subprocess.run( + [sys.executable, str(Path(__file__).resolve().parent.parent / "src" / "honeydew_osi_converter.py"), + "honeydew-to-osi", "-i", str(tmp_path), "-o", str(output_file)], + capture_output=True, text=True, + ) + assert result.returncode == 0 + assert output_file.exists() + doc = yaml.safe_load(output_file.read_text()) + assert doc["semantic_model"][0]["name"] == "ws" + + +def test_main_path_traversal_rejected(tmp_path): + import subprocess, sys + # Entity name containing traversal sequences generates paths that escape output_dir + input_file = tmp_path / "model.yaml" + input_file.write_text( + f"version: '{OSI_VERSION}'\nsemantic_model:\n" + " - name: m\n datasets:\n" + " - name: '../../evil'\n source: db.s.evil\n fields: []\n" + ) + output_dir = tmp_path / "out" + result = subprocess.run( + [sys.executable, str(Path(__file__).resolve().parent.parent / "src" / "honeydew_osi_converter.py"), + "osi-to-honeydew", "-i", str(input_file), "-o", str(output_dir)], + capture_output=True, text=True, + ) + assert result.returncode == 1 + assert "refusing to write" in result.stderr + assert not (tmp_path / "evil.yml").exists() From 58495559a6c2407cd1572ad87dcf371a43905055 Mon Sep 17 00:00:00 2001 From: Baruch Oxman Date: Wed, 27 May 2026 18:56:27 +0300 Subject: [PATCH 22/89] Rewrite tests to be fully parametrized with complete output verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapse 13 individual OSI→Honeydew tests, 7 Honeydew→OSI tests, 10 OSI round-trip tests, and 13 Honeydew round-trip tests into four @pytest.mark.parametrize blocks. Every assertion now compares the entire output dict rather than cherry-picked fields. Co-Authored-By: Claude Sonnet 4.6 --- .../tests/test_honeydew_osi_converter.py | 1442 ++++++++++------- 1 file changed, 815 insertions(+), 627 deletions(-) diff --git a/converters/honeydew/tests/test_honeydew_osi_converter.py b/converters/honeydew/tests/test_honeydew_osi_converter.py index f46fb2f..46e1e7a 100644 --- a/converters/honeydew/tests/test_honeydew_osi_converter.py +++ b/converters/honeydew/tests/test_honeydew_osi_converter.py @@ -323,183 +323,227 @@ def test_check_safe_path(rel_path, expected): # ───────────────────────────────────────────────────────────────────────────── -# OSI → Honeydew integration tests +# OSI → Honeydew: file content # ───────────────────────────────────────────────────────────────────────────── -def test_osi_to_honeydew_workspace_yml(): - files = convert_osi_to_honeydew(_osi(_minimal_model())) - ws = yaml.safe_load(files["workspace.yml"]) - assert ws["name"] == "test_model" and ws["type"] == "workspace" - - -def test_osi_to_honeydew_entity_yml(): - files = convert_osi_to_honeydew(_osi(_minimal_model())) - entity = yaml.safe_load(files["schema/orders/orders.yml"]) - assert entity["name"] == "orders" - assert entity["keys"] == ["order_id"] - assert entity["key_dataset"] == "orders" - - -def test_osi_to_honeydew_dataset_yml(): - files = convert_osi_to_honeydew(_osi(_minimal_model())) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - assert ds["sql"] == "db.schema.orders" - assert ds["dataset_type"] == "table" - - -def test_osi_to_honeydew_simple_fields_become_dataset_attributes(): - files = convert_osi_to_honeydew(_osi(_minimal_model())) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - names = [a["name"] for a in ds["attributes"]] - assert "order_id" in names and "order_date" in names and "total" in names - - -@pytest.mark.parametrize("field_name,expected_dt", [ - ("order_date", "timestamp"), - ("total", "number"), -]) -def test_osi_to_honeydew_field_datatypes(field_name, expected_dt): - files = convert_osi_to_honeydew(_osi(_minimal_model())) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - attrs = {a["name"]: a for a in ds["attributes"]} - assert attrs[field_name]["datatype"] == expected_dt - - -def test_osi_to_honeydew_complex_expression_becomes_calculated_attribute(): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ - "name": "disc_price", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "price * (1 - discount)"}]}, - "dimension": {"is_time": False}, - }]}]} - files = convert_osi_to_honeydew(_osi(model)) - assert "schema/orders/attributes/disc_price.yml" in files - calc = yaml.safe_load(files["schema/orders/attributes/disc_price.yml"]) - assert calc["type"] == "calculated_attribute" - assert calc["sql"] == "price * (1 - discount)" - - -def test_osi_to_honeydew_label_mapped_to_honeydew_labels(): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ - "name": "status", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, - "dimension": {"is_time": False}, - "label": "sales", - }]}]} - files = convert_osi_to_honeydew(_osi(model)) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - attrs = {a["name"]: a for a in ds["attributes"]} - assert "sales" in attrs["status"]["labels"] - - -def test_osi_to_honeydew_ai_context_string_merged_into_description(): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ - "name": "total", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, - "description": "Base desc", - "ai_context": "revenue, earnings", - }]}]} - files = convert_osi_to_honeydew(_osi(model)) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - attrs = {a["name"]: a for a in ds["attributes"]} - assert "revenue, earnings" in attrs["total"]["description"] - - -def test_osi_to_honeydew_ai_context_dict_instructions_merged_into_description(): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ - "name": "total", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, - "ai_context": {"instructions": "Use for revenue", "synonyms": ["rev", "earnings"]}, - }]}]} - files = convert_osi_to_honeydew(_osi(model)) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - attrs = {a["name"]: a for a in ds["attributes"]} - assert "Use for revenue" in attrs["total"]["description"] - assert "rev" in attrs["total"]["labels"] - - -def test_osi_to_honeydew_ai_context_dict_stored_in_metadata(): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ - "name": "total", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, - "ai_context": {"instructions": "Use for revenue", "synonyms": ["rev"]}, - }]}]} - files = convert_osi_to_honeydew(_osi(model)) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - attr = next(a for a in ds["attributes"] if a["name"] == "total") - osi_section = next((s for s in attr.get("metadata", []) if s["name"] == "osi"), None) - assert osi_section is not None - assert any(i["name"] == "ai_context" for i in osi_section["metadata"]) - - -def test_osi_to_honeydew_unique_keys_stored_in_entity_metadata(): - model = {"name": "m", "datasets": [{"name": "items", "source": "db.s.items", - "primary_key": ["item_id"], - "unique_keys": [["sku"], ["item_id", "variant"]], - "fields": []}]} - files = convert_osi_to_honeydew(_osi(model)) - entity = yaml.safe_load(files["schema/items/items.yml"]) - osi_section = next((s for s in entity.get("metadata", []) if s["name"] == "osi"), None) - assert osi_section is not None - uk_item = next((i for i in osi_section["metadata"] if i["name"] == "unique_keys"), None) - assert uk_item is not None - assert json.loads(uk_item["value"]) == [["sku"], ["item_id", "variant"]] - - -def test_osi_to_honeydew_non_honeydew_extensions_stored_in_metadata(): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", - "custom_extensions": [{"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "WH"}'}], - "fields": []}]} - files = convert_osi_to_honeydew(_osi(model)) - entity = yaml.safe_load(files["schema/orders/orders.yml"]) - osi_section = next((s for s in entity.get("metadata", []) if s["name"] == "osi"), None) - assert osi_section is not None - ext_item = next((i for i in osi_section["metadata"] if i["name"] == "custom_extensions"), None) - assert ext_item is not None - exts = json.loads(ext_item["value"]) - assert any(e["vendor_name"] == "SNOWFLAKE" for e in exts) - - -def test_osi_to_honeydew_relationship_name_stored_in_relation(): - model = {"name": "m", "datasets": [ +_REL_MODEL = { + "name": "m", + "datasets": [ {"name": "orders", "source": "db.s.orders", "fields": []}, {"name": "customers", "source": "db.s.customers", "fields": []}, - ], "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", - "from_columns": ["cid"], "to_columns": ["id"]}]} - files = convert_osi_to_honeydew(_osi(model)) - entity = yaml.safe_load(files["schema/orders/orders.yml"]) - assert entity["relations"][0]["name"] == "orders_to_customers" - - -def test_osi_to_honeydew_model_ai_context_stored_in_workspace_metadata(): - model = {"name": "m", "datasets": [], - "ai_context": {"instructions": "Use for retail analytics", "synonyms": ["store"]}} - files = convert_osi_to_honeydew(_osi(model)) - ws = yaml.safe_load(files["workspace.yml"]) - assert any(s["name"] == "osi" for s in ws.get("metadata", [])) - - -def test_osi_to_honeydew_relationship_on_from_entity_only(): - model = {"name": "m", "datasets": [ - {"name": "orders", "source": "db.s.orders", "fields": []}, - {"name": "customers", "source": "db.s.customers", "fields": []}, - ], "relationships": [{"name": "r", "from": "orders", "to": "customers", - "from_columns": ["cid"], "to_columns": ["id"]}]} - files = convert_osi_to_honeydew(_osi(model)) - orders = yaml.safe_load(files["schema/orders/orders.yml"]) - customers = yaml.safe_load(files["schema/customers/customers.yml"]) - assert len(orders["relations"]) == 1 - assert customers["relations"] == [] - rel = orders["relations"][0] - assert rel["target_entity"] == "customers" and rel["rel_type"] == "many-to-one" - assert rel["connection"] == [{"src_field": "cid", "target_field": "id"}] - - -def test_osi_to_honeydew_metric_assigned_by_expression_entity(): - model = {"name": "m", - "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], - "metrics": [{"name": "total", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}]} + ], + "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", + "from_columns": ["cid"], "to_columns": ["id"]}], +} + +@pytest.mark.parametrize("model,path,expected", [ + # ── minimal model ────────────────────────────────────────────────────────── + pytest.param( + _minimal_model(), + "workspace.yml", + {"type": "workspace", "name": "test_model"}, + id="minimal-workspace", + ), + pytest.param( + _minimal_model(), + "schema/orders/orders.yml", + {"type": "entity", "name": "orders", "keys": ["order_id"], + "key_dataset": "orders", "relations": []}, + id="minimal-entity", + ), + pytest.param( + _minimal_model(), + "schema/orders/datasets/orders.yml", + { + "type": "dataset", "entity": "orders", "name": "orders", + "sql": "db.schema.orders", "dataset_type": "table", + "attributes": [ + {"column": "order_id", "name": "order_id", "datatype": "string"}, + {"column": "order_date", "name": "order_date", "datatype": "timestamp"}, + {"column": "total_amount", "name": "total", "datatype": "number"}, + ], + }, + id="minimal-dataset", + ), + # ── complex expression → calculated attribute ────────────────────────────── + pytest.param( + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "disc_price", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "price * (1 - discount)"}]}, + "dimension": {"is_time": False}, + }]}]}, + "schema/orders/attributes/disc_price.yml", + {"type": "calculated_attribute", "entity": "orders", "name": "disc_price", + "datatype": "string", "sql": "price * (1 - discount)"}, + id="calc-attr-file", + ), + pytest.param( + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "disc_price", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "price * (1 - discount)"}]}, + "dimension": {"is_time": False}, + }]}]}, + "schema/orders/datasets/orders.yml", + {"type": "dataset", "entity": "orders", "name": "orders", + "sql": "db.s.orders", "dataset_type": "table", "attributes": []}, + id="calc-attr-dataset-empty", + ), + # ── label → labels in attr ──────────────────────────────────────────────── + pytest.param( + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "status", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, + "dimension": {"is_time": False}, + "label": "sales", + }]}]}, + "schema/orders/datasets/orders.yml", + { + "type": "dataset", "entity": "orders", "name": "orders", + "sql": "db.s.orders", "dataset_type": "table", + "attributes": [{"column": "status", "name": "status", + "datatype": "string", "labels": ["sales"]}], + }, + id="label-in-attr", + ), + # ── ai_context string → description merged ──────────────────────────────── + pytest.param( + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "total", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, + "description": "Base desc", + "ai_context": "revenue, earnings", + }]}]}, + "schema/orders/datasets/orders.yml", + { + "type": "dataset", "entity": "orders", "name": "orders", + "sql": "db.s.orders", "dataset_type": "table", + "attributes": [{"column": "total", "name": "total", "datatype": "number", + "description": "Base desc\nrevenue, earnings"}], + }, + id="ai-context-string", + ), + # ── ai_context dict → labels + description + metadata ──────────────────── + pytest.param( + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{ + "name": "total", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, + "ai_context": {"instructions": "Use for revenue", "synonyms": ["rev", "earnings"]}, + }]}]}, + "schema/orders/datasets/orders.yml", + { + "type": "dataset", "entity": "orders", "name": "orders", + "sql": "db.s.orders", "dataset_type": "table", + "attributes": [{ + "column": "total", "name": "total", "datatype": "number", + "description": "Use for revenue", + "labels": ["rev", "earnings"], + "metadata": [{"name": "osi", "metadata": [ + {"name": "ai_context", + "value": '{"instructions": "Use for revenue", "synonyms": ["rev", "earnings"]}'}, + ]}], + }], + }, + id="ai-context-dict", + ), + # ── unique_keys → entity metadata ───────────────────────────────────────── + pytest.param( + {"name": "m", "datasets": [{"name": "items", "source": "db.s.items", + "primary_key": ["item_id"], + "unique_keys": [["sku"], ["item_id", "variant"]], + "fields": []}]}, + "schema/items/items.yml", + { + "type": "entity", "name": "items", "keys": ["item_id"], + "key_dataset": "items", "relations": [], + "metadata": [{"name": "osi", "metadata": [ + {"name": "unique_keys", "value": '[["sku"], ["item_id", "variant"]]'}, + ]}], + }, + id="unique-keys", + ), + # ── non-HONEYDEW custom_extensions → entity metadata ────────────────────── + pytest.param( + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "custom_extensions": [{"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "WH"}'}], + "fields": []}]}, + "schema/orders/orders.yml", + { + "type": "entity", "name": "orders", "key_dataset": "orders", "relations": [], + "metadata": [{"name": "osi", "metadata": [ + {"name": "custom_extensions", + "value": '[{"vendor_name": "SNOWFLAKE", "data": "{\\"warehouse\\": \\"WH\\"}"}]'}, + ]}], + }, + id="custom-ext", + ), + # ── relationship on from-entity; nothing on to-entity ──────────────────── + pytest.param( + _REL_MODEL, + "schema/orders/orders.yml", + { + "type": "entity", "name": "orders", "key_dataset": "orders", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "name": "orders_to_customers", + "connection": [{"src_field": "cid", "target_field": "id"}]}], + }, + id="relation-from-entity", + ), + pytest.param( + _REL_MODEL, + "schema/customers/customers.yml", + {"type": "entity", "name": "customers", "key_dataset": "customers", "relations": []}, + id="relation-to-entity-empty", + ), + # ── model-level ai_context → workspace metadata ─────────────────────────── + pytest.param( + {"name": "m", "datasets": [], + "ai_context": {"instructions": "Use for retail analytics", "synonyms": ["store"]}}, + "workspace.yml", + { + "type": "workspace", "name": "m", + "metadata": [{"name": "osi", "metadata": [ + {"name": "ai_context", + "value": '{"instructions": "Use for retail analytics", "synonyms": ["store"]}'}, + ]}], + }, + id="model-ai-context", + ), + # ── metric ──────────────────────────────────────────────────────────────── + pytest.param( + {"name": "m", + "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], + "metrics": [{"name": "total_rev", "description": "Sum of sales", + "expression": {"dialects": [{"dialect": "ANSI_SQL", + "expression": "SUM(orders.total)"}]}}]}, + "schema/orders/metrics/total_rev.yml", + {"type": "metric", "entity": "orders", "name": "total_rev", + "datatype": "number", "sql": "SUM(orders.total)", "description": "Sum of sales"}, + id="metric", + ), + # ── subquery source → dataset_type sql ─────────────────────────────────── + pytest.param( + {"name": "m", "datasets": [{"name": "orders", + "source": "SELECT * FROM raw.orders WHERE active = true", "fields": []}]}, + "schema/orders/datasets/orders.yml", + {"type": "dataset", "entity": "orders", "name": "orders", + "sql": "SELECT * FROM raw.orders WHERE active = true", + "dataset_type": "sql", "attributes": []}, + id="subquery-source", + ), + # ── composite primary key ───────────────────────────────────────────────── + pytest.param( + {"name": "m", "datasets": [{"name": "li", "source": "db.s.li", + "primary_key": ["order_id", "line_number"], "fields": []}]}, + "schema/li/li.yml", + {"type": "entity", "name": "li", "keys": ["order_id", "line_number"], + "key_dataset": "li", "relations": []}, + id="composite-pk", + ), +]) +def test_osi_to_honeydew_file_content(model, path, expected): files = convert_osi_to_honeydew(_osi(model)) - assert "schema/orders/metrics/total.yml" in files + assert path in files + assert yaml.safe_load(files[path]) == expected def test_osi_to_honeydew_metric_entity_hint_overrides_expression(): @@ -528,22 +572,6 @@ def test_osi_to_honeydew_missing_semantic_model_raises(): convert_osi_to_honeydew(f"version: '{OSI_VERSION}'\n") -def test_osi_to_honeydew_subquery_source_uses_sql_type(): - model = {"name": "m", "datasets": [{"name": "orders", - "source": "SELECT * FROM raw.orders WHERE active = true", "fields": []}]} - files = convert_osi_to_honeydew(_osi(model)) - ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - assert ds["dataset_type"] == "sql" - - -def test_osi_to_honeydew_composite_primary_key(): - model = {"name": "m", "datasets": [{"name": "li", "source": "db.s.li", - "primary_key": ["order_id", "line_number"], "fields": []}]} - files = convert_osi_to_honeydew(_osi(model)) - entity = yaml.safe_load(files["schema/li/li.yml"]) - assert entity["keys"] == ["order_id", "line_number"] - - def test_osi_to_honeydew_multiple_models_warns(): doc = yaml.dump({"version": OSI_VERSION, "semantic_model": [ {"name": "m1", "datasets": []}, @@ -553,82 +581,207 @@ def test_osi_to_honeydew_multiple_models_warns(): warnings.simplefilter("always") files = convert_osi_to_honeydew(doc) assert any("only the first" in str(x.message) for x in w) - assert yaml.safe_load(files["workspace.yml"])["name"] == "m1" + assert yaml.safe_load(files["workspace.yml"]) == {"type": "workspace", "name": "m1"} # ───────────────────────────────────────────────────────────────────────────── -# Honeydew → OSI integration tests +# Honeydew → OSI: full document # ───────────────────────────────────────────────────────────────────────────── -def test_honeydew_to_osi_basic(tmp_path): - _write_workspace(str(tmp_path), "tpch", [{ - "name": "orders", "keys": ["orderkey"], "key_dataset": "tpch_orders", - "sql": "SNOWFLAKE_SAMPLE_DATA.TPCH_SF1.ORDERS", - "dataset_attrs": [ - {"column": "o_orderkey", "name": "orderkey", "datatype": "number"}, - {"column": "o_orderdate", "name": "orderdate", "datatype": "date"}, - ], - }]) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - sm = result["semantic_model"][0] - assert sm["name"] == "tpch" - ds = sm["datasets"][0] - assert ds["source"] == "SNOWFLAKE_SAMPLE_DATA.TPCH_SF1.ORDERS" - assert ds["primary_key"] == ["orderkey"] +def _hd_root(sm): + return {"version": OSI_VERSION, "vendors": ["HONEYDEW"], "semantic_model": [sm]} + + +def _ansi(expr): + return {"dialects": [{"dialect": "ANSI_SQL", "expression": expr}]} -@pytest.mark.parametrize("col_name,datatype,expected_dim", [ - ("id", "number", None), - ("status", "string", {"is_time": False}), - ("created_at", "timestamp", {"is_time": True}), +@pytest.mark.parametrize("ws_name,entities,expected_root", [ + # ── basic entity with two dataset attributes ────────────────────────────── + pytest.param( + "tpch", + [{"name": "orders", "keys": ["orderkey"], "key_dataset": "tpch_orders", + "sql": "SNOWFLAKE_SAMPLE_DATA.TPCH_SF1.ORDERS", + "dataset_attrs": [ + {"column": "o_orderkey", "name": "orderkey", "datatype": "number"}, + {"column": "o_orderdate", "name": "orderdate", "datatype": "date"}, + ]}], + _hd_root({ + "name": "tpch", + "datasets": [{ + "name": "orders", + "source": "SNOWFLAKE_SAMPLE_DATA.TPCH_SF1.ORDERS", + "primary_key": ["orderkey"], + "fields": [ + {"name": "orderkey", "expression": _ansi("o_orderkey")}, + {"name": "orderdate", "expression": _ansi("o_orderdate"), + "dimension": {"is_time": True}}, + ], + }], + }), + id="basic", + ), + # ── field types ─────────────────────────────────────────────────────────── + pytest.param( + "ws", + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "db.s.orders", + "dataset_attrs": [{"column": "id", "name": "id", "datatype": "number"}]}], + _hd_root({"name": "ws", "datasets": [{ + "name": "orders", "source": "db.s.orders", "primary_key": ["id"], + "fields": [{"name": "id", "expression": _ansi("id")}], + }]}), + id="field-number", + ), + pytest.param( + "ws", + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "db.s.orders", + "dataset_attrs": [{"column": "status", "name": "status", "datatype": "string"}]}], + _hd_root({"name": "ws", "datasets": [{ + "name": "orders", "source": "db.s.orders", "primary_key": ["id"], + "fields": [{"name": "status", "expression": _ansi("status"), + "dimension": {"is_time": False}}], + }]}), + id="field-string", + ), + pytest.param( + "ws", + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "db.s.orders", + "dataset_attrs": [{"column": "created_at", "name": "created_at", "datatype": "timestamp"}]}], + _hd_root({"name": "ws", "datasets": [{ + "name": "orders", "source": "db.s.orders", "primary_key": ["id"], + "fields": [{"name": "created_at", "expression": _ansi("created_at"), + "dimension": {"is_time": True}}], + }]}), + id="field-timestamp", + ), + # ── labels → label + ai_context + custom_extension ─────────────────────── + pytest.param( + "ws", + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "db.s.orders", + "dataset_attrs": [{"column": "status", "name": "status", "datatype": "string", + "labels": ["sales", "reporting"]}]}], + _hd_root({"name": "ws", "datasets": [{ + "name": "orders", "source": "db.s.orders", "primary_key": ["id"], + "fields": [{ + "name": "status", "expression": _ansi("status"), + "dimension": {"is_time": False}, + "ai_context": {"synonyms": ["sales", "reporting"]}, + "label": "sales", + "custom_extensions": [ + {"vendor_name": "HONEYDEW", "data": '{"labels": ["sales", "reporting"]}'}, + ], + }], + }]}), + id="labels", + ), + # ── many-to-one relationship ────────────────────────────────────────────── + pytest.param( + "ws", + [ + {"name": "orders", "keys": ["order_id"], "key_dataset": "orders", "sql": "db.s.orders", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "connection": [{"src_field": "customer_id", "target_field": "id"}]}], + "dataset_attrs": []}, + {"name": "customers", "keys": ["id"], "key_dataset": "customers", + "sql": "db.s.customers", "dataset_attrs": []}, + ], + _hd_root({ + "name": "ws", + "datasets": [ + {"name": "customers", "source": "db.s.customers", "primary_key": ["id"]}, + {"name": "orders", "source": "db.s.orders", "primary_key": ["order_id"]}, + ], + "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", + "from_columns": ["customer_id"], "to_columns": ["id"]}], + }), + id="many-to-one", + ), + # ── one-to-many (direction flipped) ────────────────────────────────────── + pytest.param( + "ws", + [ + {"name": "customers", "keys": ["id"], "key_dataset": "customers", "sql": "db.s.customers", + "relations": [{"target_entity": "orders", "rel_type": "one-to-many", + "connection": [{"src_field": "id", "target_field": "customer_id"}]}], + "dataset_attrs": []}, + {"name": "orders", "keys": ["order_id"], "key_dataset": "orders", + "sql": "db.s.orders", "dataset_attrs": []}, + ], + _hd_root({ + "name": "ws", + "datasets": [ + {"name": "customers", "source": "db.s.customers", "primary_key": ["id"]}, + {"name": "orders", "source": "db.s.orders", "primary_key": ["order_id"]}, + ], + "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", + "from_columns": ["customer_id"], "to_columns": ["id"]}], + }), + id="one-to-many-flipped", + ), + # ── metric ──────────────────────────────────────────────────────────────── + pytest.param( + "ws", + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "db.s.orders", + "dataset_attrs": [], + "metrics": [{"type": "metric", "entity": "orders", "name": "count", + "datatype": "number", "sql": "COUNT(*)"}]}], + _hd_root({"name": "ws", "datasets": [ + {"name": "orders", "source": "db.s.orders", "primary_key": ["id"]}, + ], "metrics": [{ + "name": "count", + "expression": _ansi("COUNT(*)"), + "custom_extensions": [{"vendor_name": "HONEYDEW", "data": '{"entity": "orders"}'}], + }]}), + id="metric", + ), + # ── calculated attribute → OSI field with HONEYDEW extension ───────────── + pytest.param( + "ws", + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "db.s.orders", + "dataset_attrs": [], + "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", + "name": "discounted", "datatype": "number", + "sql": "orders.price * (1 - orders.discount)"}]}], + _hd_root({"name": "ws", "datasets": [{ + "name": "orders", "source": "db.s.orders", "primary_key": ["id"], + "fields": [{ + "name": "discounted", + "expression": _ansi("orders.price * (1 - orders.discount)"), + "custom_extensions": [ + {"vendor_name": "HONEYDEW", + "data": '{"type": "calculated_attribute", "entity": "orders"}'}, + ], + }], + }]}), + id="calc-attr", + ), ]) -def test_honeydew_to_osi_field_types(tmp_path, col_name, datatype, expected_dim): - _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], - "key_dataset": "orders", "sql": "db.s.orders", - "dataset_attrs": [{"column": col_name, "name": col_name, "datatype": datatype}]}]) +def test_honeydew_to_osi_output(tmp_path, ws_name, entities, expected_root): + _write_workspace(str(tmp_path), ws_name, entities) result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - fields = {f["name"]: f for f in result["semantic_model"][0]["datasets"][0]["fields"]} - assert fields[col_name].get("dimension") == expected_dim + assert result == expected_root -def test_honeydew_to_osi_labels_become_label_and_ai_context(tmp_path): - _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], - "key_dataset": "orders", "sql": "db.s.orders", - "dataset_attrs": [ - {"column": "status", "name": "status", "datatype": "string", - "labels": ["sales", "reporting"]}, - ]}]) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - f = next(f for f in result["semantic_model"][0]["datasets"][0]["fields"] if f["name"] == "status") - assert f["label"] == "sales" - assert "sales" in (f.get("ai_context") or {}).get("synonyms", []) +def test_honeydew_to_osi_missing_workspace_raises(tmp_path): + with pytest.raises(HoneydewConversionError, match="workspace.yml"): + convert_honeydew_to_osi(str(tmp_path)) -def test_honeydew_to_osi_many_to_one_relation(tmp_path): - _write_workspace(str(tmp_path), "ws", [ - {"name": "orders", "keys": ["order_id"], "key_dataset": "orders", "sql": "db.s.orders", - "relations": [{"target_entity": "customers", "rel_type": "many-to-one", - "connection": [{"src_field": "customer_id", "target_field": "id"}]}], - "dataset_attrs": []}, - {"name": "customers", "keys": ["id"], "key_dataset": "customers", "sql": "db.s.customers", "dataset_attrs": []}, - ]) +def test_honeydew_to_osi_missing_schema_dir_empty_model(tmp_path): + (tmp_path / "workspace.yml").write_text(yaml.dump({"type": "workspace", "name": "ws"})) result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - rels = result["semantic_model"][0]["relationships"] - assert len(rels) == 1 - assert rels[0]["from"] == "orders" and rels[0]["to"] == "customers" + assert result == {"version": OSI_VERSION, "vendors": ["HONEYDEW"], + "semantic_model": [{"name": "ws", "datasets": []}]} -def test_honeydew_to_osi_one_to_many_direction_flipped(tmp_path): - _write_workspace(str(tmp_path), "ws", [ - {"name": "customers", "keys": ["id"], "key_dataset": "customers", "sql": "db.s.customers", - "relations": [{"target_entity": "orders", "rel_type": "one-to-many", - "connection": [{"src_field": "id", "target_field": "customer_id"}]}], - "dataset_attrs": []}, - {"name": "orders", "keys": ["order_id"], "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": []}, - ]) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - rel = result["semantic_model"][0]["relationships"][0] - assert rel["from"] == "orders" and rel["to"] == "customers" +def test_honeydew_to_osi_empty_metric_sql_skipped(tmp_path): + _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], + "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], + "metrics": [{"type": "metric", "entity": "orders", "name": "bad", + "datatype": "number", "sql": ""}]}]) + with warnings.catch_warnings(record=True): + result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) + assert "metrics" not in result["semantic_model"][0] def test_honeydew_to_osi_duplicate_relations_deduplicated(tmp_path): @@ -646,179 +799,129 @@ def test_honeydew_to_osi_duplicate_relations_deduplicated(tmp_path): assert len(result["semantic_model"][0].get("relationships", [])) == 1 -def test_honeydew_to_osi_metric_converted(tmp_path): - _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], - "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], - "metrics": [{"type": "metric", "entity": "orders", "name": "count", - "datatype": "number", "sql": "COUNT(*)"}]}]) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - m = result["semantic_model"][0]["metrics"][0] - assert m["name"] == "count" - assert m["expression"]["dialects"][0]["expression"] == "COUNT(*)" - - -def test_honeydew_to_osi_metric_entity_preserved_in_extension(tmp_path): - _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], - "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], - "metrics": [{"type": "metric", "entity": "orders", "name": "cnt", - "datatype": "number", "sql": "COUNT(*)"}]}]) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - m = result["semantic_model"][0]["metrics"][0] - ext = m["custom_extensions"][0] - assert ext["vendor_name"] == "HONEYDEW" - assert json.loads(ext["data"])["entity"] == "orders" - - -def test_honeydew_to_osi_calculated_attribute_as_field(tmp_path): - _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], - "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], - "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", - "name": "discounted", "datatype": "number", - "sql": "orders.price * (1 - orders.discount)"}]}]) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - fields = {f["name"]: f for f in result["semantic_model"][0]["datasets"][0]["fields"]} - assert "discounted" in fields - assert "orders.price" in fields["discounted"]["expression"]["dialects"][0]["expression"] - - -def test_honeydew_to_osi_missing_workspace_raises(tmp_path): - with pytest.raises(HoneydewConversionError, match="workspace.yml"): - convert_honeydew_to_osi(str(tmp_path)) - - -def test_honeydew_to_osi_missing_schema_dir_empty_model(tmp_path): - (tmp_path / "workspace.yml").write_text(yaml.dump({"type": "workspace", "name": "ws"})) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - assert result["semantic_model"][0]["datasets"] == [] - - -def test_honeydew_to_osi_vendors_includes_honeydew(tmp_path): - (tmp_path / "workspace.yml").write_text(yaml.dump({"type": "workspace", "name": "ws"})) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - assert "HONEYDEW" in result.get("vendors", []) - - -def test_honeydew_to_osi_empty_metric_sql_skipped(tmp_path): - _write_workspace(str(tmp_path), "ws", [{"name": "orders", "keys": ["id"], - "key_dataset": "orders", "sql": "db.s.orders", "dataset_attrs": [], - "metrics": [{"type": "metric", "entity": "orders", "name": "bad", - "datatype": "number", "sql": ""}]}]) - with warnings.catch_warnings(record=True): - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - assert "metrics" not in result["semantic_model"][0] - - # ───────────────────────────────────────────────────────────────────────────── -# OSI → Honeydew → OSI round-trip tests +# OSI → Honeydew → OSI round-trip: full semantic model # ───────────────────────────────────────────────────────────────────────────── -def test_osi_roundtrip_name_and_description(tmp_path): - model = {"name": "retail", "description": "Retail model", "datasets": []} - sm = _osi_roundtrip(model, tmp_path) - assert sm["name"] == "retail" and sm["description"] == "Retail model" - - -@pytest.mark.parametrize("primary_key", [ - ["order_id"], - ["order_id", "line_no"], +@pytest.mark.parametrize("model,expected_sm", [ + pytest.param( + {"name": "retail", "description": "Retail model", "datasets": []}, + {"name": "retail", "datasets": [], "description": "Retail model"}, + id="name-desc", + ), + pytest.param( + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "primary_key": ["order_id"], "fields": []}]}, + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "primary_key": ["order_id"]}]}, + id="pk-single", + ), + pytest.param( + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "primary_key": ["order_id", "line_no"], "fields": []}]}, + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "primary_key": ["order_id", "line_no"]}]}, + id="pk-composite", + ), + pytest.param( + {"name": "m", "datasets": [{"name": "items", "source": "db.s.items", + "primary_key": ["id"], + "unique_keys": [["sku"], ["id", "variant"]], + "fields": []}]}, + {"name": "m", "datasets": [{"name": "items", "source": "db.s.items", + "primary_key": ["id"], + "unique_keys": [["sku"], ["id", "variant"]]}]}, + id="unique-keys", + ), + pytest.param( + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "fields": [{"name": "status", "label": "sales", + "expression": _ansi("status"), + "dimension": {"is_time": False}}]}]}, + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "fields": [{"name": "status", "expression": _ansi("status"), + "dimension": {"is_time": False}, + "ai_context": {"synonyms": ["sales"]}, + "label": "sales"}]}]}, + id="field-label", + ), + pytest.param( + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "fields": [{"name": "total", + "expression": _ansi("total"), + "ai_context": {"instructions": "Use for revenue analysis", + "synonyms": ["revenue", "sales"]}}]}]}, + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "fields": [{"name": "total", "expression": _ansi("total"), + "description": "Use for revenue analysis", + "ai_context": {"instructions": "Use for revenue analysis", + "synonyms": ["revenue", "sales"]}, + "label": "revenue", + "custom_extensions": [ + {"vendor_name": "HONEYDEW", + "data": '{"labels": ["revenue", "sales"]}'}, + ]}]}]}, + id="ai-context-dict", + ), + pytest.param( + {"name": "m", "ai_context": {"instructions": "Retail analytics", "synonyms": ["store"]}, + "datasets": []}, + {"name": "m", "datasets": [], + "ai_context": {"instructions": "Retail analytics", "synonyms": ["store"]}}, + id="model-ai-context", + ), + pytest.param( + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "custom_extensions": [{"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "WH"}'}], + "fields": []}]}, + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "custom_extensions": [{"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "WH"}'}]}]}, + id="custom-ext", + ), + pytest.param( + {"name": "m", "datasets": [ + {"name": "orders", "source": "db.s.orders", "fields": []}, + {"name": "customers", "source": "db.s.customers", "fields": []}, + ], "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", + "from_columns": ["cid"], "to_columns": ["id"]}]}, + {"name": "m", "datasets": [ + {"name": "customers", "source": "db.s.customers"}, + {"name": "orders", "source": "db.s.orders"}, + ], "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", + "from_columns": ["cid"], "to_columns": ["id"]}]}, + id="relationship", + ), + pytest.param( + {"name": "m", + "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], + "metrics": [{"name": "total_revenue", "description": "Sum of sales", + "expression": _ansi("SUM(orders.total)")}]}, + {"name": "m", + "datasets": [{"name": "orders", "source": "db.s.orders"}], + "metrics": [{"name": "total_revenue", + "expression": _ansi("SUM(orders.total)"), + "custom_extensions": [ + {"vendor_name": "HONEYDEW", "data": '{"entity": "orders"}'}, + ], + "description": "Sum of sales"}]}, + id="metric", + ), + # ai_context string is merged into description and not stored in metadata for fields + pytest.param( + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "fields": [{"name": "status", + "expression": _ansi("status"), + "ai_context": "order status, order state", + "dimension": {"is_time": False}}]}]}, + {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", + "fields": [{"name": "status", "expression": _ansi("status"), + "dimension": {"is_time": False}, + "description": "order status, order state"}]}]}, + id="ai-context-string-becomes-desc", + ), ]) -def test_osi_roundtrip_primary_key(tmp_path, primary_key): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", - "primary_key": primary_key, "fields": []}]} - sm = _osi_roundtrip(model, tmp_path) - assert sm["datasets"][0]["primary_key"] == primary_key - - -def test_osi_roundtrip_unique_keys(tmp_path): - model = {"name": "m", "datasets": [{"name": "items", "source": "db.s.items", - "primary_key": ["id"], - "unique_keys": [["sku"], ["id", "variant"]], - "fields": []}]} - sm = _osi_roundtrip(model, tmp_path) - assert sm["datasets"][0]["unique_keys"] == [["sku"], ["id", "variant"]] - - -def test_osi_roundtrip_field_label(tmp_path): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", - "fields": [{"name": "status", "label": "sales", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, - "dimension": {"is_time": False}}]}]} - sm = _osi_roundtrip(model, tmp_path) - f = next(f for f in sm["datasets"][0]["fields"] if f["name"] == "status") - assert f["label"] == "sales" - - -def test_osi_roundtrip_ai_context_string(tmp_path): - ai_ctx_value = "order status, order state" - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", - "fields": [{"name": "status", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, - "ai_context": ai_ctx_value, - "dimension": {"is_time": False}}]}]} - sm = _osi_roundtrip(model, tmp_path) - f = next(f for f in sm["datasets"][0]["fields"] if f["name"] == "status") - # String ai_context is merged into description on OSI→Honeydew; value must be recoverable - assert ai_ctx_value in (f.get("description") or "") or f.get("ai_context") == ai_ctx_value - - -def test_osi_roundtrip_ai_context_dict(tmp_path): - ctx = {"instructions": "Use for revenue analysis", "synonyms": ["revenue", "sales"]} - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", - "fields": [{"name": "total", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "total"}]}, - "ai_context": ctx}]}]} - sm = _osi_roundtrip(model, tmp_path) - f = next(f for f in sm["datasets"][0]["fields"] if f["name"] == "total") - assert f.get("ai_context") == ctx - - -def test_osi_roundtrip_model_ai_context(tmp_path): - ctx = {"instructions": "Retail analytics", "synonyms": ["store"]} - model = {"name": "m", "ai_context": ctx, "datasets": []} - sm = _osi_roundtrip(model, tmp_path) - assert sm.get("ai_context") == ctx - - -def test_osi_roundtrip_non_honeydew_custom_extensions(tmp_path): - model = {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", - "custom_extensions": [{"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "WH"}'}], - "fields": []}]} - sm = _osi_roundtrip(model, tmp_path) - exts = sm["datasets"][0].get("custom_extensions") or [] - assert any(e["vendor_name"] == "SNOWFLAKE" for e in exts) - - -def test_osi_roundtrip_relationship_name(tmp_path): - model = {"name": "m", "datasets": [ - {"name": "orders", "source": "db.s.orders", "fields": []}, - {"name": "customers", "source": "db.s.customers", "fields": []}, - ], "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", - "from_columns": ["cid"], "to_columns": ["id"]}]} - sm = _osi_roundtrip(model, tmp_path) - assert sm["relationships"][0]["name"] == "orders_to_customers" - - -def test_osi_roundtrip_relationship_columns(tmp_path): - model = {"name": "m", "datasets": [ - {"name": "orders", "source": "db.s.orders", "fields": []}, - {"name": "customers", "source": "db.s.customers", "fields": []}, - ], "relationships": [{"name": "r", "from": "orders", "to": "customers", - "from_columns": ["cid"], "to_columns": ["id"]}]} - sm = _osi_roundtrip(model, tmp_path) - rel = sm["relationships"][0] - assert rel["from_columns"] == ["cid"] and rel["to_columns"] == ["id"] - - -def test_osi_roundtrip_metric(tmp_path): - model = {"name": "m", - "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], - "metrics": [{"name": "total_revenue", "description": "Sum of sales", - "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}]} - sm = _osi_roundtrip(model, tmp_path) - m = sm["metrics"][0] - assert m["name"] == "total_revenue" - assert m["expression"]["dialects"][0]["expression"] == "SUM(orders.total)" - assert m["description"] == "Sum of sales" +def test_osi_roundtrip_sm(tmp_path, model, expected_sm): + assert _osi_roundtrip(model, tmp_path) == expected_sm def test_osi_roundtrip_tpcds_example(tmp_path): @@ -842,172 +945,253 @@ def test_osi_roundtrip_tpcds_example(tmp_path): # ───────────────────────────────────────────────────────────────────────────── -# Honeydew → OSI → Honeydew round-trip tests +# Honeydew → OSI → Honeydew round-trip: full file content # ───────────────────────────────────────────────────────────────────────────── -def test_honeydew_roundtrip_entity_name_and_keys(tmp_path): - out_dir = _honeydew_roundtrip([{ - "name": "orders", "keys": ["order_id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", "dataset_attrs": [], - }], tmp_path) - entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) - assert entity["name"] == "orders" and entity["keys"] == ["order_id"] - - -def test_honeydew_roundtrip_source(tmp_path): - out_dir = _honeydew_roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.SCHEMA.ORDERS", "dataset_attrs": [], - }], tmp_path) - ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) - assert ds["sql"] == "DB.SCHEMA.ORDERS" - - -def test_honeydew_roundtrip_column_attributes(tmp_path): - out_dir = _honeydew_roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", - "dataset_attrs": [ - {"column": "o_id", "name": "id", "datatype": "number"}, - {"column": "o_status", "name": "status", "datatype": "string"}, +@pytest.mark.parametrize("entities,path,expected", [ + # ── entity name + keys ──────────────────────────────────────────────────── + pytest.param( + [{"name": "orders", "keys": ["order_id"], "key_dataset": "orders", + "sql": "DB.S.ORDERS", "dataset_attrs": []}], + "schema/orders/orders.yml", + {"type": "entity", "name": "orders", "keys": ["order_id"], + "key_dataset": "orders", "relations": []}, + id="entity-keys", + ), + # ── dataset source ──────────────────────────────────────────────────────── + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", + "sql": "DB.SCHEMA.ORDERS", "dataset_attrs": []}], + "schema/orders/datasets/orders.yml", + {"type": "dataset", "entity": "orders", "name": "orders", + "sql": "DB.SCHEMA.ORDERS", "dataset_type": "table", "attributes": []}, + id="dataset-source", + ), + # ── column attributes ───────────────────────────────────────────────────── + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [ + {"column": "o_id", "name": "id", "datatype": "number"}, + {"column": "o_status", "name": "status", "datatype": "string"}, + ]}], + "schema/orders/datasets/orders.yml", + { + "type": "dataset", "entity": "orders", "name": "orders", + "sql": "DB.S.ORDERS", "dataset_type": "table", + "attributes": [ + {"column": "o_id", "name": "id", "datatype": "number"}, + {"column": "o_status", "name": "status", "datatype": "string"}, + ], + }, + id="column-attrs", + ), + # ── labels on column ────────────────────────────────────────────────────── + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [{"column": "status", "name": "status", "datatype": "string", + "labels": ["sales"]}]}], + "schema/orders/datasets/orders.yml", + { + "type": "dataset", "entity": "orders", "name": "orders", + "sql": "DB.S.ORDERS", "dataset_type": "table", + "attributes": [{ + "column": "status", "name": "status", "datatype": "string", + "labels": ["sales"], + "metadata": [{"name": "osi", "metadata": [ + {"name": "ai_context", "value": '{"synonyms": ["sales"]}'}, + ]}], + }], + }, + id="labels-on-column", + ), + # ── calculated attribute sql ─────────────────────────────────────────────── + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [], + "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", + "name": "disc", "datatype": "number", + "sql": "orders.price * (1 - orders.discount)"}]}], + "schema/orders/attributes/disc.yml", + {"type": "calculated_attribute", "entity": "orders", "name": "disc", + "datatype": "number", "sql": "orders.price * (1 - orders.discount)"}, + id="calc-attr-sql", + ), + # ── calc attr with simple identifier stays as calc_attr ────────────────── + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [], + "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", + "name": "revenue", "datatype": "number", "sql": "revenue"}]}], + "schema/orders/attributes/revenue.yml", + {"type": "calculated_attribute", "entity": "orders", "name": "revenue", + "datatype": "number", "sql": "revenue"}, + id="calc-simple-stays-calc", + ), + # ── metric entity assignment ────────────────────────────────────────────── + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [], + "metrics": [{"type": "metric", "entity": "orders", "name": "cnt", + "datatype": "number", "sql": "COUNT(*)"}]}], + "schema/orders/metrics/cnt.yml", + {"type": "metric", "entity": "orders", "name": "cnt", + "datatype": "number", "sql": "COUNT(*)"}, + id="metric", + ), + # ── many-to-one relation ────────────────────────────────────────────────── + pytest.param( + [ + {"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "connection": [{"src_field": "cid", "target_field": "id"}]}], + "dataset_attrs": []}, + {"name": "customers", "keys": ["id"], "key_dataset": "customers", + "sql": "DB.S.CUSTOMERS", "dataset_attrs": []}, ], - }], tmp_path) - ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) - attrs = {a["name"]: a for a in ds["attributes"]} - assert attrs["id"]["column"] == "o_id" - assert attrs["status"]["datatype"] == "string" - - -def test_honeydew_roundtrip_labels_on_column(tmp_path): - out_dir = _honeydew_roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", - "dataset_attrs": [{"column": "status", "name": "status", "datatype": "string", "labels": ["sales"]}], - }], tmp_path) - ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) - attrs = {a["name"]: a for a in ds["attributes"]} - assert "sales" in attrs["status"].get("labels", []) - - -def test_honeydew_roundtrip_calculated_attribute_sql(tmp_path): - out_dir = _honeydew_roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", "dataset_attrs": [], - "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", - "name": "disc", "datatype": "number", - "sql": "orders.price * (1 - orders.discount)"}], - }], tmp_path) - calc = yaml.safe_load((out_dir / "schema/orders/attributes/disc.yml").read_text()) - assert calc["sql"] == "orders.price * (1 - orders.discount)" - - -def test_honeydew_roundtrip_metric_entity_assignment(tmp_path): - out_dir = _honeydew_roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", "dataset_attrs": [], - "metrics": [{"type": "metric", "entity": "orders", "name": "cnt", - "datatype": "number", "sql": "COUNT(*)"}], - }], tmp_path) - m = yaml.safe_load((out_dir / "schema/orders/metrics/cnt.yml").read_text()) - assert m["entity"] == "orders" and m["sql"] == "COUNT(*)" - - -def test_honeydew_roundtrip_relation(tmp_path): - out_dir = _honeydew_roundtrip([ - {"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", - "relations": [{"target_entity": "customers", "rel_type": "many-to-one", - "connection": [{"src_field": "cid", "target_field": "id"}]}], - "dataset_attrs": []}, - {"name": "customers", "keys": ["id"], "key_dataset": "customers", - "sql": "DB.S.CUSTOMERS", "dataset_attrs": []}, - ], tmp_path) - entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) - assert entity["relations"][0]["target_entity"] == "customers" - assert entity["relations"][0]["connection"][0]["src_field"] == "cid" - - -def test_honeydew_roundtrip_bool_datatype(tmp_path): - out_dir = _honeydew_roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", - "dataset_attrs": [{"column": "is_active", "name": "is_active", "datatype": "bool"}], - }], tmp_path) - ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) - attrs = {a["name"]: a for a in ds["attributes"]} - assert attrs["is_active"]["datatype"] == "bool" - - -def test_honeydew_roundtrip_connection_expr(tmp_path): - out_dir = _honeydew_roundtrip([ - {"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", - "relations": [{"target_entity": "customers", "rel_type": "many-to-one", - "connection_expr": {"sql": "orders.cid = customers.id AND orders.region = customers.region"}}], - "dataset_attrs": []}, - {"name": "customers", "keys": ["id"], "key_dataset": "customers", - "sql": "DB.S.CUSTOMERS", "dataset_attrs": []}, - ], tmp_path) - entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) - rel = entity["relations"][0] - assert rel.get("connection_expr", {}).get("sql") == "orders.cid = customers.id AND orders.region = customers.region" - - -@pytest.mark.parametrize("attr_extra,check_key,check_val", [ - ({"display_name": "Order Status"}, "display_name", "Order Status"), - ({"hidden": True}, "hidden", True), - ({"format_string": "##,###"}, "format_string", "##,###"), -]) -def test_honeydew_roundtrip_dataset_attr_honeydew_field(tmp_path, attr_extra, check_key, check_val): - attr = {"column": "status", "name": "status", "datatype": "string", **attr_extra} - out_dir = _honeydew_roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", "dataset_attrs": [attr], - }], tmp_path) - ds = yaml.safe_load((out_dir / "schema/orders/datasets/orders.yml").read_text()) - attrs = {a["name"]: a for a in ds["attributes"]} - assert attrs["status"][check_key] == check_val - - -@pytest.mark.parametrize("calc_extra,check_key,check_val", [ - ({"display_name": "Discounted Price"}, "display_name", "Discounted Price"), - ({"timegrain": "day"}, "timegrain", "day"), -]) -def test_honeydew_roundtrip_calc_attr_honeydew_field(tmp_path, calc_extra, check_key, check_val): - calc = {"type": "calculated_attribute", "entity": "orders", - "name": "disc", "datatype": "number", "sql": "orders.price * 0.9", **calc_extra} - out_dir = _honeydew_roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", "dataset_attrs": [], "calc_attrs": [calc], - }], tmp_path) - result = yaml.safe_load((out_dir / "schema/orders/attributes/disc.yml").read_text()) - assert result[check_key] == check_val - - -@pytest.mark.parametrize("entity_extra,check_key,check_val", [ - ({"owner": "analytics_team"}, "owner", "analytics_team"), - ({"display_name": "Orders Table"}, "display_name", "Orders Table"), - ({"hidden": True}, "hidden", True), - ({"folder": "finance"}, "folder", "finance"), + "schema/orders/orders.yml", + { + "type": "entity", "name": "orders", "keys": ["id"], + "key_dataset": "orders", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "name": "orders_to_customers", + "connection": [{"src_field": "cid", "target_field": "id"}]}], + }, + id="relation", + ), + # ── connection_expr round-trip ──────────────────────────────────────────── + pytest.param( + [ + {"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "connection_expr": {"sql": "orders.cid = customers.id AND orders.region = customers.region"}}], + "dataset_attrs": []}, + {"name": "customers", "keys": ["id"], "key_dataset": "customers", + "sql": "DB.S.CUSTOMERS", "dataset_attrs": []}, + ], + "schema/orders/orders.yml", + { + "type": "entity", "name": "orders", "keys": ["id"], + "key_dataset": "orders", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "name": "orders_to_customers", + "connection_expr": {"sql": "orders.cid = customers.id AND orders.region = customers.region"}}], + }, + id="connection-expr", + ), + # ── bool datatype ───────────────────────────────────────────────────────── + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [{"column": "is_active", "name": "is_active", "datatype": "bool"}]}], + "schema/orders/datasets/orders.yml", + { + "type": "dataset", "entity": "orders", "name": "orders", + "sql": "DB.S.ORDERS", "dataset_type": "table", + "attributes": [{"column": "is_active", "name": "is_active", "datatype": "bool"}], + }, + id="bool-datatype", + ), + # ── Honeydew-specific attribute fields ─────────────────────────────────── + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [{"column": "status", "name": "status", "datatype": "string", + "display_name": "Order Status"}]}], + "schema/orders/datasets/orders.yml", + { + "type": "dataset", "entity": "orders", "name": "orders", + "sql": "DB.S.ORDERS", "dataset_type": "table", + "attributes": [{"column": "status", "name": "status", "datatype": "string", + "display_name": "Order Status"}], + }, + id="attr-display-name", + ), + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [{"column": "status", "name": "status", "datatype": "string", + "hidden": True}]}], + "schema/orders/datasets/orders.yml", + { + "type": "dataset", "entity": "orders", "name": "orders", + "sql": "DB.S.ORDERS", "dataset_type": "table", + "attributes": [{"column": "status", "name": "status", "datatype": "string", + "hidden": True}], + }, + id="attr-hidden", + ), + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [{"column": "status", "name": "status", "datatype": "string", + "format_string": "##,###"}]}], + "schema/orders/datasets/orders.yml", + { + "type": "dataset", "entity": "orders", "name": "orders", + "sql": "DB.S.ORDERS", "dataset_type": "table", + "attributes": [{"column": "status", "name": "status", "datatype": "string", + "format_string": "##,###"}], + }, + id="attr-format-string", + ), + # ── Honeydew-specific calc attr fields ──────────────────────────────────── + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [], + "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", + "name": "disc", "datatype": "number", + "sql": "orders.price * 0.9", "display_name": "Discounted Price"}]}], + "schema/orders/attributes/disc.yml", + {"type": "calculated_attribute", "entity": "orders", "name": "disc", + "datatype": "number", "sql": "orders.price * 0.9", "display_name": "Discounted Price"}, + id="calc-display-name", + ), + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [], + "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", + "name": "disc", "datatype": "number", + "sql": "orders.price * 0.9", "timegrain": "day"}]}], + "schema/orders/attributes/disc.yml", + {"type": "calculated_attribute", "entity": "orders", "name": "disc", + "datatype": "number", "sql": "orders.price * 0.9", "timegrain": "day"}, + id="calc-timegrain", + ), + # ── Honeydew-specific entity fields ────────────────────────────────────── + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [], "owner": "analytics_team"}], + "schema/orders/orders.yml", + {"type": "entity", "name": "orders", "keys": ["id"], "key_dataset": "orders", + "owner": "analytics_team", "relations": []}, + id="entity-owner", + ), + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [], "display_name": "Orders Table"}], + "schema/orders/orders.yml", + {"type": "entity", "name": "orders", "keys": ["id"], "key_dataset": "orders", + "display_name": "Orders Table", "relations": []}, + id="entity-display-name", + ), + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [], "hidden": True}], + "schema/orders/orders.yml", + {"type": "entity", "name": "orders", "keys": ["id"], "key_dataset": "orders", + "hidden": True, "relations": []}, + id="entity-hidden", + ), + pytest.param( + [{"name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", + "dataset_attrs": [], "folder": "finance"}], + "schema/orders/orders.yml", + {"type": "entity", "name": "orders", "keys": ["id"], "key_dataset": "orders", + "folder": "finance", "relations": []}, + id="entity-folder", + ), ]) -def test_honeydew_roundtrip_entity_honeydew_field(tmp_path, entity_extra, check_key, check_val): - out_dir = _honeydew_roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", "dataset_attrs": [], **entity_extra, - }], tmp_path) - entity = yaml.safe_load((out_dir / "schema/orders/orders.yml").read_text()) - assert entity.get(check_key) == check_val - - -def test_honeydew_roundtrip_calc_attr_simple_identifier_stays_calc(tmp_path): - out_dir = _honeydew_roundtrip([{ - "name": "orders", "keys": ["id"], "key_dataset": "orders", - "sql": "DB.S.ORDERS", "dataset_attrs": [], - "calc_attrs": [{"type": "calculated_attribute", "entity": "orders", - "name": "revenue", "datatype": "number", "sql": "revenue"}], - }], tmp_path) - calc_path = out_dir / "schema/orders/attributes/revenue.yml" - assert calc_path.exists(), "calculated_attribute with simple-id sql should not become a dataset column" - calc = yaml.safe_load(calc_path.read_text()) - assert calc["sql"] == "revenue" +def test_honeydew_roundtrip_file(tmp_path, entities, path, expected): + out_dir = _honeydew_roundtrip(entities, tmp_path) + p = out_dir / path + assert p.exists(), f"Expected file {path!r} was not generated" + assert yaml.safe_load(p.read_text()) == expected # ───────────────────────────────────────────────────────────────────────────── @@ -1026,7 +1210,8 @@ def test_empty_or_whitespace_field_expression_skipped(expression): }]}]} files = convert_osi_to_honeydew(_osi(model)) ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - assert all(a["name"] != "bad" for a in ds["attributes"]) + assert ds == {"type": "dataset", "entity": "orders", "name": "orders", + "sql": "db.s.orders", "dataset_type": "table", "attributes": []} assert "schema/orders/attributes/bad.yml" not in files @@ -1053,7 +1238,8 @@ def test_non_dict_expression_warns(): files = convert_osi_to_honeydew(_osi(model)) assert any("must be a mapping" in str(x.message) for x in w) ds = yaml.safe_load(files["schema/orders/datasets/orders.yml"]) - assert all(a["name"] != "bad" for a in ds["attributes"]) + assert ds == {"type": "dataset", "entity": "orders", "name": "orders", + "sql": "db.s.orders", "dataset_type": "table", "attributes": []} def test_duplicate_metric_name_warns(): @@ -1067,8 +1253,10 @@ def test_duplicate_metric_name_warns(): warnings.simplefilter("always") files = convert_osi_to_honeydew(_osi(model)) assert any("total" in str(x.message) for x in w) - m = yaml.safe_load(files["schema/orders/metrics/total.yml"]) - assert "orders.b" in m["sql"] + assert yaml.safe_load(files["schema/orders/metrics/total.yml"]) == { + "type": "metric", "entity": "orders", "name": "total", + "datatype": "number", "sql": "SUM(orders.b)", + } def test_metric_string_ai_context_preserved_in_roundtrip(tmp_path): @@ -1076,14 +1264,18 @@ def test_metric_string_ai_context_preserved_in_roundtrip(tmp_path): "datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}], "metrics": [{"name": "rev", "ai_context": "Use for revenue analysis", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}]} - files = convert_osi_to_honeydew(_osi(model)) - for rel_path, content in files.items(): - p = tmp_path / rel_path - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(content) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - m = result["semantic_model"][0]["metrics"][0] - assert m.get("ai_context") == "Use for revenue analysis" + sm = _osi_roundtrip(model, tmp_path) + assert sm == { + "name": "m", + "datasets": [{"name": "orders", "source": "db.s.orders"}], + "metrics": [{ + "name": "rev", + "expression": _ansi("SUM(orders.total)"), + "custom_extensions": [{"vendor_name": "HONEYDEW", "data": '{"entity": "orders"}'}], + "description": "Use for revenue analysis", + "ai_context": "Use for revenue analysis", + }], + } def test_malformed_osi_metadata_json_warns(tmp_path): @@ -1117,15 +1309,16 @@ def test_fields_to_honeydew_simple_identifier_goes_to_dataset(): fields = [{"name": "status", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "status"}]}, "dimension": {"is_time": False}}] dataset_attrs, calc_attrs = _fields_to_honeydew(fields, "orders") - assert len(dataset_attrs) == 1 and len(calc_attrs) == 0 - assert dataset_attrs[0]["column"] == "status" + assert dataset_attrs == [{"column": "status", "name": "status", "datatype": "string"}] + assert calc_attrs == [] def test_fields_to_honeydew_complex_sql_goes_to_calc(): fields = [{"name": "disc", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "price * 0.9"}]}}] dataset_attrs, calc_attrs = _fields_to_honeydew(fields, "orders") - assert len(dataset_attrs) == 0 and len(calc_attrs) == 1 - assert calc_attrs[0]["sql"] == "price * 0.9" + assert dataset_attrs == [] + assert calc_attrs == [{"type": "calculated_attribute", "entity": "orders", "name": "disc", + "datatype": "number", "sql": "price * 0.9"}] def test_fields_to_honeydew_missing_name_raises(): @@ -1153,34 +1346,25 @@ def test_connectionless_relation_warns(): warnings.simplefilter("always") files = convert_osi_to_honeydew(_osi(model)) assert any("resolve the join" in str(x.message) for x in w) - entity = yaml.safe_load(files["schema/orders/orders.yml"]) - assert entity["relations"][0]["target_entity"] == "customers" + assert yaml.safe_load(files["schema/orders/orders.yml"]) == { + "type": "entity", "name": "orders", "key_dataset": "orders", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", "name": "r"}], + } # ───────────────────────────────────────────────────────────────────────────── -# vendors round-trip +# Vendors round-trip # ───────────────────────────────────────────────────────────────────────────── -def test_vendors_roundtrip_preserves_non_honeydew(tmp_path): - doc = yaml.dump({ - "version": OSI_VERSION, - "vendors": ["SNOWFLAKE", "HONEYDEW"], - "semantic_model": [{"name": "m", "datasets": []}], - }) - files = convert_osi_to_honeydew(doc) - for rel_path, content in files.items(): - p = tmp_path / rel_path - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(content) - result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - assert "SNOWFLAKE" in result["vendors"] - assert "HONEYDEW" in result["vendors"] - - -def test_vendors_always_includes_honeydew(tmp_path): +@pytest.mark.parametrize("input_vendors,expected_vendors", [ + (["SNOWFLAKE", "HONEYDEW"], ["HONEYDEW", "SNOWFLAKE"]), + (["SNOWFLAKE"], ["HONEYDEW", "SNOWFLAKE"]), + (["HONEYDEW"], ["HONEYDEW"]), +]) +def test_vendors_roundtrip(tmp_path, input_vendors, expected_vendors): doc = yaml.dump({ "version": OSI_VERSION, - "vendors": ["SNOWFLAKE"], + "vendors": input_vendors, "semantic_model": [{"name": "m", "datasets": []}], }) files = convert_osi_to_honeydew(doc) @@ -1189,7 +1373,8 @@ def test_vendors_always_includes_honeydew(tmp_path): p.parent.mkdir(parents=True, exist_ok=True) p.write_text(content) result = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path))) - assert result["vendors"][0] == "HONEYDEW" + assert result == {"version": OSI_VERSION, "vendors": expected_vendors, + "semantic_model": [{"name": "m", "datasets": []}]} # ───────────────────────────────────────────────────────────────────────────── @@ -1197,7 +1382,7 @@ def test_vendors_always_includes_honeydew(tmp_path): # ───────────────────────────────────────────────────────────────────────────── def test_main_osi_to_honeydew(tmp_path): - import subprocess, sys + import subprocess input_file = tmp_path / "model.yaml" input_file.write_text(yaml.dump({ "version": OSI_VERSION, @@ -1212,13 +1397,13 @@ def test_main_osi_to_honeydew(tmp_path): capture_output=True, text=True, ) assert result.returncode == 0 - assert (output_dir / "workspace.yml").exists() - ws = yaml.safe_load((output_dir / "workspace.yml").read_text()) - assert ws["name"] == "m" + assert yaml.safe_load((output_dir / "workspace.yml").read_text()) == { + "type": "workspace", "name": "m", + } def test_main_honeydew_to_osi(tmp_path): - import subprocess, sys + import subprocess _write_workspace(str(tmp_path), "ws", [{ "name": "orders", "keys": ["id"], "key_dataset": "orders", "sql": "DB.S.ORDERS", "dataset_attrs": [], @@ -1230,14 +1415,17 @@ def test_main_honeydew_to_osi(tmp_path): capture_output=True, text=True, ) assert result.returncode == 0 - assert output_file.exists() - doc = yaml.safe_load(output_file.read_text()) - assert doc["semantic_model"][0]["name"] == "ws" + assert yaml.safe_load(output_file.read_text()) == { + "version": OSI_VERSION, + "vendors": ["HONEYDEW"], + "semantic_model": [{"name": "ws", "datasets": [ + {"name": "orders", "source": "DB.S.ORDERS", "primary_key": ["id"]}, + ]}], + } def test_main_path_traversal_rejected(tmp_path): - import subprocess, sys - # Entity name containing traversal sequences generates paths that escape output_dir + import subprocess input_file = tmp_path / "model.yaml" input_file.write_text( f"version: '{OSI_VERSION}'\nsemantic_model:\n" From 8627fb4d278fb74b406f5ff0c973c814138d9a73 Mon Sep 17 00:00:00 2001 From: Baruch Oxman Date: Wed, 27 May 2026 21:32:28 +0300 Subject: [PATCH 23/89] Address PR review: fix docstring and README inaccuracies - Docstring: relationship name is mapped directly to Honeydew's relation name field (not osi metadata); ai_context is mapped natively to description/labels/AI metadata, not treated as having no equivalent - README mapping table: rename rows to use Honeydew's canonical terms (Source Attribute, Calculated Attribute) and add missing ai_context row - README limitations: rewrite the confusing "One dataset per entity" bullet to clarify that OSI dataset = one table/query, Honeydew supports multiple dataset files per entity but the converter generates exactly one Co-Authored-By: Claude Sonnet 4.6 --- converters/honeydew/README.md | 7 ++++--- converters/honeydew/src/honeydew_osi_converter.py | 10 ++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/converters/honeydew/README.md b/converters/honeydew/README.md index 62e1e8b..ad3158a 100644 --- a/converters/honeydew/README.md +++ b/converters/honeydew/README.md @@ -17,8 +17,9 @@ Bidirectional converter between [OSI](../../core-spec/spec.md) semantic models a | `dataset` | Entity + dataset files under `schema//` | | `dataset.source` | `dataset.sql` | | `dataset.primary_key` | `entity.keys` | -| Simple column field | `dataset.attributes` entry | -| Computed field expression | `calculated_attribute` YAML | +| Simple column field | Source Attribute (`dataset.attributes` entry) | +| Computed field expression | Calculated Attribute (`calculated_attribute` YAML) | +| `field.ai_context` | AI Metadata on the attribute or entity | | `relationship` (from → to) | `entity.relations` on the "from" entity (`rel_type: many-to-one`) | | `metric` | `metric` YAML (assigned to entity by expression parse) | @@ -66,7 +67,7 @@ python -m pytest tests/ ## Limitations -- **One dataset per entity**: The converter maps each OSI dataset to a single Honeydew entity with one source dataset. Multiple datasets per entity are not generated. +- **One source dataset per entity**: Honeydew entities can have multiple source dataset files; the converter always generates exactly one, because an OSI `dataset` block describes a single table or SQL query. - **Datatype inference**: OSI fields have no explicit datatype; the converter infers Honeydew datatypes from the `dimension.is_time` flag (`timestamp`) and the presence/absence of the `dimension` key (`string` vs `number`). - **Honeydew SQL expressions**: Calculated attributes and metrics use Honeydew's `entity.attribute` reference syntax. These are exported as `ANSI_SQL` dialect expressions in OSI; they remain valid for round-tripping but may not run on other databases without adaptation. - **Perspectives and domains**: Not converted (no OSI equivalent). diff --git a/converters/honeydew/src/honeydew_osi_converter.py b/converters/honeydew/src/honeydew_osi_converter.py index 3857bb7..b12a539 100644 --- a/converters/honeydew/src/honeydew_osi_converter.py +++ b/converters/honeydew/src/honeydew_osi_converter.py @@ -51,10 +51,12 @@ def convert_osi_to_honeydew(osi_yaml_str: str) -> dict[str, str]: attributes/.yml (computed fields only) metrics/.yml - OSI fields with no direct Honeydew equivalent (``ai_context``, - ``unique_keys``, non-Honeydew ``custom_extensions``, relationship - ``name``) are stored in the Honeydew ``metadata`` section under a section - named ``"osi"`` so they can be recovered on the return trip. + ``ai_context`` is mapped to Honeydew's native fields (``description``, + ``labels``, and the AI metadata section); the structured form is also + stored in ``metadata`` for lossless round-tripping. ``unique_keys`` and + non-Honeydew ``custom_extensions`` have no direct Honeydew equivalent and + are stored in the Honeydew ``metadata`` section under a section named + ``"osi"`` so they can be recovered on the return trip. Args: osi_yaml_str: OSI YAML document as a string. From f930a663e4f62fc9320308b4e4fae59fe1eabc7c Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Wed, 27 May 2026 16:24:37 -0400 Subject: [PATCH 24/89] Now each spec defines one ontology (not an array of ontologies) to mirror changes to the core semantic model spec. --- examples/flights.yaml | 192 ++++++++++++----------- ontology/ontology.json | 57 +++---- ontology/ontology.md | 341 +++++++++++++++++++---------------------- 3 files changed, 274 insertions(+), 316 deletions(-) diff --git a/examples/flights.yaml b/examples/flights.yaml index 4162200..56abd26 100644 --- a/examples/flights.yaml +++ b/examples/flights.yaml @@ -1,104 +1,102 @@ version: 0.1.1 -ontologies: - - name: flights - description: Ontology for flight-related concepts and relationships - concepts: - - concept: - name: Example_Runway - type: EntityType - identify_by: [ id ] - relationships: - - name: id - roles: - - concept: String - verbalizes: [ '{Example_Runway} id {String}' ] - multiplicity: ManyToOne - - name: shapelength - roles: - - concept: Float - verbalizes: [ '{Example_Runway} shapeLength {Float}' ] - multiplicity: ManyToOne - - name: airportid - roles: - - concept: String - verbalizes: [ '{Example_Runway} airportId {String}' ] - multiplicity: ManyToOne - - name: length - roles: - - concept: Float - verbalizes: [ '{Example_Runway} length {Float}' ] - multiplicity: ManyToOne - - name: geometry - roles: - - concept: String - verbalizes: [ '{Example_Runway} geometry {String}' ] - multiplicity: ManyToOne - - name: designator - roles: - - concept: String - verbalizes: [ '{Example_Runway} designator {String}' ] - multiplicity: ManyToOne - - name: airports - roles: - - concept: Example_Airport - verbalizes: [ '{Example_Runway} airports {Example_Airport}' ] - multiplicity: ManyToOne - derived_by: [ 'Example_Runway.airportid == Example_Airport.airportid' ] - - concept: - name: Example_RouteAlertComment - type: EntityType - identify_by: [ "id" ] - relationships: - - name: id - roles: - - concept: String - verbalizes: [ '{Example_RouteAlertComment} id {String}' ] - multiplicity: ManyToOne - - name: routealertid - roles: - - concept: String - verbalizes: [ '{Example_RouteAlertComment} routeAlertId {String}' ] - multiplicity: ManyToOne - - name: comment - roles: - - concept: String - verbalizes: [ '{Example_RouteAlertComment} comment {String}' ] - multiplicity: ManyToOne - - name: routealert - roles: - - concept: Example_RouteAlert - verbalizes: [ '{Example_RouteAlertComment} routeAlerts {Example_RouteAlert}' ] - multiplicity: ManyToOne - derived_by: [ 'Example_RouteAlertComment.routealertid == Example_RouteAlert.routealertid' ] - - concept: - name: Example_Airport - type: EntityType - identify_by: [ "airportid" ] - relationships: - - name: airportid - roles: - - concept: String - verbalizes: [ '{Example_Airport} airportId {String}' ] - multiplicity: ManyToOne - - name: displayairportname - roles: - - concept: String - verbalizes: [ '{Example_Airport} displayAirportName {String}' ] - multiplicity: ManyToOne - - name: displaycitymarketnamefull - roles: - - concept: String - verbalizes: [ '{Example_Airport} displayCityMarketNameFull {String}' ] - multiplicity: ManyToOne - - name: flights - roles: - - concept: Example_Flight - verbalizes: [ '{Example_Airport} flights {Example_Flight}' ] - derived_by: [ 'Example_Airport.airportid == Example_Flight.originairportid OR Example_Airport.airportid == Example_Flight.destairportid' ] +name: flights +description: Ontology for flight-related concepts and relationships +ontology: + - concept: + name: Example_Runway + type: EntityType + identify_by: [ id ] + relationships: + - name: id + roles: + - concept: String + verbalizes: [ '{Example_Runway} id {String}' ] + multiplicity: ManyToOne + - name: shapelength + roles: + - concept: Float + verbalizes: [ '{Example_Runway} shapeLength {Float}' ] + multiplicity: ManyToOne + - name: airportid + roles: + - concept: String + verbalizes: [ '{Example_Runway} airportId {String}' ] + multiplicity: ManyToOne + - name: length + roles: + - concept: Float + verbalizes: [ '{Example_Runway} length {Float}' ] + multiplicity: ManyToOne + - name: geometry + roles: + - concept: String + verbalizes: [ '{Example_Runway} geometry {String}' ] + multiplicity: ManyToOne + - name: designator + roles: + - concept: String + verbalizes: [ '{Example_Runway} designator {String}' ] + multiplicity: ManyToOne + - name: airports + roles: + - concept: Example_Airport + verbalizes: [ '{Example_Runway} airports {Example_Airport}' ] + multiplicity: ManyToOne + derived_by: [ 'Example_Runway.airportid == Example_Airport.airportid' ] + - concept: + name: Example_RouteAlertComment + type: EntityType + identify_by: [ "id" ] + relationships: + - name: id + roles: + - concept: String + verbalizes: [ '{Example_RouteAlertComment} id {String}' ] + multiplicity: ManyToOne + - name: routealertid + roles: + - concept: String + verbalizes: [ '{Example_RouteAlertComment} routeAlertId {String}' ] + multiplicity: ManyToOne + - name: comment + roles: + - concept: String + verbalizes: [ '{Example_RouteAlertComment} comment {String}' ] + multiplicity: ManyToOne + - name: routealert + roles: + - concept: Example_RouteAlert + verbalizes: [ '{Example_RouteAlertComment} routeAlerts {Example_RouteAlert}' ] + multiplicity: ManyToOne + derived_by: [ 'Example_RouteAlertComment.routealertid == Example_RouteAlert.routealertid' ] + - concept: + name: Example_Airport + type: EntityType + identify_by: [ "airportid" ] + relationships: + - name: airportid + roles: + - concept: String + verbalizes: [ '{Example_Airport} airportId {String}' ] + multiplicity: ManyToOne + - name: displayairportname + roles: + - concept: String + verbalizes: [ '{Example_Airport} displayAirportName {String}' ] + multiplicity: ManyToOne + - name: displaycitymarketnamefull + roles: + - concept: String + verbalizes: [ '{Example_Airport} displayCityMarketNameFull {String}' ] + multiplicity: ManyToOne + - name: flights + roles: + - concept: Example_Flight + verbalizes: [ '{Example_Airport} flights {Example_Flight}' ] + derived_by: [ 'Example_Airport.airportid == Example_Flight.originairportid OR Example_Airport.airportid == Example_Flight.destairportid' ] ontology_mappings: - name: flights_map description: Example mapping of logical fields to ontology concepts and relationships - ontology: flights semantic_model: name: Logical_Flights_Model description: Logical model for flight data diff --git a/ontology/ontology.json b/ontology/ontology.json index f820b01..cddbe7d 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -24,12 +24,24 @@ "$ref": "#/$defs/Multiplicity" } }, - "ontologies": { + "name": { + "type": "string", + "description": "Unique identifier for the ontology" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/AIContext" + }, + "ontology": { "type": "array", - "description": "Collection of ontologies", "items": { - "$ref": "#/$defs/Ontology" - } + "$ref": "#/$defs/OntologyComponent" + }, + "minItems": 1, + "description": "Components that define the concepts and relationships in this ontology" }, "ontology_mappings": { "type": "array", @@ -39,10 +51,10 @@ } } }, - "required": ["version", "ontologies"], + "required": ["version", "name", "ontology"], "additionalProperties": false, "$defs": { - "Component": { + "OntologyComponent": { "type": "object", "description": "Ontology component that defines a single concept and any relationships that are keyed primarily by that concept", "properties": { @@ -283,10 +295,6 @@ "type": "string", "description": "Human-readable description of this ontology map" }, - "ontology": { - "type": "string", - "description": "Name of the ontology being mapped to" - }, "semantic_model": { "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/SemanticModel" }, @@ -298,34 +306,7 @@ "description": "Maps logical model constructs to some concept and its relationships in the ontology" } }, - "required": ["semantic_model", "ontology", "concept_mappings"], - "additionalProperties": false - }, - "Ontology": { - "type": "object", - "description": "Top-level container representing a complete ontology", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the ontology" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "ai_context": { - "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/AIContext" - }, - "concepts": { - "type": "array", - "items": { - "$ref": "#/$defs/Component" - }, - "minItems": 1, - "description": "Components that define the concepts and relationships in this ontology" - } - }, - "required": ["name", "concepts"], + "required": ["semantic_model", "concept_mappings"], "additionalProperties": false } } diff --git a/ontology/ontology.md b/ontology/ontology.md index df6008e..bd3247d 100644 --- a/ontology/ontology.md +++ b/ontology/ontology.md @@ -60,15 +60,15 @@ The allowable multiplicities of relationships defined in the [Relationships](#re ## Ontologies Ontologies are conceptual models of enterprise data that describe the enterprise in terms -of concepts, relationships, and business rules. This specification represents ontologies +of concepts, relationships, and business rules. This specification represents an ontology hierarchically, grouping each relationship under the concept that plays its first role. | Field | Type | Required | Description | |-------|------|----------|-------------| -| `name` | string | Yes | Unique name of this ontology | +| `name` | string | Yes | Unique name of this specification | | `description` | string | No | Human-readable description | | `ai_context` | string/object | No | Additional context for AI tools | -| `concepts` | list | Yes | Concepts and relationships they group that form this ontology | +| `ontology` | list | Yes | Concepts and relationships they group that form this ontology | ### Concepts @@ -105,17 +105,16 @@ implicitly extends the built-in concept `Any`. This ontology snippet: ```yaml -ontologies: - - name: EnterpriseOntology - concepts: - - concept: - name: SocialSecurityNr - type: ValueType - extends: [Integer] - - concept: - name: Employee - type: EntityType - extends: [Person] +name: EnterpriseOntology +ontology: + - concept: + name: SocialSecurityNr + type: ValueType + extends: [Integer] + - concept: + name: Employee + type: EntityType + extends: [Person] ``` declares two concepts that extend other concepts. @@ -141,25 +140,23 @@ Each relationship is uniquely identified by a prepending its declared name with concept. For instance, in: ```yaml -ontologies: - - name: EnterpriseOntology - concepts: - - concept: - name: Person - type: EntityType - identify_by: [ nr ] - relationships: - - name: nr - roles: - - concept: SocialSecurityNr - verbalizes: [ '{Person} is identified by {SocialSecurityNr}' ] - multiplicity: OneToOne - - name: earns - roles: - - concept: Salary - multiplicity: ManyToOne - verbalizes: [ "{Person} earns {Salary}" ] - ... +ontology: + - concept: + name: Person + type: EntityType + identify_by: [ nr ] + relationships: + - name: nr + roles: + - concept: SocialSecurityNr + verbalizes: [ '{Person} is identified by {SocialSecurityNr}' ] + multiplicity: OneToOne + - name: earns + roles: + - concept: Salary + multiplicity: ManyToOne + verbalizes: [ "{Person} earns {Salary}" ] + ... ``` the relationship is identified by the string `Person.earns`. This convention naturally supports @@ -186,21 +183,19 @@ using this schema: For instance, in: ```yaml -ontologies: - - name: EnterpriseOntology - concepts: - - concept: - name: Person - type: EntityType - relationships: - - name: files_married_joint - verbalizes: [ "{Person} files married filing joint" ] - - name: purchased_on - roles: - - concept: Vehicle - - concept: Date - multiplicity: ManyToOne - verbalizes: [ "{Person} puchased {Vehicle} on {Date}" ] +ontology: + - concept: + name: Person + type: EntityType + relationships: + - name: files_married_joint + verbalizes: [ "{Person} files married filing joint" ] + - name: purchased_on + roles: + - concept: Vehicle + - concept: Date + multiplicity: ManyToOne + verbalizes: [ "{Person} puchased {Vehicle} on {Date}" ] ``` the unary relationship `Person.files_married_joint` has an empty roles list, while the @@ -213,20 +208,18 @@ any additional role whose player's name does not distinguish it from other roles the same relationship. For instance, in: ```yaml -ontologies: - - name: EnterpriseOntology - concepts: - - concept: - name: Store - type: EntityType - relationships: - - name: ships_to_in_days - roles: - - concept: Store - name: destination - - concept: NrDays - multiplicity: ManyToOne - verbalizes: [ "{Store} ships to {Store:destination} in {NrDays}" ] +ontology: + - concept: + name: Store + type: EntityType + relationships: + - name: ships_to_in_days + roles: + - concept: Store + name: destination + - concept: NrDays + multiplicity: ManyToOne + verbalizes: [ "{Store} ships to {Store:destination} in {NrDays}" ] ``` the role name `destination` distinguishes the second `Store`-playing role from the first in @@ -272,32 +265,30 @@ relationship as a view whose objects or links are derived from those of other co relationships. For instance: ```yaml -ontologies: - - name: EnterpriseOntology - concepts: - - concept: - name: Person - type: EntityType - relationships: - - name: parent_of - roles: - - concept: Person - name: "child" - verbalizes: [ "{Person} is a parent of {Person:child}", "{Person:child} is a child of {Person}" ] - - name: ancestor_of - roles: - - concept: Person - name: "descendant" - derived_by: - - "Person.parent_of(descendant)" - "Person.ancestor_of.parent_of(descendant)" - - name: taxed_at - roles: - - concept: TaxRate - derived_by: - - "Person.files_single AND Person.earns <= 11925 AND TaxRate == 10.0" - - "Person.files_married_joint AND Person.earns <= 23850 AND TaxRate == 10.0" - - ... +ontology: + - concept: + name: Person + type: EntityType + relationships: + - name: parent_of + roles: + - concept: Person + name: "child" + verbalizes: [ "{Person} is a parent of {Person:child}", "{Person:child} is a child of {Person}" ] + - name: ancestor_of + roles: + - concept: Person + name: "descendant" + derived_by: + - "Person.parent_of(descendant)" + "Person.ancestor_of.parent_of(descendant)" + - name: taxed_at + roles: + - concept: TaxRate + derived_by: + - "Person.files_single AND Person.earns <= 11925 AND TaxRate == 10.0" + - "Person.files_married_joint AND Person.earns <= 23850 AND TaxRate == 10.0" + - ... ``` declares two derived relationships -- `ancestor_of` and `taxed_at`. Each link of `Person.ancestor_of` @@ -323,14 +314,12 @@ A derived concept is one whose population is derived from that of its supertype using one or more expressions. For instance: ```yaml -ontologies: - - name: EnterpriseOntology - concepts: - - concept: - name: Employee - type: EntityType - extends: [Person] - derived_by: [ "EXISTS ( Person.earns )" ] +ontology: + - concept: + name: Employee + type: EntityType + extends: [Person] + derived_by: [ "EXISTS ( Person.earns )" ] ``` declares that the population of Employee is derived from the population of Person by @@ -343,39 +332,35 @@ by declaring conditions that must hold over their populations. When applied to a expression must reference the concept, as in: ```yaml -ontologies: - - name: EnterpriseOntology - concepts: - - concept: - name: SocialSecurityNr - type: ValueType - extends: [Integer] - requires: [ "0 < SocialSecurityNr", "SocialSecurityNr <= 999999999" ] +ontology: + - concept: + name: SocialSecurityNr + type: ValueType + extends: [Integer] + requires: [ "0 < SocialSecurityNr", "SocialSecurityNr <= 999999999" ] ``` When applied to a relationship, each expression must reference one or more roles of the relationship. For instance, in: ```yaml -ontologies: - - name: EnterpriseOntology - concepts: - - concept: - name: Item - type: EntityType - relationships: - - name: offers_in - roles: - - concept: Store - verbalizations: [ "{Item} is offered for sale in {Store}", "{Store} offers sale of {Item}" ] - - name: total_sales_in - roles: - - concept: Store - - concept: Amount - verbalizations: [ "{Item} sold for cumulative {Amount} in {Store}" ] - requires: - - "Amount > 0.0" - - "Item.offers_in(Store)" +ontology: + - concept: + name: Item + type: EntityType + relationships: + - name: offers_in + roles: + - concept: Store + verbalizations: [ "{Item} is offered for sale in {Store}", "{Store} offers sale of {Item}" ] + - name: total_sales_in + roles: + - concept: Store + - concept: Amount + verbalizations: [ "{Item} sold for cumulative {Amount} in {Store}" ] + requires: + - "Amount > 0.0" + - "Item.offers_in(Store)" ``` the first expression requires any value that plays the `Amount` role to be positive while the second @@ -418,24 +403,22 @@ When the concept is a value type or an entity type with a simple identifier, the a SQL expression. For instance, given this ontology snippet: ```yaml -ontologies: - - name: EnterpriseOntology - concepts: - - concept: - name: SocialSecurityNr - type: ValueType - extends: [ Integer ] - requires: [ "0 < SocialSecurityNr", "SocialSecurityNr <= 999999999" ] - - concept: - name: Person - type: EntityType - identify_by: [ nr ] - relationships: - - name: nr - roles: - - concept: SocialSecurityNr - multiplicity: OneToOne - verbalizes: [ "{Person} is identified by {SocialSecurityNr}" ] +ontology: + - concept: + name: SocialSecurityNr + type: ValueType + extends: [ Integer ] + requires: [ "0 < SocialSecurityNr", "SocialSecurityNr <= 999999999" ] + - concept: + name: Person + type: EntityType + identify_by: [ nr ] + relationships: + - name: nr + roles: + - concept: SocialSecurityNr + multiplicity: OneToOne + verbalizes: [ "{Person} is identified by {SocialSecurityNr}" ] ``` an object mapping that computes `SocialSecurityNumber` values would use a SQL expression to retrieve or @@ -470,21 +453,19 @@ Referent mappings have the following schema: For instance, consider this ontology snippet: ```yaml -ontologies: - - name: EnterpriseOntology - concepts: - - concept: - name: OrderLineItem - type: EntityType - identify_by: [ "nr", "order" ] - requires: [ "OrderLineItem.nr", "OrderLineItem.order" ] - relationships: - - name: nr - roles: [ concept: LineNr ] - multiplicity: ManyToOne - - name: order - roles: [ concept: CustOrder ] - multiplicity: ManyToOne +ontology: + - concept: + name: OrderLineItem + type: EntityType + identify_by: [ "nr", "order" ] + requires: [ "OrderLineItem.nr", "OrderLineItem.order" ] + relationships: + - name: nr + roles: [ concept: LineNr ] + multiplicity: ManyToOne + - name: order + roles: [ concept: CustOrder ] + multiplicity: ManyToOne ``` and notice that `OrderLineItem` has a compound identifier. This concept mapping: @@ -533,31 +514,29 @@ relationship, and so forth. For instance, this ontology snippet: ```yaml -ontologies: - - name: EnterpriseOntology - concepts: - - concept: - name: Item - type: EntityType - identify_by: [ nr ] - relationships: - - name: nr - roles: [ concept: SkuNr ] - multiplicity: OneToOne - verbalises: "{Item} is identified by {SkuNr}" - - name: active # A unary relationship - verbalizes: [ "{Item} is actively sold" ] - - name: active_in - roles: [ concept: Store ] - verbalises: [ "{Item} is actively sold in {Store}" ] - - name: returned_in_for - roles: [ concept: Store, concept: Amount ] - verbalizes: [ "{Item} returned in {Store} for {Amount}" ] - multiplicitly: ManyToOne - - name: sold_in_for - roles: [ concept: Store, concept: Amount ] - verbalizes: [ "{Item} sells in {Store} for {Amount}" ] - multiplicitly: ManyToOne +ontology: + - concept: + name: Item + type: EntityType + identify_by: [ nr ] + relationships: + - name: nr + roles: [ concept: SkuNr ] + multiplicity: OneToOne + verbalises: "{Item} is identified by {SkuNr}" + - name: active # A unary relationship + verbalizes: [ "{Item} is actively sold" ] + - name: active_in + roles: [ concept: Store ] + verbalises: [ "{Item} is actively sold in {Store}" ] + - name: returned_in_for + roles: [ concept: Store, concept: Amount ] + verbalizes: [ "{Item} returned in {Store} for {Amount}" ] + multiplicitly: ManyToOne + - name: sold_in_for + roles: [ concept: Store, concept: Amount ] + verbalizes: [ "{Item} sells in {Store} for {Amount}" ] + multiplicitly: ManyToOne ``` declares one unary, one binary, and two ternary relationships whose links would be tuples of the form (Item), (Item, Store), and (Item, Store, Amount) respectively. And suppose a From 14dea79198d5fd5bf733096f72ffb04f094198a2 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Wed, 27 May 2026 16:33:23 -0400 Subject: [PATCH 25/89] Mirroring changes to the core semantic model spec. --- ontology/ontology.json | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ontology/ontology.json b/ontology/ontology.json index cddbe7d..ebf1cc0 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -10,20 +10,6 @@ "const": "0.1.1", "description": "Ontology specification version" }, - "concept_types": { - "type": "array", - "description": "Concept types (enumeration definition)", - "items": { - "$ref": "#/$defs/ConceptType" - } - }, - "multiplicities": { - "type": "array", - "description": "Relationship multiplicities (enumeration definition)", - "items": { - "$ref": "#/$defs/Multiplicity" - } - }, "name": { "type": "string", "description": "Unique identifier for the ontology" From 6324b64708e339542481e5c53b4fed941f6a18e2 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Wed, 27 May 2026 16:43:26 -0400 Subject: [PATCH 26/89] Fixed title --- ontology/ontology.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ontology/ontology.json b/ontology/ontology.json index ebf1cc0..2b14bf5 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/ontology/ontology.json", - "title": "OSI Core Metadata Specification", + "title": "OSI Ontology Metadata Specification", "description": "JSON Schema for validating OSI (Open Semantic Interoperability) ontology definitions", "type": "object", "properties": { From de12249a3b9cc397fd13375527f37232cac0cdce Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Wed, 27 May 2026 16:45:50 -0400 Subject: [PATCH 27/89] Bumped version --- examples/flights.yaml | 2 +- ontology/ontology.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/flights.yaml b/examples/flights.yaml index 56abd26..8c9d4a1 100644 --- a/examples/flights.yaml +++ b/examples/flights.yaml @@ -1,4 +1,4 @@ -version: 0.1.1 +version: 0.2.0.dev0 name: flights description: Ontology for flight-related concepts and relationships ontology: diff --git a/ontology/ontology.json b/ontology/ontology.json index 2b14bf5..e885e7d 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -7,7 +7,7 @@ "properties": { "version": { "type": "string", - "const": "0.1.1", + "const": "0.2.0.dev0", "description": "Ontology specification version" }, "name": { From e2758f492761e5a351b42c7feac2ea25351b1847 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Fri, 29 May 2026 08:55:24 -0400 Subject: [PATCH 28/89] Fixed typos and incorporated other reviewer comments and suggestions --- ontology/ontology.json | 2 +- ontology/ontology.md | 30 ++++++++++++++++++------------ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/ontology/ontology.json b/ontology/ontology.json index e885e7d..473f69b 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/ontology/ontology.json", + "$id": "https://github.com/open-semantic-interchange/OSI/core-spec/osi-schema.json", "title": "OSI Ontology Metadata Specification", "description": "JSON Schema for validating OSI (Open Semantic Interoperability) ontology definitions", "type": "object", diff --git a/ontology/ontology.md b/ontology/ontology.md index bd3247d..921e061 100644 --- a/ontology/ontology.md +++ b/ontology/ontology.md @@ -1,6 +1,6 @@ # OSI - Ontology Specification -**Version:** 0.1.1 +**Version:** 0.2.0.dev0 ## Table of Contents @@ -70,6 +70,14 @@ hierarchically, grouping each relationship under the concept that plays its firs | `ai_context` | string/object | No | Additional context for AI tools | | `ontology` | list | Yes | Concepts and relationships they group that form this ontology | +Each component of an ontology defines a concept and a list of relationships where that +concept plays the first role: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `concept` | Concept | Yes | A concept in this ontology | +| `relationships` | list | No | Relationships where this concept plays the first role | + ### Concepts Concepts represent the types of things that have meaning in a business setting, e.g., person, company, @@ -83,14 +91,12 @@ Concepts have the following schema: | `name` | string | Yes | Unique name of this concept | | `type` | ConceptType | Yes | Entity type or value type | | `description` | string | No | Human-readable description | -| `relationships` | list | No | Relationships where this concept plays the first role | | `extends` | list | No | Names of this concept's supertypes | | `derived_by` | list | No | Expressions that derive this concept's population | | `identify_by` | list | No | Names of relationships that uniquely reference objects of this concept | | `requires` | list | No | Expressions that constrain this concept's population | -Each concept is either an entity type or a value type, and each concept groups any relationships -where that concept plays the first role. +Each concept is either an entity type or a value type. ### Extends @@ -352,12 +358,12 @@ ontology: - name: offers_in roles: - concept: Store - verbalizations: [ "{Item} is offered for sale in {Store}", "{Store} offers sale of {Item}" ] + verbalizes: [ "{Item} is offered for sale in {Store}", "{Store} offers sale of {Item}" ] - name: total_sales_in roles: - concept: Store - concept: Amount - verbalizations: [ "{Item} sold for cumulative {Amount} in {Store}" ] + verbalizes: [ "{Item} sold for cumulative {Amount} in {Store}" ] requires: - "Amount > 0.0" - "Item.offers_in(Store)" @@ -383,7 +389,7 @@ Concept mappings have the following schema: | Field | Type | Required | Description | |---------------|---------|-----|-------| | `concept` | string | Yes | Names the concept whose part of the ontology is covered by this concept mapping | -| `object_mappings` | string | if no `link_mappings` | Mappings that populate this concept | +| `object_mappings` | list | if no `link_mappings` | Mappings that populate this concept | | `link_mappings` | list | if no `object_mappings` | Mappings that populate the relationships grouped under this concept | ### Object mappings @@ -523,20 +529,20 @@ ontology: - name: nr roles: [ concept: SkuNr ] multiplicity: OneToOne - verbalises: "{Item} is identified by {SkuNr}" + verbalizes: "{Item} is identified by {SkuNr}" - name: active # A unary relationship verbalizes: [ "{Item} is actively sold" ] - name: active_in roles: [ concept: Store ] - verbalises: [ "{Item} is actively sold in {Store}" ] + verbalizes: [ "{Item} is actively sold in {Store}" ] - name: returned_in_for roles: [ concept: Store, concept: Amount ] verbalizes: [ "{Item} returned in {Store} for {Amount}" ] - multiplicitly: ManyToOne + multiplicity: ManyToOne - name: sold_in_for roles: [ concept: Store, concept: Amount ] verbalizes: [ "{Item} sells in {Store} for {Amount}" ] - multiplicitly: ManyToOne + multiplicity: ManyToOne ``` declares one unary, one binary, and two ternary relationships whose links would be tuples of the form (Item), (Item, Store), and (Item, Store, Amount) respectively. And suppose a @@ -576,7 +582,7 @@ though `Store` plays a role in three of the relationships. ## Version History -- **0.1.1** (2026-05-21): Basic support for ontologies and logical schema mappings +- **0.2.0.dev0** (2026-05-29): Basic support for ontologies and logical schema mappings - Core ontology structure: Concepts, relationships, and business rules (requires and derived_by) - Schema mappings from one or more logical models into an ontology From ba2af82ad337dc8e8b9205e1e0c9687c34a5ef57 Mon Sep 17 00:00:00 2001 From: Baruch Oxman Date: Tue, 2 Jun 2026 09:52:29 -0700 Subject: [PATCH 29/89] Fix label idempotency bug and add multi-dataset warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Store OSI `label` in osi metadata when field has label but no dict ai_context, so the Honeydew→OSI path can distinguish OSI-originated labels from native Honeydew labels and avoid injecting spurious ai_context.synonyms on round-trip - Add `label` parameter to `_build_osi_metadata` and corresponding read support in `_read_osi_metadata` - Warn when an entity has more than one dataset file during Honeydew→OSI conversion (only the primary dataset is converted) - Update test expectations to reflect correct idempotent behavior Co-Authored-By: Claude Sonnet 4.6 --- .../honeydew/src/honeydew_osi_converter.py | 30 +++++++++++++++---- .../tests/test_honeydew_osi_converter.py | 6 ++-- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/converters/honeydew/src/honeydew_osi_converter.py b/converters/honeydew/src/honeydew_osi_converter.py index b12a539..79e3772 100644 --- a/converters/honeydew/src/honeydew_osi_converter.py +++ b/converters/honeydew/src/honeydew_osi_converter.py @@ -188,6 +188,7 @@ def _fields_to_honeydew( field_meta = _build_osi_metadata( ai_context=field_ai_ctx if isinstance(field_ai_ctx, dict) else None, + label=field_label if field_label and not isinstance(field_ai_ctx, dict) else None, custom_extensions=field_ext or None, ) @@ -623,6 +624,12 @@ def _read_entity_dir(entity_dir: str, entity_name: str) -> dict[str, Any]: if fn.endswith((".yml", ".yaml")): with open(os.path.join(datasets_dir, fn)) as f: all_ds.append(yaml.safe_load(f) or {}) + if len(all_ds) > 1: + warnings.warn( + f"Entity '{entity_name}' has {len(all_ds)} dataset files; " + "only the primary dataset will be converted", + stacklevel=2, + ) for ds in all_ds: if ds.get("name") == data["key_dataset"] or data["primary_dataset"] is None: data["primary_dataset"] = ds @@ -700,14 +707,18 @@ def _entity_to_osi_dataset(entity_data: dict[str, Any]) -> dict[str, Any]: attr_osi_meta = _read_osi_metadata(attr) attr_labels = attr.get("labels") or [] - # Restore ai_context (structured form takes priority, else build from labels) + # Restore ai_context (structured form takes priority) + # Only synthesise synonyms from labels when they are native Honeydew labels + # (i.e. osi_meta has no 'label' key, meaning the label didn't come from OSI) if attr_osi_meta.get("ai_context"): field["ai_context"] = attr_osi_meta["ai_context"] - elif attr_labels: + elif attr_labels and "label" not in attr_osi_meta: field["ai_context"] = {"synonyms": list(attr_labels)} - # Restore label (first Honeydew label maps to OSI label) - if attr_labels: + # Restore label: prefer osi_meta (exact round-trip), else first Honeydew label + if "label" in attr_osi_meta: + field["label"] = attr_osi_meta["label"] + elif attr_labels: field["label"] = attr_labels[0] # Honeydew-specific metadata → HONEYDEW custom_extension @@ -754,10 +765,12 @@ def _entity_to_osi_dataset(entity_data: dict[str, Any]) -> dict[str, Any]: if calc_osi_meta.get("ai_context"): field["ai_context"] = calc_osi_meta["ai_context"] - elif calc_labels: + elif calc_labels and "label" not in calc_osi_meta: field["ai_context"] = {"synonyms": list(calc_labels)} - if calc_labels: + if "label" in calc_osi_meta: + field["label"] = calc_osi_meta["label"] + elif calc_labels: field["label"] = calc_labels[0] calc_honeydew_extra = { @@ -882,6 +895,7 @@ def _honeydew_metric_to_osi(metric: dict[str, Any], entity_name: str) -> dict[st def _build_osi_metadata( *, ai_context: Any = None, + label: str | None = None, unique_keys: Any = None, custom_extensions: list | None = None, extra_vendors: list[str] | None = None, @@ -892,6 +906,8 @@ def _build_osi_metadata( if ai_context is not None: val = ai_context if isinstance(ai_context, str) else json.dumps(ai_context) items.append({"name": "ai_context", "value": val}) + if label is not None: + items.append({"name": "label", "value": label}) if unique_keys: items.append({"name": "unique_keys", "value": json.dumps(unique_keys)}) if custom_extensions: @@ -918,6 +934,8 @@ def _read_osi_metadata(obj: dict[str, Any]) -> dict[str, Any]: result[key] = json.loads(raw) except (json.JSONDecodeError, TypeError): result[key] = raw + elif key == "label": + result[key] = raw elif key in ("unique_keys", "custom_extensions", "vendors"): try: result[key] = json.loads(raw) diff --git a/converters/honeydew/tests/test_honeydew_osi_converter.py b/converters/honeydew/tests/test_honeydew_osi_converter.py index 46e1e7a..a3c9dab 100644 --- a/converters/honeydew/tests/test_honeydew_osi_converter.py +++ b/converters/honeydew/tests/test_honeydew_osi_converter.py @@ -401,7 +401,10 @@ def test_check_safe_path(rel_path, expected): "type": "dataset", "entity": "orders", "name": "orders", "sql": "db.s.orders", "dataset_type": "table", "attributes": [{"column": "status", "name": "status", - "datatype": "string", "labels": ["sales"]}], + "datatype": "string", "labels": ["sales"], + "metadata": [{"name": "osi", "metadata": [ + {"name": "label", "value": "sales"} + ]}]}], }, id="label-in-attr", ), @@ -841,7 +844,6 @@ def test_honeydew_to_osi_duplicate_relations_deduplicated(tmp_path): {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "fields": [{"name": "status", "expression": _ansi("status"), "dimension": {"is_time": False}, - "ai_context": {"synonyms": ["sales"]}, "label": "sales"}]}]}, id="field-label", ), From 5b69dd7e2d6ee6f607f678ed229b9ff21dff874a Mon Sep 17 00:00:00 2001 From: Baruch Oxman Date: Tue, 2 Jun 2026 09:56:43 -0700 Subject: [PATCH 30/89] Fix label restoration to not fire when labels came from ai_context synonyms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a field has ai_context.synonyms but no label, the osi metadata section contains ai_context but no label key. Guard the label fallback on the absence of osi_meta ai_context so that synonyms don't get promoted to OSI label on round-trip. Verified with TPC-DS: zero semantic differences over OSI→HD→OSI→HD. Co-Authored-By: Claude Sonnet 4.6 --- converters/honeydew/src/honeydew_osi_converter.py | 5 +++-- converters/honeydew/tests/test_honeydew_osi_converter.py | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/converters/honeydew/src/honeydew_osi_converter.py b/converters/honeydew/src/honeydew_osi_converter.py index 79e3772..a3c92f1 100644 --- a/converters/honeydew/src/honeydew_osi_converter.py +++ b/converters/honeydew/src/honeydew_osi_converter.py @@ -716,9 +716,10 @@ def _entity_to_osi_dataset(entity_data: dict[str, Any]) -> dict[str, Any]: field["ai_context"] = {"synonyms": list(attr_labels)} # Restore label: prefer osi_meta (exact round-trip), else first Honeydew label + # Don't set label when labels came from ai_context.synonyms (osi_meta has ai_context) if "label" in attr_osi_meta: field["label"] = attr_osi_meta["label"] - elif attr_labels: + elif attr_labels and not attr_osi_meta.get("ai_context"): field["label"] = attr_labels[0] # Honeydew-specific metadata → HONEYDEW custom_extension @@ -770,7 +771,7 @@ def _entity_to_osi_dataset(entity_data: dict[str, Any]) -> dict[str, Any]: if "label" in calc_osi_meta: field["label"] = calc_osi_meta["label"] - elif calc_labels: + elif calc_labels and not calc_osi_meta.get("ai_context"): field["label"] = calc_labels[0] calc_honeydew_extra = { diff --git a/converters/honeydew/tests/test_honeydew_osi_converter.py b/converters/honeydew/tests/test_honeydew_osi_converter.py index a3c9dab..d9a778e 100644 --- a/converters/honeydew/tests/test_honeydew_osi_converter.py +++ b/converters/honeydew/tests/test_honeydew_osi_converter.py @@ -858,7 +858,6 @@ def test_honeydew_to_osi_duplicate_relations_deduplicated(tmp_path): "description": "Use for revenue analysis", "ai_context": {"instructions": "Use for revenue analysis", "synonyms": ["revenue", "sales"]}, - "label": "revenue", "custom_extensions": [ {"vendor_name": "HONEYDEW", "data": '{"labels": ["revenue", "sales"]}'}, From ae4f48e95db64a10b85d1aca4929962ab986a66c Mon Sep 17 00:00:00 2001 From: Baruch Oxman Date: Tue, 2 Jun 2026 10:01:51 -0700 Subject: [PATCH 31/89] Add HONEYDEW to well-known vendors in spec and converters index Co-Authored-By: Claude Sonnet 4.6 --- converters/index.md | 12 +++++++----- core-spec/spec.md | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/converters/index.md b/converters/index.md index 6477291..5395865 100644 --- a/converters/index.md +++ b/converters/index.md @@ -18,11 +18,12 @@ OSI converters follow a **hub-and-spoke** architecture: │ ┌─────────────┐ ┌─────┴─────┐ ┌─────────────┐ │ dbt ├────┤ OSI ├────┤ Salesforce │ -└─────────────┘ └─────┬─────┘ └─────────────┘ - │ - ┌──────┴──────┐ - │ Databricks │ - └─────────────┘ +└─────────────┘ └──┬─────┬──┘ └─────────────┘ + │ │ + ┌────────┘ └────────┐ + ┌──────┴──────┐ ┌───────┴─────┐ + │ Databricks │ │ Honeydew │ + └─────────────┘ └─────────────┘ ``` This approach avoids the need for point-to-point converters between every pair of vendors. With N vendors, a point-to-point strategy would require N*(N-1) converters. With OSI as the hub, only 2*N converters are needed (one import and one export per vendor), and interoperability with all other vendors comes for free. @@ -54,6 +55,7 @@ The OSI specification currently defines extensions for the following vendors: | `SALESFORCE` | Salesforce / Tableau semantic layer | | `DBT` | dbt semantic models | | `DATABRICKS` | Databricks semantic layer | +| `HONEYDEW` | Honeydew workspace | Each vendor may define custom extensions (via the `custom_extensions` field in the OSI spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/core-spec/spec.md b/core-spec/spec.md index 089f56d..ac410e3 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -54,6 +54,7 @@ The following are well-known examples: | `DBT` | dbt-specific attributes | | `DATABRICKS` | Databricks-specific attributes | | `GOODDATA` | GoodData-specific attributes | +| `HONEYDEW` | Honeydew-specific attributes | ## Semantic Model From 4303054193069d959068a30cf70c75ea5938d2ab Mon Sep 17 00:00:00 2001 From: Baruch Oxman Date: Tue, 2 Jun 2026 10:11:15 -0700 Subject: [PATCH 32/89] Revert hub-and-spoke diagram change in converters/index.md Co-Authored-By: Claude Sonnet 4.6 --- converters/index.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/converters/index.md b/converters/index.md index 5395865..462d290 100644 --- a/converters/index.md +++ b/converters/index.md @@ -18,12 +18,11 @@ OSI converters follow a **hub-and-spoke** architecture: │ ┌─────────────┐ ┌─────┴─────┐ ┌─────────────┐ │ dbt ├────┤ OSI ├────┤ Salesforce │ -└─────────────┘ └──┬─────┬──┘ └─────────────┘ - │ │ - ┌────────┘ └────────┐ - ┌──────┴──────┐ ┌───────┴─────┐ - │ Databricks │ │ Honeydew │ - └─────────────┘ └─────────────┘ +└─────────────┘ └─────┬─────┘ └─────────────┘ + │ + ┌──────┴──────┐ + │ Databricks │ + └─────────────┘ ``` This approach avoids the need for point-to-point converters between every pair of vendors. With N vendors, a point-to-point strategy would require N*(N-1) converters. With OSI as the hub, only 2*N converters are needed (one import and one export per vendor), and interoperability with all other vendors comes for free. From 333bfb462d31f58d8b3d6538b923edf92b4e4851 Mon Sep 17 00:00:00 2001 From: Baruch Oxman Date: Tue, 2 Jun 2026 10:13:26 -0700 Subject: [PATCH 33/89] Revert HONEYDEW entry from converters/index.md supported vendors table Co-Authored-By: Claude Sonnet 4.6 --- converters/index.md | 1 - 1 file changed, 1 deletion(-) diff --git a/converters/index.md b/converters/index.md index 462d290..6477291 100644 --- a/converters/index.md +++ b/converters/index.md @@ -54,7 +54,6 @@ The OSI specification currently defines extensions for the following vendors: | `SALESFORCE` | Salesforce / Tableau semantic layer | | `DBT` | dbt semantic models | | `DATABRICKS` | Databricks semantic layer | -| `HONEYDEW` | Honeydew workspace | Each vendor may define custom extensions (via the `custom_extensions` field in the OSI spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. From 39f699987e3afe60c608a0654382a325077ab873 Mon Sep 17 00:00:00 2001 From: 1waterrj <7208804+1waterrj@users.noreply.github.com> Date: Tue, 2 Jun 2026 16:41:09 +0000 Subject: [PATCH 34/89] Add BIGQUERY to the Dialect enum Adds BIGQUERY (GoogleSQL) as a supported expression dialect alongside the existing ANSI_SQL, SNOWFLAKE, MDX, TABLEAU, DATABRICKS, and MAQL values. - core-spec/osi-schema.json: add BIGQUERY to the Dialect enum - core-spec/spec.yaml: add BIGQUERY to the dialects list - core-spec/spec.md: add BIGQUERY to the dialect table and the multi-dialect field example (SAFE_CAST GoogleSQL form) --- core-spec/osi-schema.json | 2 +- core-spec/spec.md | 3 +++ core-spec/spec.yaml | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core-spec/osi-schema.json b/core-spec/osi-schema.json index 72cb164..b9ea0e8 100644 --- a/core-spec/osi-schema.json +++ b/core-spec/osi-schema.json @@ -23,7 +23,7 @@ "$defs": { "Dialect": { "type": "string", - "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL"], + "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL", "BIGQUERY"], "description": "Supported SQL and expression language dialects" }, "Vendor": { diff --git a/core-spec/spec.md b/core-spec/spec.md index ea7e775..d8fd9e3 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -38,6 +38,7 @@ Supported SQL and expression language dialects for metrics and field definitions | `TABLEAU` | Tableau calculations | | `DATABRICKS` | Databricks SQL | | `MAQL` | GoodData MAQL (Metric Analysis and Query Language) | +| `BIGQUERY` | Google BigQuery (GoogleSQL) | ## Semantic Model @@ -278,6 +279,8 @@ expression: expression: LOWER(email) - dialect: SNOWFLAKE expression: LOWER(email)::VARCHAR + - dialect: BIGQUERY + expression: SAFE_CAST(LOWER(email) AS STRING) description: Normalized email address ``` diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index ed46203..3925ffa 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -19,6 +19,7 @@ dialects: - "TABLEAU" # Tableau - "DATABRICKS" # Databricks SQL - "MAQL" # GoodData MAQL (Multi-Dimensional Analytical Query Language) + - "BIGQUERY" # Google BigQuery GoogleSQL From 0e044fd34f46eb4702f3bbb393aa2e1c8d3fe54f Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Tue, 9 Jun 2026 10:34:05 -0400 Subject: [PATCH 35/89] Added the full flights example following the demo at Summit 2026 and fixed omissions in the spec regarding where requires fields can appear. --- examples/flights.yaml | 2191 ++++++++++++++++++++-------------------- ontology/ontology.json | 14 + 2 files changed, 1118 insertions(+), 1087 deletions(-) diff --git a/examples/flights.yaml b/examples/flights.yaml index 8c9d4a1..d0c1a88 100644 --- a/examples/flights.yaml +++ b/examples/flights.yaml @@ -1,1092 +1,1109 @@ version: 0.2.0.dev0 -name: flights -description: Ontology for flight-related concepts and relationships +name: Flights ontology: - - concept: - name: Example_Runway - type: EntityType - identify_by: [ id ] - relationships: - - name: id - roles: - - concept: String - verbalizes: [ '{Example_Runway} id {String}' ] - multiplicity: ManyToOne - - name: shapelength - roles: - - concept: Float - verbalizes: [ '{Example_Runway} shapeLength {Float}' ] - multiplicity: ManyToOne - - name: airportid - roles: - - concept: String - verbalizes: [ '{Example_Runway} airportId {String}' ] - multiplicity: ManyToOne +- concept: + name: NrFeet + description: "Unit of measure for distance in feet" + type: ValueType + extends: [ Float ] +- concept: + name: NrPounds + description: "Unit of measure for weight in pounds" + type: ValueType + extends: [ Integer ] +- concept: + name: NrMiles + description: "Unit of measure for distance in miles" + type: ValueType + extends: [ Float ] +- concept: + name: NrMinutes + description: "Unit of measure for time in minutes" + type: ValueType + extends: [ Decimal ] +- concept: + name: Capacity + description: "The capacity of an aircraft, measured in pounds." + type: ValueType + extends: [ NrPounds ] +- concept: + name: DegreesLatitude + type: ValueType + extends: [ Float ] + requires: [ DegreesLatitude <= 90, DegreesLatitude >= -90 ] +- concept: + name: DegreesLongitude + type: ValueType + extends: [ Float ] + requires: [ DegreesLongitude <= 180, DegreesLongitude >= -180 ] +- concept: + name: Polygon + description: "A polygon represented as a list of vertices, where each vertex is a pair of latitude and longitude coordinates." + type: ValueType + extends: [ String ] +- concept: + name: CityName + type: ValueType + extends: [ String ] +- concept: + name: Delay + type: ValueType + extends: [ NrMinutes ] +- concept: + name: Distance + type: ValueType + extends: [ NrMiles ] +- concept: + name: RunwayGeometry + description: "A polygon that models the shape of a runway." + type: ValueType + extends: [ Polygon ] +- concept: + name: RunwayLength + description: "The unit for measuring the lengths of runways in American airports." + type: ValueType + extends: [ NrFeet ] +- concept: + name: RunwayDesignator + description: "Used to distinguish runways within a given airport." + type: ValueType + extends: [ String ] +- concept: + name: SerialNr + type: ValueType + extends: [ String ] +- concept: + name: StateCode + type: ValueType + extends: [ String ] +- concept: + name: StateName + type: ValueType + extends: [ String ] +- concept: + name: TailNr + type: ValueType + extends: [ String ] +- concept: + name: Year + type: ValueType + extends: [ String ] +- concept: + name: State + type: EntityType + identify_by: [ code ] + relationships: + - name: code + roles: + - concept: StateCode + verbalizes: + - '{State} is identified by {StateCode}' + multiplicity: OneToOne + - name: name + roles: + - concept: StateName + verbalizes: + - '{State} has {StateName}' + multiplicity: ManyToOne +- concept: + name: City + type: EntityType + identify_by: [ name, state ] + relationships: + - name: name + roles: + - concept: CityName + verbalizes: + - '{City} has {CityName}' + multiplicity: ManyToOne + - name: state + roles: + - concept: State + verbalizes: + - '{City} is located in {State}' + multiplicity: ManyToOne +- concept: + name: Runway + type: EntityType + identify_by: [ designator, airport ] + relationships: + - name: airport + roles: + - concept: Airport + verbalizes: + - '{Runway} belongs to {Airport}' + multiplicity: ManyToOne + - name: designator + roles: + - concept: RunwayDesignator + verbalizes: + - '{Runway} uses {RunwayDesignator}' + multiplicity: ManyToOne + - name: length + roles: + - concept: RunwayLength + verbalizes: + - '{Runway} hase {RunwayLength}' + multiplicity: ManyToOne + - name: geometry + roles: + - concept: RunwayGeometry + verbalizes: + - '{Runway} has {RunwayGeometry}' + multiplicity: ManyToOne +- concept: + name: ManufacturerName + type: ValueType + extends: [ String ] +- concept: + name: Manufacturer + type: EntityType + identify_by: [ name ] + relationships: + - name: name + roles: + - concept: ManufacturerName + verbalizes: + - '{Manufacturer} is identified by {ManufacturerName}' + multiplicity: OneToOne +- concept: + name: ModelName + type: ValueType + extends: [ String ] +- concept: + name: Model + type: EntityType + identify_by: [ name, manufacturer ] + relationships: + - name: name + roles: + - concept: ModelName + verbalizes: + - '{Model} has {ModelName}' + multiplicity: ManyToOne + - name: manufacturer + roles: + - concept: Manufacturer + verbalizes: + - '{Model} is manufactured by {Manufacturer}' + multiplicity: ManyToOne +- concept: + name: Aircraft + type: EntityType + identify_by: [ tailnum ] + requires: [ tailnum ] + relationships: + - name: serial_number + roles: + - concept: SerialNr + verbalizes: + - '{Aircraft} has {SerialNr}' + multiplicity: ManyToOne + - name: name + roles: + - concept: String + verbalizes: + - '{Aircraft} has name {String}' + multiplicity: ManyToOne + - name: number_of_seats + roles: + - concept: Integer + verbalizes: + - '{Aircraft} has {Integer} seats' + multiplicity: ManyToOne + - name: tailnum + roles: + - concept: TailNr + verbalizes: + - '{Aircraft} is identified by {TailNr}' + multiplicity: OneToOne + - name: model + roles: + - concept: Model + verbalizes: + - '{Aircraft} has {Model}' + multiplicity: ManyToOne + - name: year_manufactured + roles: + - concept: Year + verbalizes: + - '{Aircraft} was manufactured in {Year}' + multiplicity: ManyToOne + - name: capacity + roles: + - concept: Capacity + verbalizes: + - '{Aircraft} has {Capacity}' + multiplicity: ManyToOne + - name: carrier + roles: + - concept: Carrier + verbalizes: + - '{Aircraft} carrier {Carrier}' + multiplicity: ManyToOne +- concept: + name: AirportName + type: ValueType + extends: [ String ] +- concept: + name: AirportCode + description: "The three-letter IATA code for the airport." + type: ValueType + extends: [ String ] +- concept: + name: AirportId + description: "Five digit number used as an alternate identifier for airports." + type: ValueType + extends: [ String ] +- concept: + name: Airport + type: EntityType + identify_by: [ code ] + requires: [ Airport.latitude, Airport.longitude ] + relationships: + - name: city + roles: + - concept: City + verbalizes: + - '{Airport} is located in {City}' + multiplicity: ManyToOne + - name: serves + roles: + - concept: Market + verbalizes: + - '{Airport} serves {Market}' + multiplicity: ManyToOne + - name: average_departure_delay + roles: + - concept: Delay + verbalizes: + - '{Airport} averageDepDelay {Delay}' + multiplicity: ManyToOne + derived_by: [ 'Delay == AVG[Flight.departure_delay WHERE Airport == Flight.route.departure GROUP BY Airport]' ] + - name: longitude + roles: + - concept: DegreesLongitude + verbalizes: + - '{Airport} centers at {DegreesLongitude}' + multiplicity: ManyToOne + - name: average_arrival_delay + roles: + - concept: Delay + verbalizes: + - '{Airport} has average arrival {Delay}' + multiplicity: ManyToOne + derived_by: [ 'Decimal == AVG[Flight.arrival_delay WHERE Airport == Flight.route.destination GROUP BY Airport]' ] + - name: name + roles: + - concept: AirportName + verbalizes: + - '{Airport} has {AirportName}' + multiplicity: ManyToOne + - name: code + roles: + - concept: AirportCode + verbalizes: + - '{Airport} has {AirportCode}' + multiplicity: OneToOne + - name: latitude + roles: + - concept: DegreesLatitude + verbalizes: + - '{Airport} centers at {DegreesLatitude}' + multiplicity: ManyToOne +- concept: + name: MarketName + type: ValueType + extends: [ String ] +- concept: + name: Market + type: EntityType + identify_by: [name] + relationships: + - name: name + roles: [ { concept: MarketName }] + verbalizes: [ '{Market} is identified by {MarketName}' ] + multiplicity: OneToOne +- concept: + name: FlightNr + description: "The IATA flight number, which is typically a combination of the airline's IATA code and a numeric code (e.g., 'AA1234')." + type: ValueType + extends: [ String ] +- concept: + name: FlightId + description: "A unique identifier for an instance of a flight" + type: ValueType + extends: [ String ] +- concept: + name: Flight + type: EntityType + identify_by: [ id ] + relationships: + - name: departure_delay + roles: + - concept: Delay + verbalizes: + - '{Flight} has departure {Delay}' + multiplicity: ManyToOne + - name: number + roles: + - concept: FlightNr + verbalizes: + - '{Flight} has {FlightNr}' + multiplicity: ManyToOne + - name: scheduled_departure + roles: [ { concept: DateTime } ] + verbalizes: + - '{Flight} is scheduled to depart at {DateTime}' + multiplicity: ManyToOne + requires: [ Flight.scheduled_departure < Flight.scheduled_arrival ] + - name: arrival_delay + roles: + - concept: Delay + verbalizes: + - '{Flight} has arrival {Delay}' + multiplicity: ManyToOne + - name: cancelled + roles: + - concept: Boolean + verbalizes: + - '{Flight} cancelled {Boolean}' + multiplicity: ManyToOne + - name: cancellationcode + roles: + - concept: String + verbalizes: + - '{Flight} cancellationCode {String}' + multiplicity: ManyToOne + - name: distance + roles: + - concept: Distance + verbalizes: + - '{Flight} spans actual {Distance}' + multiplicity: ManyToOne + - name: id + roles: + - concept: FlightId + verbalizes: + - '{Flight} is identified by {FlightId}' + multiplicity: OneToOne + - name: scheduled_arrival + roles: + - concept: DateTime + verbalizes: + - '{Flight} is scheduled to arrive at {DateTime}' + multiplicity: ManyToOne + - name: registers_longitude_series + roles: + - concept: DateTime + - concept: DegreesLongitude + verbalizes: + - '{Flight} at {DateTime} registers {DegreesLongitude}' + multiplicity: ManyToOne + - name: registers_latitude_series + roles: + - concept: DateTime + - concept: DegreesLatitude + verbalizes: + - '{Flight} at {DateTime} registers {DegreesLatitude}' + multiplicity: ManyToOne + - name: date + roles: + - concept: Date + verbalizes: + - '{Flight} is scheduled to depart on {Date}' + multiplicity: ManyToOne + - name: departs_at + roles: + - concept: DateTime + verbalizes: + - '{Flight} departs at {DateTime}' + multiplicity: ManyToOne + requires: [ Flight.departs_at < Flight.arrives_at ] + - name: diverted + roles: + - concept: Boolean + verbalizes: + - '{Flight} diverted {Boolean}' + multiplicity: ManyToOne + - name: arrives_at + roles: + - concept: DateTime + verbalizes: + - '{Flight} arrives at {DateTime}' + multiplicity: ManyToOne + - name: route + roles: + - concept: Route + verbalizes: + - '{Flight} traverses {Route}' + multiplicity: ManyToOne + - name: aircraft + roles: + - concept: Aircraft + verbalizes: + - '{Flight} uses {Aircraft}' + multiplicity: ManyToOne + requires: [ Flight.operated_by(Aircraft.carrier) ] + - name: operated_by + roles: + - concept: Carrier + verbalizes: + - '{Flight} is operated by {Carrier}' + multiplicity: ManyToOne +- concept: + name: CarrierCode + description: "The two-letter IATA code for the airline carrier." + type: ValueType + extends: [ String ] +- concept: + name: CarrierName + type: ValueType + extends: [ String ] +- concept: + name: Carrier + type: EntityType + identify_by: [ code ] + relationships: + - name: code + roles: + - concept: CarrierCode + verbalizes: + - '{Carrier} uses {CarrierCode}' + multiplicity: OneToOne + - name: name + roles: + - concept: CarrierName + verbalizes: + - '{Carrier} has {CarrierName}' + multiplicity: ManyToOne +- concept: + name: RouteId + type: ValueType + description: "A unique identifier for a route between two airports. Constructed by concatenating the IATA codes of the departure and destination airports (e.g., 'ATL -> DCA')" + extends: [ String ] +- concept: + name: Route + type: EntityType + identify_by: [ id ] + relationships: + - name: average_departure_delay + roles: + - concept: Delay + verbalizes: + - '{Route} has average departure {Delay}' + multiplicity: ManyToOne + derived_by: [ 'Delay == AVG[Flight.departure_delay WHERE Flight.route(Route) GROUP BY Route]' ] + - name: average_arrival_delay + roles: + - concept: Delay + verbalizes: + - '{Route} has average arrival {Delay}' + multiplicity: ManyToOne + derived_by: [ 'Delay == AVG[Flight.arrival_delay WHERE Flight.route(Route) GROUP BY Route]' ] + - name: distance + roles: + - concept: Distance + verbalizes: + - '{Route} spans {Distance}' + multiplicity: ManyToOne + - name: distancegroup + roles: + - concept: Integer + verbalizes: + - '{Route} distanceGroup {Integer}' + multiplicity: ManyToOne + - name: id + roles: + - concept: RouteId + verbalizes: + - '{Route} is identified by {RouteId}' + multiplicity: OneToOne + - name: route_name + roles: + - concept: String + verbalizes: + - '{Route} has name- {String}' + multiplicity: ManyToOne + - name: destination + roles: + - concept: Airport + verbalizes: + - '{Route} connects to destination {Airport}' + multiplicity: ManyToOne + requires: [ 'NOT Route.departure(Airport)' ] + - name: departure + roles: + - concept: Airport + verbalizes: + - '{Route} connects to departure {Airport}' + multiplicity: ManyToOne + requires: [ 'NOT Route.destination(Airport)' ] +ontology_mappings: +- name: flights_mapping + semantic_model: + name: Flights semantic model + datasets: + - name: RUNWAY + source: DATABASE.SCHEMA.RUNWAYS + fields: + - name: airport_code + expression: + dialects: + - dialect: ANSI_SQL + expression: airport_code - name: length - roles: - - concept: Float - verbalizes: [ '{Example_Runway} length {Float}' ] - multiplicity: ManyToOne - - name: geometry - roles: - - concept: String - verbalizes: [ '{Example_Runway} geometry {String}' ] - multiplicity: ManyToOne + expression: + dialects: + - dialect: ANSI_SQL + expression: length + - name: shape + expression: + dialects: + - dialect: ANSI_SQL + expression: shape - name: designator - roles: - - concept: String - verbalizes: [ '{Example_Runway} designator {String}' ] - multiplicity: ManyToOne - - name: airports - roles: - - concept: Example_Airport - verbalizes: [ '{Example_Runway} airports {Example_Airport}' ] - multiplicity: ManyToOne - derived_by: [ 'Example_Runway.airportid == Example_Airport.airportid' ] - - concept: - name: Example_RouteAlertComment - type: EntityType - identify_by: [ "id" ] - relationships: + expression: + dialects: + - dialect: ANSI_SQL + expression: designator + - name: AIRCRAFT + source: DATABASE.SCHEMA.AIRCRAFT + description: "An airplane, helicopter, or other machine capable of flight" + fields: + - name: serial_nr + expression: + dialects: + - dialect: ANSI_SQL + expression: serial_nr + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + - name: nr_seats + expression: + dialects: + - dialect: ANSI_SQL + expression: nr_seats + - name: tail_nr + expression: + dialects: + - dialect: ANSI_SQL + expression: tail_nr + - name: carrier_code + expression: + dialects: + - dialect: ANSI_SQL + expression: carrier_code + - name: manufacturer + expression: + dialects: + - dialect: ANSI_SQL + expression: manufacturer + - name: model + expression: + dialects: + - dialect: ANSI_SQL + expression: model + - name: year + expression: + dialects: + - dialect: ANSI_SQL + expression: year + - name: capacity + expression: + dialects: + - dialect: ANSI_SQL + expression: capacity + - name: AIRPORT + source: DATABASE.SCHEMA.AIRPORTS + description: "An airport that is identified by an IATA code." + fields: + - name: state_code + expression: + dialects: + - dialect: ANSI_SQL + expression: state_code + - name: market_nm + expression: + dialects: + - dialect: ANSI_SQL + expression: market_nm + - name: longitude + expression: + dialects: + - dialect: ANSI_SQL + expression: longitude + - name: opened + expression: + dialects: + - dialect: ANSI_SQL + expression: opened + - name: state_nm + expression: + dialects: + - dialect: ANSI_SQL + expression: state_nm + - name: city_nm + expression: + dialects: + - dialect: ANSI_SQL + expression: city_nm + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + - name: code + expression: + dialects: + - dialect: ANSI_SQL + expression: code + - name: latitude + expression: + dialects: + - dialect: ANSI_SQL + expression: latitude + - name: FLIGHT + source: DATABASE.SCHEMA.FLIGHTS + description: "A commercial passenger flight." + fields: + - name: dep_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: dep_delay + - name: air_time + expression: + dialects: + - dialect: ANSI_SQL + expression: air_time + - name: nr + expression: + dialects: + - dialect: ANSI_SQL + expression: nr + - name: carrier_code + expression: + dialects: + - dialect: ANSI_SQL + expression: carrier_code + - name: duration + expression: + dialects: + - dialect: ANSI_SQL + expression: duration + - name: scheduled_departure + expression: + dialects: + - dialect: ANSI_SQL + expression: scheduled_departure + - name: arr_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: arr_delay + - name: cancelled + expression: + dialects: + - dialect: ANSI_SQL + expression: cancelled + - name: cancel_code + expression: + dialects: + - dialect: ANSI_SQL + expression: cancel_code + - name: distance + expression: + dialects: + - dialect: ANSI_SQL + expression: distance - name: id - roles: - - concept: String - verbalizes: [ '{Example_RouteAlertComment} id {String}' ] - multiplicity: ManyToOne - - name: routealertid - roles: - - concept: String - verbalizes: [ '{Example_RouteAlertComment} routeAlertId {String}' ] - multiplicity: ManyToOne - - name: comment - roles: - - concept: String - verbalizes: [ '{Example_RouteAlertComment} comment {String}' ] - multiplicity: ManyToOne - - name: routealert - roles: - - concept: Example_RouteAlert - verbalizes: [ '{Example_RouteAlertComment} routeAlerts {Example_RouteAlert}' ] - multiplicity: ManyToOne - derived_by: [ 'Example_RouteAlertComment.routealertid == Example_RouteAlert.routealertid' ] - - concept: - name: Example_Airport - type: EntityType - identify_by: [ "airportid" ] - relationships: - - name: airportid - roles: - - concept: String - verbalizes: [ '{Example_Airport} airportId {String}' ] - multiplicity: ManyToOne - - name: displayairportname - roles: - - concept: String - verbalizes: [ '{Example_Airport} displayAirportName {String}' ] - multiplicity: ManyToOne - - name: displaycitymarketnamefull - roles: - - concept: String - verbalizes: [ '{Example_Airport} displayCityMarketNameFull {String}' ] - multiplicity: ManyToOne - - name: flights - roles: - - concept: Example_Flight - verbalizes: [ '{Example_Airport} flights {Example_Flight}' ] - derived_by: [ 'Example_Airport.airportid == Example_Flight.originairportid OR Example_Airport.airportid == Example_Flight.destairportid' ] -ontology_mappings: - - name: flights_map - description: Example mapping of logical fields to ontology concepts and relationships - semantic_model: - name: Logical_Flights_Model - description: Logical model for flight data - datasets: - - name: Example_Runway_example_runway - source: PBACON_ATT_DB.GANDALF.EXAMPLE_RUNWAY - fields: - - name: id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(id AS VARCHAR) - - name: shape_length - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(shape_length AS DOUBLE) - - name: airport_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(airport_id AS VARCHAR) - - name: length - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(length AS DOUBLE) - - name: geometry - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(geometry AS VARCHAR) - - name: designator - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(designator AS VARCHAR) - - name: Example_RouteAlertComment_example_route_alert_comment - source: PBACON_ATT_DB.GANDALF.EXAMPLE_ROUTE_ALERT_COMMENT - fields: - - name: id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(id AS VARCHAR) - - name: alert_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(alert_id AS VARCHAR) - - name: type - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(type AS VARCHAR) - - name: created_by - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(created_by AS VARCHAR) - - name: title - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(title AS VARCHAR) - - name: comment - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(comment AS VARCHAR) - - name: created_at - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(created_at AS TIMESTAMP) - - name: Example_Aircraft_example_aircraft - source: PBACON_ATT_DB.GANDALF.EXAMPLE_AIRCRAFT - description: Aircraft data with basic properties. Full 2023 flight history available - for a subset of aircraft where the "Complete Flight History" property is "true". - fields: - - name: flight_count - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(flight_count AS INTEGER) - - name: serial_number - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(serial_number AS VARCHAR) - - name: mode_scode_hex - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(mode_scode_hex AS VARCHAR) - - name: title - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(title AS VARCHAR) - - name: number_of_seats - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(number_of_seats AS INTEGER) - - name: tail_num - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(tail_num AS VARCHAR) - - name: op_carrier_airline_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(op_carrier_airline_id AS VARCHAR) - - name: carrier_name - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(carrier_name AS VARCHAR) - - name: mode_scode - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(mode_scode AS INTEGER) - - name: complete_flight_history - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(complete_flight_history AS BOOLEAN) - - name: manufacturer - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(manufacturer AS VARCHAR) - - name: latest_flight_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(latest_flight_id AS VARCHAR) - - name: model - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(model AS VARCHAR) - - name: carrier_iata_code - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(carrier_iata_code AS VARCHAR) - - name: year_mfr - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(year_mfr AS VARCHAR) - - name: acquisition_date - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(acquisition_date AS TIMESTAMP) - - name: capacity_in_pounds - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(capacity_in_pounds AS INTEGER) - - name: Example_Airport_example_airport - source: PBACON_ATT_DB.GANDALF.EXAMPLE_AIRPORT - description: Airports with various geospatial properties. Useful for map examples. - Full 2023 flight history available for a subset of airports where the "Complete - Flight History" property is "true". - fields: - - name: airport_state_code - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(airport_state_code AS VARCHAR) - - name: arriving_flight_count - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(arriving_flight_count AS INTEGER) - - name: display_city_market_name_full - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(display_city_market_name_full AS VARCHAR) - - name: average_dep_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(average_dep_delay AS DOUBLE) - - name: longitude - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(longitude AS DOUBLE) - - name: airport_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(airport_id AS VARCHAR) - - name: airport_start_date - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(airport_start_date AS TIMESTAMP) - - name: airport_state_name - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(airport_state_name AS VARCHAR) - - name: daily_avg_dep_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(daily_avg_dep_delay AS VARCHAR) - - name: average_arr_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(average_arr_delay AS DOUBLE) - - name: display_airport_city_name_full - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(display_airport_city_name_full AS VARCHAR) - - name: daily_avg_arr_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(daily_avg_arr_delay AS VARCHAR) - - name: daily_count_of_flights - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(daily_count_of_flights AS VARCHAR) - - name: complete_flight_history - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(complete_flight_history AS BOOLEAN) - - name: display_airport_name - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(display_airport_name AS VARCHAR) - - name: airport - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(airport AS VARCHAR) - - name: departing_flight_count - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(departing_flight_count AS INTEGER) - - name: geopoint - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(geopoint AS VARCHAR) - - name: latitude - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(latitude AS DOUBLE) - - name: Example_Flight_example_flight - source: PBACON_ATT_DB.GANDALF.EXAMPLE_FLIGHT - description: Represents each individual commercial passenger flight in the US - in 2023. With geospatial and timeseries property examples. - fields: - - name: dep_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(dep_delay AS INTEGER) - - name: air_time - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(air_time AS DOUBLE) - - name: flight_number - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(flight_number AS VARCHAR) - - name: airline_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(airline_id AS VARCHAR) - - name: late_aircraft_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(late_aircraft_delay AS INTEGER) - - name: actual_elapsed_time - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(actual_elapsed_time AS DOUBLE) - - name: taxi_in - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(taxi_in AS INTEGER) - - name: scheduled_departure_timestamp - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(scheduled_departure_timestamp AS TIMESTAMP) - - name: dest_airport_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(dest_airport_id AS VARCHAR) - - name: arr_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(arr_delay AS INTEGER) - - name: cancelled - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(cancelled AS BOOLEAN) - - name: carrier_code - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(carrier_code AS VARCHAR) - - name: destination_display_airport_city_name_full - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(destination_display_airport_city_name_full AS VARCHAR) - - name: carrier_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(carrier_delay AS INTEGER) - - name: cancellation_code - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(cancellation_code AS VARCHAR) - - name: origin_latitude - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_latitude AS DOUBLE) - - name: destination_geopoint - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(destination_geopoint AS VARCHAR) - - name: flights - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(flights AS DOUBLE) - - name: destination_display_city_market_name_full - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(destination_display_city_market_name_full AS VARCHAR) - - name: distance - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(distance AS DOUBLE) - - name: flight_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(flight_id AS VARCHAR) - - name: origin_geopoint - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_geopoint AS VARCHAR) - - name: scheduled_arrival_timestamp - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(scheduled_arrival_timestamp AS TIMESTAMP) - - name: destination_latitude - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(destination_latitude AS DOUBLE) - - name: wheels_on_timestamp - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(wheels_on_timestamp AS TIMESTAMP) - - name: longitude_sensor_series_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(longitude_sensor_series_id AS VARCHAR) - - name: nas_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(nas_delay AS INTEGER) - - name: latitude_sensor_series_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(latitude_sensor_series_id AS VARCHAR) - - name: destination_longitude - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(destination_longitude AS DOUBLE) - - name: origin_airport_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_airport_id AS VARCHAR) - - name: origin_display_airport_name - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_display_airport_name AS VARCHAR) - - name: security_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(security_delay AS INTEGER) - - name: origin_display_airport_city_name_full - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_display_airport_city_name_full AS VARCHAR) - - name: origin_longitude - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_longitude AS DOUBLE) - - name: taxi_out - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(taxi_out AS INTEGER) - - name: origin_display_city_market_name_full - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_display_city_market_name_full AS VARCHAR) - - name: destination_airport - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(destination_airport AS VARCHAR) - - name: origin_airport - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_airport AS VARCHAR) - - name: date - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(date AS DATE) - - name: destination_display_airport_name - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(destination_display_airport_name AS VARCHAR) - - name: wheels_off_timestamp - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(wheels_off_timestamp AS TIMESTAMP) - - name: scheduled_elapsed_time - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(scheduled_elapsed_time AS INTEGER) - - name: tail_num - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(tail_num AS VARCHAR) - - name: departure_timestamp - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(departure_timestamp AS TIMESTAMP) - - name: carrier_name - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(carrier_name AS VARCHAR) - - name: route_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(route_id AS VARCHAR) - - name: weather_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(weather_delay AS INTEGER) - - name: diverted - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(diverted AS BOOLEAN) - - name: arrival_timestamp - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(arrival_timestamp AS TIMESTAMP) - - name: flight_title - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(flight_title AS VARCHAR) - - name: route_title - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(route_title AS VARCHAR) - - name: Example_Carrier_example_carrier - source: PBACON_ATT_DB.GANDALF.EXAMPLE_CARRIER - description: Carriers with derived fleet metrics from associated aircraft. Full - 2023 flight history available for a subset of carriers where the "Complete Flight - History" property is "true". - fields: - - name: carrier - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(carrier AS VARCHAR) - - name: average_manufacture_year - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(average_manufacture_year AS DOUBLE) - - name: std_dev_manufacture_year - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(std_dev_manufacture_year AS DOUBLE) - - name: carrier_name - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(carrier_name AS VARCHAR) - - name: daily_avg_arr_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(daily_avg_arr_delay AS VARCHAR) - - name: daily_count_of_flights - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(daily_count_of_flights AS VARCHAR) - - name: complete_flight_history - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(complete_flight_history AS BOOLEAN) - - name: total_capacity_in_pounds - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(total_capacity_in_pounds AS INTEGER) - - name: airline_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(airline_id AS VARCHAR) - - name: total_seats - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(total_seats AS INTEGER) - - name: aircraft_count - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(aircraft_count AS INTEGER) - - name: daily_avg_dep_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(daily_avg_dep_delay AS VARCHAR) - - name: Example_FlightSensor_example_flight_sensor - source: PBACON_ATT_DB.GANDALF.EXAMPLE_FLIGHT_SENSOR - description: 'The flight sensor holds any sensor readings for the flight object. - It is linked to flights as a sensor object and therefore analyses on the flight - object can automatically access time series on the flight sensor. ' - fields: - - name: series_name - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(series_name AS VARCHAR) - - name: units - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(units AS VARCHAR) - - name: flight_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(flight_id AS VARCHAR) - - name: title - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(title AS VARCHAR) - - name: flight_sensor_series_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(flight_sensor_series_id AS VARCHAR) - - name: unique_sensor_flight_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(unique_sensor_flight_id AS VARCHAR) - - name: Example_RouteAlert_example_route_alert - source: PBACON_ATT_DB.GANDALF.EXAMPLE_ROUTE_ALERT - description: Alerts generated based on historical route performance, structured - as an operational object type to demonstrate basic status and assignment patterns. - fields: - - name: normalized_deviation_metric - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(normalized_deviation_metric AS DOUBLE) - - name: normalized_weighted_change_metric - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(normalized_weighted_change_metric AS DOUBLE) - - name: percent_change - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(percent_change AS DOUBLE) - - name: priority - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(priority AS VARCHAR) - - name: assignee - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(assignee AS VARCHAR) - - name: id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(id AS VARCHAR) - - name: flight_count - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(flight_count AS INTEGER) - - name: alert_type - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(alert_type AS VARCHAR) - - name: risk_level - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(risk_level AS INTEGER) - - name: deviation_metric - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(deviation_metric AS DOUBLE) - - name: floor_threshold - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(floor_threshold AS DOUBLE) - - name: description_markdown - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(description_markdown AS VARCHAR) - - name: status - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(status AS VARCHAR) - - name: title - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(title AS VARCHAR) - - name: alert_period_length - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(alert_period_length AS INTEGER) - - name: percent_change_threshold - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(percent_change_threshold AS DOUBLE) - - name: route_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(route_id AS VARCHAR) - - name: metric_description - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(metric_description AS VARCHAR) - - name: description - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(description AS VARCHAR) - - name: previous_deviation_metric - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(previous_deviation_metric AS DOUBLE) - - name: investigation_notes - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(investigation_notes AS VARCHAR) - - name: alert_period_start - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(alert_period_start AS DATE) - - name: percent_change_display - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(percent_change_display AS DOUBLE) - - name: Example_Explainer_example_explainer - source: PBACON_ATT_DB.GANDALF.EXAMPLE_EXPLAINER - description: Stores re-usable callout references to integrate into Workshop and - other one state examples - fields: - - name: id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(id AS VARCHAR) - - name: link_url - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(link_url AS VARCHAR) - - name: text - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(text AS VARCHAR) - - name: title - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(title AS VARCHAR) - - name: link_text - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(link_text AS VARCHAR) - - name: Example_Log_UpdateRouteAlertStatus_example_log_update_route_alert_status - source: PBACON_ATT_DB.GANDALF.EXAMPLE_LOG_UPDATE_ROUTE_ALERT_STATUS - description: 'This object type was automatically created when logging was enabled - for the Action # type: Update Route Alert Status' - fields: - - name: action_triggerer - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(action_triggerer AS VARCHAR) - - name: status - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(status AS VARCHAR) - - name: action_rid - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(action_rid AS VARCHAR) - - name: action_type_rid - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(action_type_rid AS VARCHAR) - - name: action_type_version - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(action_type_version AS VARCHAR) - - name: action_summary - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(action_summary AS VARCHAR) - - name: action_timestamp - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(action_timestamp AS TIMESTAMP) - - name: Example_Route_example_route - source: PBACON_ATT_DB.GANDALF.EXAMPLE_ROUTE - description: Derived from flights between pairs of departure and arrival airports. - Demonstrates the use of deriving "intermediate" object types from granular data - as well as pre-calculating default metrics in a pipeline. Full 2023 flight history - available for a subset of routes where the "Complete Flight History" property - is "true". - fields: - - name: average_dep_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(average_dep_delay AS DOUBLE) - - name: destination_longitude - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(destination_longitude AS DOUBLE) - - name: origin_airport_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_airport_id AS VARCHAR) - - name: destination_airport_1 - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(destination_airport_1 AS VARCHAR) - - name: origin_display_airport_name - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_display_airport_name AS VARCHAR) - - name: daily_avg_dep_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(daily_avg_dep_delay AS VARCHAR) - - name: origin_display_airport_city_name_full - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_display_airport_city_name_full AS VARCHAR) - - name: average_arr_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(average_arr_delay AS DOUBLE) - - name: origin_longitude - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_longitude AS DOUBLE) - - name: daily_count_of_flights - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(daily_count_of_flights AS VARCHAR) - - name: daily_avg_arr_delay - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(daily_avg_arr_delay AS VARCHAR) - - name: dest_airport_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(dest_airport_id AS VARCHAR) - - name: origin_airport - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_airport AS VARCHAR) - - name: destination_display_airport_city_name_full - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(destination_display_airport_city_name_full AS VARCHAR) - - name: origin_latitude - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_latitude AS DOUBLE) - - name: actual_elapsed_time_standard_deviation - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(actual_elapsed_time_standard_deviation AS DOUBLE) - - name: destination_display_airport_name - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(destination_display_airport_name AS VARCHAR) - - name: destination_geopoint - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(destination_geopoint AS VARCHAR) - - name: average_actual_elapsed_time - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(average_actual_elapsed_time AS DOUBLE) - - name: flights_count - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(flights_count AS INTEGER) - - name: distance - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(distance AS DOUBLE) - - name: distance_group - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(distance_group AS INTEGER) - - name: route_id - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(route_id AS VARCHAR) - - name: origin_geopoint - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(origin_geopoint AS VARCHAR) - - name: complete_flight_history - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(complete_flight_history AS BOOLEAN) - - name: destination_latitude - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(destination_latitude AS DOUBLE) - - name: route_title - expression: - dialects: - - dialect: ANSI_SQL - expression: CAST(route_title AS VARCHAR) - concept_mappings: - - concept: Example_Runway - object_mappings: - - referent_mappings: - - relationship: id - expression: Example_Runway_example_runway.id - link_mappings: - - object_mapping: - referent_mappings: - - relationship: id - expression: Example_Runway_example_runway.id - children: - - object_mapping: - concept: Float - expression: Example_Runway_example_runway.length - relationship: length - - object_mapping: - concept: String - expression: Example_Runway_example_runway.geometry - relationship: geometry - - object_mapping: - concept: String - expression: Example_Runway_example_runway.designator - relationship: designator - - object_mapping: - concept: Float - expression: Example_Runway_example_runway.shape_length - relationship: shape_length - - object_mapping: - concept: String - expression: Example_Runway_example_runway.airport_id - relationship: airport_id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + - name: scheduled_arrival + expression: + dialects: + - dialect: ANSI_SQL + expression: scheduled_arrival + - name: wheels_on + expression: + dialects: + - dialect: ANSI_SQL + expression: wheels_on + - name: date + expression: + dialects: + - dialect: ANSI_SQL + expression: date + - name: wheels_off + expression: + dialects: + - dialect: ANSI_SQL + expression: wheels_off + - name: scheduled_duration + expression: + dialects: + - dialect: ANSI_SQL + expression: scheduled_duration + - name: tail_nr + expression: + dialects: + - dialect: ANSI_SQL + expression: tail_nr + - name: departure + expression: + dialects: + - dialect: ANSI_SQL + expression: departure + - name: route_id + expression: + dialects: + - dialect: ANSI_SQL + expression: route_id + - name: diverted + expression: + dialects: + - dialect: ANSI_SQL + expression: diverted + - name: arrival + expression: + dialects: + - dialect: ANSI_SQL + expression: arrival + - name: CARRIER + source: DATABASE.SCHEMA.CARRIERS + description: "An airline, such as Delta, United, or American." + fields: + - name: code + expression: + dialects: + - dialect: ANSI_SQL + expression: code + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + - name: ROUTE + source: DATABASE.SCHEMA.ROUTES + description: "Represents the existence of one or more flights between a pair of departure + arrival airports." + fields: + - name: orig_airport_code + expression: + dialects: + - dialect: ANSI_SQL + expression: orig_airport_code + - name: dest_airport_code + expression: + dialects: + - dialect: ANSI_SQL + expression: dest_airport_code + - name: distance + expression: + dialects: + - dialect: ANSI_SQL + expression: distance + - name: dist_grp + expression: + dialects: + - dialect: ANSI_SQL + expression: dist_grp + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + concept_mappings: + - concept: Runway + object_mappings: + - referent_mappings: + - relationship: designator + expression: RUNWAY.designator + - relationship: airport + referent_mappings: + - relationship: code + expression: RUNWAY.airport_code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: designator + expression: RUNWAY.designator + - relationship: airport + referent_mappings: + - relationship: code + expression: RUNWAY.airport_code + children: + - object_mapping: + concept: RunwayLength + expression: RUNWAY.length + relationship: length + - object_mapping: + concept: RunwayGeometry + expression: RUNWAY.shape + relationship: geometry + - concept: Manufacturer + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRCRAFT.manufacturer + - concept: Model + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRCRAFT.model + - relationship: manufacturer + referent_mappings: + - relationship: name + expression: AIRCRAFT.manufacturer + - concept: Aircraft + object_mappings: + - referent_mappings: + - relationship: tailnum + expression: AIRCRAFT.tail_nr + link_mappings: + - object_mapping: + referent_mappings: + - relationship: tailnum + expression: AIRCRAFT.tail_nr + children: + - object_mapping: + concept: SerialNr + expression: AIRCRAFT.serial_nr + relationship: serial_number + - object_mapping: + concept: String + expression: AIRCRAFT.name + relationship: name + - object_mapping: + concept: Integer + expression: AIRCRAFT.nr_seats + relationship: number_of_seats + - object_mapping: + concept: Model + referent_mappings: + - relationship: name + expression: AIRCRAFT.model + - relationship: manufacturer + referent_mappings: + - relationship: name + expression: AIRCRAFT.manufacturer + relationship: model + - object_mapping: + concept: Year + expression: AIRCRAFT.year + relationship: year_manufactured + - object_mapping: + concept: Capacity + expression: AIRCRAFT.capacity + relationship: capacity + - object_mapping: + concept: Carrier + referent_mappings: + - relationship: code + expression: AIRCRAFT.carrier_code + relationship: carrier + - concept: State + object_mappings: + - referent_mappings: + - relationship: code + expression: AIRPORT.state_code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: code + expression: AIRPORT.state_code + children: + - object_mapping: + concept: StateName + expression: AIRPORT.state_nm + relationship: name + - concept: City + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRPORT.city_nm + - relationship: state + referent_mappings: + - relationship: code + expression: AIRPORT.state_code + - concept: Market + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRPORT.market_nm + - concept: Airport + object_mappings: + - referent_mappings: + - relationship: code + expression: AIRPORT.code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: code + expression: AIRPORT.code + children: + - object_mapping: + concept: City + referent_mappings: + - relationship: name + expression: AIRPORT.city_nm + - relationship: state + referent_mappings: + - relationship: code + expression: AIRPORT.state_code + relationship: city + - object_mapping: + concept: Market + referent_mappings: + - relationship: name + expression: AIRPORT.market_nm + relationship: serves + - object_mapping: + concept: DegreesLongitude + expression: AIRPORT.longitude + relationship: longitude + - object_mapping: + concept: AirportName + expression: AIRPORT.name + relationship: name + - object_mapping: + concept: DegreesLatitude + expression: AIRPORT.latitude + relationship: latitude + - concept: Flight + object_mappings: + - referent_mappings: + - relationship: id + expression: FLIGHT.id + link_mappings: + - object_mapping: + referent_mappings: + - relationship: id + expression: FLIGHT.id + children: + - object_mapping: + concept: Delay + expression: FLIGHT.arr_delay + relationship: arrival_delay + - object_mapping: + concept: Delay + expression: FLIGHT.dep_delay + relationship: departure_delay + - object_mapping: + concept: FlightNr + expression: FLIGHT.nr + relationship: number + - object_mapping: + concept: DateTime + expression: FLIGHT.scheduled_departure + relationship: scheduled_departure + - object_mapping: + concept: Boolean + expression: FLIGHT.cancelled + relationship: cancelled + - object_mapping: + concept: String + expression: FLIGHT.cancel_code + relationship: cancellationcode + - object_mapping: + concept: Distance + expression: FLIGHT.distance + relationship: distance + - object_mapping: + concept: DateTime + expression: FLIGHT.scheduled_arrival + relationship: scheduled_arrival + - object_mapping: + concept: Date + expression: FLIGHT.date + relationship: date + - object_mapping: + concept: DateTime + expression: FLIGHT.departure + relationship: departs_at + - object_mapping: + concept: Boolean + expression: FLIGHT.diverted + relationship: diverted + - object_mapping: + concept: DateTime + expression: FLIGHT.arrival + relationship: arrives_at + - object_mapping: + concept: Route + referent_mappings: + - relationship: id + expression: FLIGHT.route_id + relationship: route + - object_mapping: + concept: Aircraft + referent_mappings: + - relationship: tailnum + expression: FLIGHT.tail_nr + relationship: aircraft + - object_mapping: + concept: Carrier + referent_mappings: + - relationship: code + expression: FLIGHT.carrier_code + relationship: operated_by + - concept: Carrier + object_mappings: + - referent_mappings: + - relationship: code + expression: CARRIER.code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: code + expression: CARRIER.code + children: + - object_mapping: + concept: CarrierName + expression: CARRIER.name + relationship: name + - concept: Route + object_mappings: + - referent_mappings: + - relationship: id + expression: ROUTE.id + link_mappings: + - object_mapping: + referent_mappings: + - relationship: id + expression: ROUTE.id + children: + - object_mapping: + concept: Distance + expression: ROUTE.distance + relationship: distance + - object_mapping: + concept: Integer + expression: ROUTE.dist_grp + relationship: distancegroup + - object_mapping: + concept: String + expression: ROUTE.name + relationship: route_name + - object_mapping: + concept: Airport + referent_mappings: + - relationship: code + expression: ROUTE.dest_airport_code + relationship: destination + - object_mapping: + concept: Airport + referent_mappings: + - relationship: code + expression: ROUTE.orig_airport_code + relationship: departure diff --git a/ontology/ontology.json b/ontology/ontology.json index 473f69b..0cf2e27 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -21,6 +21,13 @@ "ai_context": { "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/AIContext" }, + "requires": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that constrain the population of this ontology" + }, "ontology": { "type": "array", "items": { @@ -95,6 +102,13 @@ }, "description": "Expressions that define how this concept is derived" }, + "requires": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that constrain the population of this relationship" + }, "verbalizes": { "type": "array", "items": { From ae7a6a228ad24c53d93dc3f6ca18a47812212881 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Wed, 10 Jun 2026 12:04:13 -0400 Subject: [PATCH 36/89] Addressed comments from reviewers --- examples/flights.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/examples/flights.yaml b/examples/flights.yaml index d0c1a88..1efd6fb 100644 --- a/examples/flights.yaml +++ b/examples/flights.yaml @@ -5,7 +5,7 @@ ontology: name: NrFeet description: "Unit of measure for distance in feet" type: ValueType - extends: [ Float ] + extends: [ Decimal ] - concept: name: NrPounds description: "Unit of measure for weight in pounds" @@ -15,7 +15,7 @@ ontology: name: NrMiles description: "Unit of measure for distance in miles" type: ValueType - extends: [ Float ] + extends: [ Decimal ] - concept: name: NrMinutes description: "Unit of measure for time in minutes" @@ -29,12 +29,12 @@ ontology: - concept: name: DegreesLatitude type: ValueType - extends: [ Float ] + extends: [ Decimal ] requires: [ DegreesLatitude <= 90, DegreesLatitude >= -90 ] - concept: name: DegreesLongitude type: ValueType - extends: [ Float ] + extends: [ Decimal ] requires: [ DegreesLongitude <= 180, DegreesLongitude >= -180 ] - concept: name: Polygon @@ -143,7 +143,7 @@ ontology: roles: - concept: RunwayLength verbalizes: - - '{Runway} hase {RunwayLength}' + - '{Runway} has {RunwayLength}' multiplicity: ManyToOne - name: geometry roles: @@ -191,7 +191,6 @@ ontology: name: Aircraft type: EntityType identify_by: [ tailnum ] - requires: [ tailnum ] relationships: - name: serial_number roles: @@ -292,7 +291,7 @@ ontology: verbalizes: - '{Airport} has average arrival {Delay}' multiplicity: ManyToOne - derived_by: [ 'Decimal == AVG[Flight.arrival_delay WHERE Airport == Flight.route.destination GROUP BY Airport]' ] + derived_by: [ 'Delay == AVG[Flight.arrival_delay WHERE Airport == Flight.route.destination GROUP BY Airport]' ] - name: name roles: - concept: AirportName From 036efc81f8a9bb74977202e64331ab6d2d805896 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Wed, 10 Jun 2026 12:16:08 -0400 Subject: [PATCH 37/89] Addressing one more comment --- examples/flights.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/flights.yaml b/examples/flights.yaml index 1efd6fb..58f017a 100644 --- a/examples/flights.yaml +++ b/examples/flights.yaml @@ -522,7 +522,7 @@ ontology: roles: - concept: String verbalizes: - - '{Route} has name- {String}' + - '{Route} has name- {String}' # The hyphen after "name" has significance when verbalizing constraints multiplicity: ManyToOne - name: destination roles: From 9619e8bb4eaa00c246f0560019308bd2e3f445d7 Mon Sep 17 00:00:00 2001 From: Baruch Oxman Date: Tue, 16 Jun 2026 10:16:59 +0300 Subject: [PATCH 38/89] Derive dataset unique_keys from entity keys Address review: a many-to-one relation's to_columns must be covered by the target dataset's unique_keys. Since Honeydew validates that relations join to the target entity's keys, derive unique_keys from those keys rather than from the relation: emit entity keys as both primary_key and a unique_keys entry. Keep round-trips clean: on OSI -> Honeydew, a unique key identical to the primary key is not persisted to metadata (it is re-derived), so Honeydew -> OSI -> Honeydew is unchanged. OSI -> Honeydew -> OSI normalizes by surfacing the primary key as a unique key. Co-Authored-By: Claude Opus 4.8 (1M context) --- converters/honeydew/README.md | 3 +- .../honeydew/src/honeydew_osi_converter.py | 18 ++++++- .../tests/test_honeydew_osi_converter.py | 54 +++++++++++++++---- 3 files changed, 63 insertions(+), 12 deletions(-) diff --git a/converters/honeydew/README.md b/converters/honeydew/README.md index ad3158a..5ee6453 100644 --- a/converters/honeydew/README.md +++ b/converters/honeydew/README.md @@ -29,7 +29,7 @@ Bidirectional converter between [OSI](../../core-spec/spec.md) semantic models a |-----------------|-------------| | `workspace.name` | `semantic_model.name` | | Entity + primary dataset | `dataset` | -| `entity.keys` | `dataset.primary_key` | +| `entity.keys` | `dataset.primary_key` (and `dataset.unique_keys`) | | `dataset.attributes` (columns) | `fields` with `ANSI_SQL` expression = column name | | `calculated_attribute` SQL | `fields` with `ANSI_SQL` expression + `HONEYDEW` custom extension | | `entity.relations` (`many-to-one`) | `relationship` with `from` = this entity | @@ -73,3 +73,4 @@ python -m pytest tests/ - **Perspectives and domains**: Not converted (no OSI equivalent). - **Connection expressions** (`connection_expr`): Preserved in `HONEYDEW` custom extensions on the OSI relationship and restored on the return trip. - **`ai_context`**: OSI `ai_context` fields (synonyms, instructions) are stored in Honeydew `metadata` for round-trip recovery. Instructions are also merged into `description` for human readability. +- **`unique_keys`**: A Honeydew entity's `keys` uniquely identify its rows — Honeydew enforces this and validates that relations join to those keys — so they are emitted as the OSI dataset's `primary_key` *and* a `unique_keys` entry. This surfaces the join-target cardinality for OSI consumers (e.g. a many-to-one relation's `to_columns` are always covered by the target's `unique_keys`). A unique key identical to the primary key is not stored back in Honeydew `metadata` on the return trip, so `Honeydew → OSI → Honeydew` stays clean; `OSI → Honeydew → OSI` normalizes by surfacing the primary key as a unique key. diff --git a/converters/honeydew/src/honeydew_osi_converter.py b/converters/honeydew/src/honeydew_osi_converter.py index a3c92f1..31f64bf 100644 --- a/converters/honeydew/src/honeydew_osi_converter.py +++ b/converters/honeydew/src/honeydew_osi_converter.py @@ -240,6 +240,12 @@ def _dataset_to_files( primary_key = ds.get("primary_key") or [] unique_keys = ds.get("unique_keys") + # A unique key identical to the primary key is re-derived from the entity's + # keys on the Honeydew → OSI path, so don't persist it as metadata — that + # would inject a redundant metadata block on a Honeydew → OSI → Honeydew round-trip. + if unique_keys: + pk_tuple = tuple(primary_key) + unique_keys = [uk for uk in unique_keys if tuple(uk) != pk_tuple] or None description = ds.get("description") ai_context = ds.get("ai_context") fields = ds.get("fields") or [] @@ -671,8 +677,16 @@ def _entity_to_osi_dataset(entity_data: dict[str, Any]) -> dict[str, Any]: osi_meta = entity_data.get("osi_meta") or {} if osi_meta.get("ai_context"): ds["ai_context"] = osi_meta["ai_context"] - if osi_meta.get("unique_keys"): - ds["unique_keys"] = osi_meta["unique_keys"] + + # A Honeydew entity's keys uniquely identify its rows — Honeydew enforces this + # and validates that relationships join to those keys — so surface them as a + # unique key as well as the primary key. Union with any unique keys preserved + # from an OSI source, de-duplicated. + unique_keys = [list(uk) for uk in (osi_meta.get("unique_keys") or [])] + if keys and tuple(keys) not in {tuple(uk) for uk in unique_keys}: + unique_keys.append(list(keys)) + if unique_keys: + ds["unique_keys"] = unique_keys restored_ext = list(osi_meta.get("custom_extensions") or []) honeydew_extra = entity_data.get("honeydew_extra") or {} diff --git a/converters/honeydew/tests/test_honeydew_osi_converter.py b/converters/honeydew/tests/test_honeydew_osi_converter.py index d9a778e..08c2265 100644 --- a/converters/honeydew/tests/test_honeydew_osi_converter.py +++ b/converters/honeydew/tests/test_honeydew_osi_converter.py @@ -615,6 +615,7 @@ def _ansi(expr): "name": "orders", "source": "SNOWFLAKE_SAMPLE_DATA.TPCH_SF1.ORDERS", "primary_key": ["orderkey"], + "unique_keys": [["orderkey"]], "fields": [ {"name": "orderkey", "expression": _ansi("o_orderkey")}, {"name": "orderdate", "expression": _ansi("o_orderdate"), @@ -631,6 +632,7 @@ def _ansi(expr): "dataset_attrs": [{"column": "id", "name": "id", "datatype": "number"}]}], _hd_root({"name": "ws", "datasets": [{ "name": "orders", "source": "db.s.orders", "primary_key": ["id"], + "unique_keys": [["id"]], "fields": [{"name": "id", "expression": _ansi("id")}], }]}), id="field-number", @@ -641,6 +643,7 @@ def _ansi(expr): "dataset_attrs": [{"column": "status", "name": "status", "datatype": "string"}]}], _hd_root({"name": "ws", "datasets": [{ "name": "orders", "source": "db.s.orders", "primary_key": ["id"], + "unique_keys": [["id"]], "fields": [{"name": "status", "expression": _ansi("status"), "dimension": {"is_time": False}}], }]}), @@ -652,6 +655,7 @@ def _ansi(expr): "dataset_attrs": [{"column": "created_at", "name": "created_at", "datatype": "timestamp"}]}], _hd_root({"name": "ws", "datasets": [{ "name": "orders", "source": "db.s.orders", "primary_key": ["id"], + "unique_keys": [["id"]], "fields": [{"name": "created_at", "expression": _ansi("created_at"), "dimension": {"is_time": True}}], }]}), @@ -665,6 +669,7 @@ def _ansi(expr): "labels": ["sales", "reporting"]}]}], _hd_root({"name": "ws", "datasets": [{ "name": "orders", "source": "db.s.orders", "primary_key": ["id"], + "unique_keys": [["id"]], "fields": [{ "name": "status", "expression": _ansi("status"), "dimension": {"is_time": False}, @@ -691,8 +696,10 @@ def _ansi(expr): _hd_root({ "name": "ws", "datasets": [ - {"name": "customers", "source": "db.s.customers", "primary_key": ["id"]}, - {"name": "orders", "source": "db.s.orders", "primary_key": ["order_id"]}, + {"name": "customers", "source": "db.s.customers", "primary_key": ["id"], + "unique_keys": [["id"]]}, + {"name": "orders", "source": "db.s.orders", "primary_key": ["order_id"], + "unique_keys": [["order_id"]]}, ], "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", "from_columns": ["customer_id"], "to_columns": ["id"]}], @@ -713,8 +720,10 @@ def _ansi(expr): _hd_root({ "name": "ws", "datasets": [ - {"name": "customers", "source": "db.s.customers", "primary_key": ["id"]}, - {"name": "orders", "source": "db.s.orders", "primary_key": ["order_id"]}, + {"name": "customers", "source": "db.s.customers", "primary_key": ["id"], + "unique_keys": [["id"]]}, + {"name": "orders", "source": "db.s.orders", "primary_key": ["order_id"], + "unique_keys": [["order_id"]]}, ], "relationships": [{"name": "orders_to_customers", "from": "orders", "to": "customers", "from_columns": ["customer_id"], "to_columns": ["id"]}], @@ -729,7 +738,8 @@ def _ansi(expr): "metrics": [{"type": "metric", "entity": "orders", "name": "count", "datatype": "number", "sql": "COUNT(*)"}]}], _hd_root({"name": "ws", "datasets": [ - {"name": "orders", "source": "db.s.orders", "primary_key": ["id"]}, + {"name": "orders", "source": "db.s.orders", "primary_key": ["id"], + "unique_keys": [["id"]]}, ], "metrics": [{ "name": "count", "expression": _ansi("COUNT(*)"), @@ -747,6 +757,7 @@ def _ansi(expr): "sql": "orders.price * (1 - orders.discount)"}]}], _hd_root({"name": "ws", "datasets": [{ "name": "orders", "source": "db.s.orders", "primary_key": ["id"], + "unique_keys": [["id"]], "fields": [{ "name": "discounted", "expression": _ansi("orders.price * (1 - orders.discount)"), @@ -802,6 +813,26 @@ def test_honeydew_to_osi_duplicate_relations_deduplicated(tmp_path): assert len(result["semantic_model"][0].get("relationships", [])) == 1 +def test_honeydew_to_osi_relation_target_columns_are_unique_keys(tmp_path): + # Honeydew validates that a many-to-one relation joins to the target entity's + # keys, so the relation's to_columns are always covered by the target dataset's + # unique_keys (derived from those keys). This preserves the cardinality metadata + # for OSI consumers (e.g. Snowflake) without inspecting the relation itself. + _write_workspace(str(tmp_path), "ws", [ + {"name": "orders", "keys": ["order_id"], "key_dataset": "orders", "sql": "db.s.orders", + "relations": [{"target_entity": "customers", "rel_type": "many-to-one", + "connection": [{"src_field": "customer_id", "target_field": "id"}]}], + "dataset_attrs": []}, + {"name": "customers", "keys": ["id"], "key_dataset": "customers", + "sql": "db.s.customers", "dataset_attrs": []}, + ]) + sm = yaml.safe_load(convert_honeydew_to_osi(str(tmp_path)))["semantic_model"][0] + datasets = {ds["name"]: ds for ds in sm["datasets"]} + rel = sm["relationships"][0] + target_ds = datasets[rel["to"]] + assert rel["to_columns"] in target_ds["unique_keys"] + + # ───────────────────────────────────────────────────────────────────────────── # OSI → Honeydew → OSI round-trip: full semantic model # ───────────────────────────────────────────────────────────────────────────── @@ -815,15 +846,18 @@ def test_honeydew_to_osi_duplicate_relations_deduplicated(tmp_path): pytest.param( {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "primary_key": ["order_id"], "fields": []}]}, + # OSI → Honeydew → OSI normalizes: the primary key surfaces as a unique key, + # because Honeydew expresses uniqueness through entity keys. {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", - "primary_key": ["order_id"]}]}, + "primary_key": ["order_id"], "unique_keys": [["order_id"]]}]}, id="pk-single", ), pytest.param( {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", "primary_key": ["order_id", "line_no"], "fields": []}]}, {"name": "m", "datasets": [{"name": "orders", "source": "db.s.orders", - "primary_key": ["order_id", "line_no"]}]}, + "primary_key": ["order_id", "line_no"], + "unique_keys": [["order_id", "line_no"]]}]}, id="pk-composite", ), pytest.param( @@ -831,9 +865,10 @@ def test_honeydew_to_osi_duplicate_relations_deduplicated(tmp_path): "primary_key": ["id"], "unique_keys": [["sku"], ["id", "variant"]], "fields": []}]}, + # The primary key is appended as a unique key on the way back out. {"name": "m", "datasets": [{"name": "items", "source": "db.s.items", "primary_key": ["id"], - "unique_keys": [["sku"], ["id", "variant"]]}]}, + "unique_keys": [["sku"], ["id", "variant"], ["id"]]}]}, id="unique-keys", ), pytest.param( @@ -1420,7 +1455,8 @@ def test_main_honeydew_to_osi(tmp_path): "version": OSI_VERSION, "vendors": ["HONEYDEW"], "semantic_model": [{"name": "ws", "datasets": [ - {"name": "orders", "source": "DB.S.ORDERS", "primary_key": ["id"]}, + {"name": "orders", "source": "DB.S.ORDERS", "primary_key": ["id"], + "unique_keys": [["id"]]}, ]}], } From b1bf4155db53647bb840e1e704dbe1936c32707b Mon Sep 17 00:00:00 2001 From: Ralf Becher Date: Tue, 16 Jun 2026 09:31:34 +0200 Subject: [PATCH 39/89] Add OrionBelt (OBML) converter Bidirectional OSI <-> OrionBelt OBML converter, plus a ROADMAP entry. OSI -> OBML and OBML -> OSI with roundtrip fidelity via vendor custom_extensions. Highlights from review on this PR: - Reads ANSI_SQL, SNOWFLAKE, and DATABRICKS metric expressions (their aggregations are ANSI-compatible); MDX/TABLEAU/MAQL are not parsed as SQL. - Resolves dataset.column references against the OSI dataset/field map case-insensitively and with SQL quoting stripped; decimal literals are kept literal. Unmappable references preserve the metric verbatim. - Metrics with no OBML representation are preserved verbatim under an OSI-vendor customExtension and re-emitted on OBML -> OSI, with a loud LOSSY: warning, so the OSI -> OBML -> OSI roundtrip stays lossless. - Emits schema-conformant OSI: no root-level dialects/vendors (those live per-expression and per-entity), matching the published core schema. - Name-collision safety and idempotent convert() on both directions. --- ROADMAP.md | 1 + converters/orionbelt/README.md | 125 + .../orionbelt/osi_obml_mapping_analysis.md | 269 ++ .../osi_obml_ontology_mapping_analysis.md | 82 + converters/orionbelt/pyproject.toml | 58 + .../orionbelt/src/osi_orionbelt/__init__.py | 41 + converters/orionbelt/src/osi_orionbelt/cli.py | 155 + .../orionbelt/src/osi_orionbelt/converter.py | 2967 +++++++++++++++++ .../osi_orionbelt/schemas/obml-schema.json | 1319 ++++++++ .../schemas/osi-ontology-schema.json | 299 ++ .../src/osi_orionbelt/schemas/osi-schema.json | 330 ++ .../orionbelt/tests/fixtures/obml_as_osi.yaml | 787 +++++ .../tests/fixtures/tpcds_as_obml.yaml | 381 +++ .../orionbelt/tests/fixtures/tpcds_osi.yaml | 578 ++++ .../tests/fixtures/tpcds_semantic_model.yaml | 578 ++++ .../tests/test_osi_converter_cumulative.py | 323 ++ .../tests/test_osi_converter_filters.py | 124 + .../test_osi_converter_measure_overrides.py | 297 ++ .../tests/test_osi_converter_ontology.py | 178 + .../orionbelt/tests/test_osi_converter_pop.py | 424 +++ .../tests/test_osi_converter_properties.py | 314 ++ .../tests/test_osi_converter_trend_v26.py | 212 ++ .../tests/test_osi_converter_vendors.py | 237 ++ .../tests/test_osi_metric_no_silent_loss.py | 365 ++ .../tests/test_osi_tpcds_baseline.py | 72 + .../orionbelt/tests/test_osi_v02_compat.py | 417 +++ converters/orionbelt/uv.lock | 1115 +++++++ 27 files changed, 12048 insertions(+) create mode 100644 converters/orionbelt/README.md create mode 100644 converters/orionbelt/osi_obml_mapping_analysis.md create mode 100644 converters/orionbelt/osi_obml_ontology_mapping_analysis.md create mode 100644 converters/orionbelt/pyproject.toml create mode 100644 converters/orionbelt/src/osi_orionbelt/__init__.py create mode 100644 converters/orionbelt/src/osi_orionbelt/cli.py create mode 100644 converters/orionbelt/src/osi_orionbelt/converter.py create mode 100644 converters/orionbelt/src/osi_orionbelt/schemas/obml-schema.json create mode 100644 converters/orionbelt/src/osi_orionbelt/schemas/osi-ontology-schema.json create mode 100644 converters/orionbelt/src/osi_orionbelt/schemas/osi-schema.json create mode 100644 converters/orionbelt/tests/fixtures/obml_as_osi.yaml create mode 100644 converters/orionbelt/tests/fixtures/tpcds_as_obml.yaml create mode 100644 converters/orionbelt/tests/fixtures/tpcds_osi.yaml create mode 100644 converters/orionbelt/tests/fixtures/tpcds_semantic_model.yaml create mode 100644 converters/orionbelt/tests/test_osi_converter_cumulative.py create mode 100644 converters/orionbelt/tests/test_osi_converter_filters.py create mode 100644 converters/orionbelt/tests/test_osi_converter_measure_overrides.py create mode 100644 converters/orionbelt/tests/test_osi_converter_ontology.py create mode 100644 converters/orionbelt/tests/test_osi_converter_pop.py create mode 100644 converters/orionbelt/tests/test_osi_converter_properties.py create mode 100644 converters/orionbelt/tests/test_osi_converter_trend_v26.py create mode 100644 converters/orionbelt/tests/test_osi_converter_vendors.py create mode 100644 converters/orionbelt/tests/test_osi_metric_no_silent_loss.py create mode 100644 converters/orionbelt/tests/test_osi_tpcds_baseline.py create mode 100644 converters/orionbelt/tests/test_osi_v02_compat.py create mode 100644 converters/orionbelt/uv.lock diff --git a/ROADMAP.md b/ROADMAP.md index 04a8ae9..992efe0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -415,6 +415,7 @@ Broad ecosystem adoption depends on practical tools that let teams validate thei - [GoodData Converter](converters/gooddata/) — bidirectional OSI ↔ GoodData LDM converter - [Salesforce Converter](converters/salesforce/) — OSI ↔ Salesforce converter - [Apache Polaris Converter](converters/polaris/) — OSI → Apache Polaris converter +- [OrionBelt Converter](converters/orionbelt/) — bidirectional OSI ↔ OrionBelt OBML converter **Related Issues:** diff --git a/converters/orionbelt/README.md b/converters/orionbelt/README.md new file mode 100644 index 0000000..94b3ae0 --- /dev/null +++ b/converters/orionbelt/README.md @@ -0,0 +1,125 @@ +# osi-orionbelt + +Bidirectional converter between **OBML** (OrionBelt Markup Language) semantic +models and **OSI** ([Open Semantic Interchange](https://open-semantic-interchange.org/)), +the open standard for portable semantic models (metrics, dimensions, +relationships). + +This package is licensed under **Apache-2.0** and may be used freely. It is the +OrionBelt converter in the OSI converter ecosystem. The canonical source is +developed in the +[orionbelt-semantic-layer](https://github.com/ralfbecher/orionbelt-semantic-layer) +repository (under `packages/osi-orionbelt`) and published to PyPI from there; +file issues and contributions upstream. + +## Requirements + +- Python 3.12+ +- [uv](https://docs.astral.sh/uv/) (recommended) or pip + +## Install + +```bash +pip install osi-orionbelt +``` + +Optional deep OBML semantic validation (cycles, duplicate names, invalid refs) +via the full OrionBelt engine: + +```bash +pip install "osi-orionbelt[obml-validation]" +``` + +Without that extra, OBML validation runs JSON-schema checks only and emits a +warning for the deeper semantic pass. + +## CLI + +A single `osi-orionbelt` command with two subcommands (mirroring `osi-dbt`): + +| Subcommand | Direction | In | Out | +|---------|-----------|----|----| +| `obml-to-osi` | OBML -> OSI core-spec | OBML YAML | OSI YAML | +| `obml-to-osi --ontology` | OBML -> OSI ontology | OBML YAML | OSI ontology YAML | +| `osi-to-obml` | OSI core-spec -> OBML | OSI YAML | OBML YAML | + +```bash +osi-orionbelt obml-to-osi -i model.obml.yaml -o model.osi.yaml +osi-orionbelt obml-to-osi --ontology -i model.obml.yaml -o model.ontology.yaml +osi-orionbelt osi-to-obml -i model.osi.yaml -o model.obml.yaml +``` + +`-i/--input` and `-o/--output` are required. Each subcommand prints conversion +warnings and a validation summary to stderr, and exits non-zero when the +produced document fails schema validation (unless `--no-validate`). Run +`osi-orionbelt --help` or `osi-orionbelt obml-to-osi --help` for the full +option list. + +## Python API + +```python +import yaml +from osi_orionbelt import OBMLtoOSI, OSItoOBML, validate_osi + +obml = yaml.safe_load(open("model.obml.yaml")) +osi = OBMLtoOSI(obml, "sales", "Sales model").convert() +result = validate_osi(osi) +assert result.valid + +obml_again = OSItoOBML(osi).convert() +``` + +## Vendor extensions + +OSI `custom_extensions` carry vendor-tagged payloads. This converter: + +- emits OrionBelt/OBML-proprietary data under the **`ORIONBELT`** vendor on OBML + to OSI (OBML-only filters, settings, owner, refresh, type info, etc.); +- stashes OSI-native fields that OBML can't represent (unique keys, field + labels, leftover `ai_context`) under the **`OSI`** vendor when going OSI to + OBML, restoring them to first-class OSI fields on the way back; +- **preserves third-party vendor extensions verbatim** (e.g. `SNOWFLAKE`, + `DBT`, `SALESFORCE`, `GOODDATA`) at the model, dataset, field, and + measure/metric levels, so a full OSI to OBML to OSI roundtrip keeps the + original vendor and data. OSI has no separate dimension entity, so an OBML + dimension's foreign extensions surface on its OSI field. + +Legacy `COMMON` / `OBSL` tags from earlier converter versions are still accepted +on read. + +## Limitations / unsupported constructs + +Some OBML constructs have no native OSI equivalent and are carried in vendor +`custom_extensions` (`obml_*` payloads) so they round-trip without loss back to +OBML, but are not interpreted by other OSI consumers: + +- **Many-to-many joins** - represented in OBML join cardinality; flagged on + export. +- **Named secondary join paths** - OBML's multiple join paths between the same + pair of objects are an OBML-specific topology feature. +- **Measures / metrics and column-level value concepts in the ontology layer** - + not represented in the OSI ontology export. +- **OSI metrics with no OBML representation** - a metric whose only expression is + in a non-SQL dialect (`MDX`, `TABLEAU`, `MAQL`), or whose SQL expression cannot + be decomposed into OBML measures/metrics, is **not** dropped: the original OSI + metric is preserved verbatim in a model-level `OSI`-vendor `custom_extension` + (`obml_unconverted_metrics`) and re-emitted on OBML to OSI, so the OSI to OBML + to OSI roundtrip stays lossless. A `LOSSY:` warning is raised for each such + metric because it is **not queryable through OBML**. SQL expressions in the + `ANSI_SQL`, `SNOWFLAKE`, and `DATABRICKS` dialects are all read on import. + +OSI v0.1.x inputs are accepted on read via a legacy normalization shim; output +targets OSI **v0.2.0.dev0**. + +See [`osi_obml_mapping_analysis.md`](./osi_obml_mapping_analysis.md) for the +full OBML <-> OSI core-spec mapping and +[`osi_obml_ontology_mapping_analysis.md`](./osi_obml_ontology_mapping_analysis.md) +for the ontology-layer mapping and its documented gaps. + +## Development + +```bash +uv sync # install +uv run pytest # run the test suite (includes a TPC-DS baseline) +uv run ruff check && uv run mypy src/osi_orionbelt +``` diff --git a/converters/orionbelt/osi_obml_mapping_analysis.md b/converters/orionbelt/osi_obml_mapping_analysis.md new file mode 100644 index 0000000..9c3f6ee --- /dev/null +++ b/converters/orionbelt/osi_obml_mapping_analysis.md @@ -0,0 +1,269 @@ +# OSI ↔ OBML Mapping Analysis + +> Bidirectional conversion between [Open Semantic Interchange (OSI)](https://github.com/open-semantic-interchange/OSI) v0.2.0.dev0 and [OrionBelt ML (OBML)](https://github.com/ralfbecher/orionbelt-semantic-layer) v1.0 semantic model formats. OSI v0.1.x inputs are still accepted on read via a legacy normalization shim; output targets v0.2.0.dev0. + +## 1. Structural Comparison + +| Aspect | OSI v0.2.0.dev0 | OBML v1.0 | +|---|---|---| +| **Top-level** | `semantic_model[]` (array of models) | Single model with `dataObjects`, `dimensions`, `measures`, `metrics` sections | +| **Tables / Entities** | `datasets[]` (flat array) | `dataObjects{}` (named dictionary) | +| **Column identifiers** | `fields[].name` (snake_case code) | `columns{}.code` (with display name as dict key) | +| **Expressions** | `expression.dialects[]` per field (multi-dialect) | Single SQL expression via `code` (single dialect) | +| **Joins / Relationships** | `relationships[]` (global, separate section) | `joins[]` (inline on each data object) | +| **Dimensions** | `field.dimension.is_time` (inline flag on fields) | `dimensions{}` (separate top-level section) | +| **Measures** | N/A (merged into metrics) | `measures{}` (explicit aggregation definitions) | +| **Metrics** | `metrics[]` with full SQL expressions | `metrics{}` with `{[Measure]}` references | +| **AI Context** | `ai_context` on every entity | `customExtensions` with `vendor: "OSI"` | +| **Extensibility** | `custom_extensions[]` per entity | `customExtensions[]` per entity | +| **Keys** | `primary_key`, `unique_keys` on datasets | N/A | +| **Secondary joins** | N/A | `secondary: true`, `pathName` | +| **Fan-out protection** | N/A | `allowFanOut`, `reduceToRelationDimensionality` | + +## 2. Key Differences + +### 2.1 Naming Convention + +- **OSI** uses snake_case codes everywhere (`name: "store_sales"`) +- **OBML** supports dual naming — a display name as the dictionary key and a `code` for the physical SQL reference + +During OSI → OBML conversion, field names are used directly as both the display name and code. During OBML → OSI conversion, the `code` value becomes the OSI field `name`. + +### 2.2 Relationship Placement + +- **OSI** defines relationships globally, referencing dataset names by string +- **OBML** defines joins inline on the "from" side data object + +The converter restructures between these two representations automatically, preserving column mappings and generating descriptive relationship names. + +### 2.3 Measures vs. Metrics + +This is the most fundamental structural difference between the two formats. + +- **OSI** has a single "metrics" concept with full SQL expressions (e.g., `SUM(store_sales.ss_ext_sales_price)`) +- **OBML** explicitly separates: + - **Measures**: Simple aggregations on columns (e.g., `SUM` of `ss_ext_sales_price`) + - **Metrics**: Derived calculations referencing measures via `{[Name]}` syntax (e.g., `{[total_sales]} / {[customer_count]}`) + +The converter handles this decomposition automatically: + +| OSI metric type | OBML mapping | +|---|---| +| `AGG(dataset.column)` | Direct measure | +| `AGG(DISTINCT dataset.column)` | Measure with `distinct: true` | +| `AGG(expr)` (e.g., `SUM(a.x * a.y)`) | Expression-based measure | +| Multi-aggregation expression | Auto-generated measures + metric formula | + +**Example** — OSI metric `customer_lifetime_value`: +```yaml +# OSI +expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) +``` + +is decomposed into OBML: +```yaml +# OBML measures (auto-generated) +measures: + total_sales: + columns: + - dataObject: store_sales + column: ss_ext_sales_price + resultType: float + aggregation: sum + + _customer_c_customer_sk_count_distinct: + columns: + - dataObject: customer + column: c_customer_sk + resultType: float + aggregation: count + distinct: true + +# OBML metric (references the measures) +metrics: + customer_lifetime_value: + expression: "{[total_sales]} / {[_customer_c_customer_sk_count_distinct]}" +``` + +When a simple OSI metric (e.g., `SUM(store_sales.ss_ext_sales_price)`) is equivalent to an existing named measure, the converter deduplicates and reuses the named measure rather than creating a redundant auto-measure. + +### 2.4 AI Context Preservation + +OSI's `ai_context` (instructions, synonyms, examples) is preserved losslessly during conversion via OBML's `customExtensions` mechanism: + +```yaml +# OSI input +ai_context: + synonyms: + - "sales transactions" + - "store purchases" + +# OBML output (via customExtensions) +customExtensions: + - vendor: OSI + data: '{"synonyms": ["sales transactions", "store purchases"]}' +``` + +This applies at all levels: datasets → data objects, fields → columns, and model-level `ai_context`. + +During OBML → OSI conversion, the `customExtensions` with `vendor: "OSI"` are read back and restored as native `ai_context` on the OSI side. + +### 2.5 OBML-Specific Features (Not Representable in OSI) + +These OBML features have no direct OSI equivalent. Where possible, metadata is preserved in OSI `ai_context` or `custom_extensions` (with `vendor_name: "COMMON"` and `obml_`-prefixed keys) for lossless roundtrip: + +- Secondary joins with `pathName` (preserved in relationship `ai_context`) +- `allowFanOut` — preserved in metric `custom_extensions` (`obml_allow_fan_out`) +- Dynamic date filters (`dynamicDate`, `dynamicDateRange`) — not yet preserved +- `timeGrain` on dimensions — preserved in field `custom_extensions` (`obml_time_grain`) +- Dimension `format` — preserved in field `custom_extensions` (`obml_dimension_format`) +- Measure filters — preserved in metric `custom_extensions` (`obml_filters`) +- Measure `total` — preserved in metric `custom_extensions` (`obml_total`) +- Measure `format` — preserved in metric `custom_extensions` (`obml_format`) +- Measure `delimiter` — preserved in metric `custom_extensions` (`obml_delimiter`) +- Measure `withinGroup` — preserved in metric `custom_extensions` (`obml_within_group`) +- Metric `format` — preserved in metric `custom_extensions` (`obml_format`) +- Locale settings — not yet preserved +- `abstractType` (OBML type system) — preserved in field `custom_extensions` (`obml_abstract_type`) + +### 2.6 OSI-Specific Features and How They Map to OBML + +- **`primary_key`** — natively represented: OSI's dataset-level `primary_key` array maps to per-column `primaryKey: true` on OBML columns (`DataObjectColumn.primaryKey`), and back to the dataset array on export. +- **`unique_keys`** — no native OBML equivalent; round-trips via an `OSI`-vendor `customExtension` (`obml_unique_keys`). +- **Multi-dialect expressions** — on import the converter reads the first available SQL dialect in the order `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`; non-SQL dialects (`MDX`, `TABLEAU`, `MAQL`) are not parsed. A metric with no SQL-parseable dialect, or an expression OBML cannot decompose, is preserved verbatim (`obml_unconverted_metrics`) with a `LOSSY:` warning rather than dropped. On export, OBML measures/metrics emit `ANSI_SQL`. +- **`ai_context`** — preserved losslessly via `customExtensions` (see Section 2.4). +- **`custom_extensions`** — mapped to OBML `customExtensions`. + +## 3. Conversion Strategies + +### 3.1 OSI → OBML + +1. Parse `source` string to extract `database`, `schema`, and `table` +2. Convert fields to columns with type inference (heuristic-based `abstractType`) +3. Restructure global relationships into inline joins on data objects +4. Decompose metric SQL expressions into OBML measures + metrics +5. Extract dimension-flagged fields into the top-level `dimensions` section (excluding FK/PK join keys) +6. Preserve `ai_context` losslessly via `customExtensions` (vendor: `"OSI"`) + +### 3.2 OBML → OSI + +1. Combine `database.schema.code` into the OSI `source` string +2. Convert columns to fields with `ANSI_SQL` dialect expressions +3. Extract inline joins into global relationships with generated names +4. Convert measures to OSI metrics with SQL expressions +5. Expand metric templates by substituting measure SQL into `{[Name]}` references +6. Map OBML dimension metadata into `field.dimension.is_time` flags +7. Preserve secondary join info in relationship `ai_context` +8. Store OBML-specific type info in `custom_extensions` with `vendor_name: "COMMON"` + +## 4. Validation + +The converter includes dual-layer validation for both formats, ensuring that converted output is structurally and semantically correct. + +### 4.1 OBML Validation + +1. **JSON Schema** — validates against `schema/obml-schema.json` (Draft 7) +2. **Semantic** — runs OrionBelt's `ReferenceResolver` + `SemanticValidator` (reference integrity, cycle detection, multipath detection, duplicate identifiers) + +### 4.2 OSI Validation + +1. **JSON Schema** — validates against `osi-schema.json` (Draft 2020-12) +2. **Unique names** — checks uniqueness of dataset, field, metric, and relationship names +3. **References** — verifies that relationship `from`/`to` reference existing datasets + +Validation runs automatically after each conversion. Use `--no-validate` to skip. + +## 5. Converter Usage + +### CLI + +A single `osi-orionbelt` command with two subcommands is installed with the package: + +```bash +# OSI → OBML +osi-orionbelt osi-to-obml -i tpcds_osi.yaml -o tpcds_as_obml.yaml + +# OBML → OSI +osi-orionbelt obml-to-osi -i tpcds_as_obml.yaml -o tpcds_obml_as_osi.yaml \ + --model-name tpcds_retail_model \ + --description "TPC-DS retail semantic model" + +# OBML → OSI ontology document +osi-orionbelt obml-to-osi --ontology -i tpcds_as_obml.yaml -o tpcds_ontology.yaml + +# Skip validation +osi-orionbelt osi-to-obml -i input.yaml -o output.yaml --no-validate +``` + +### CLI Options + +| Subcommand / Option | Description | +|---|---| +| `osi-to-obml` | Convert OSI → OBML | +| `obml-to-osi` | Convert OBML → OSI | +| `--ontology` | (`obml-to-osi`) emit an OSI ontology document instead of core-spec | +| `-i`, `--input` | Input file (required) | +| `-o`, `--output` | Output file (required) | +| `--model-name` | Model name for OBML → OSI | +| `--description` | Model description for OBML → OSI | +| `--ai-instructions` | AI instructions for OBML → OSI | +| `--database` | Default database for OSI → OBML (default: `ANALYTICS`) | +| `--schema` | Default schema for OSI → OBML (default: `PUBLIC`) | +| `--no-validate` | Skip post-conversion validation | + +### Python API + +```python +from osi_orionbelt import OSItoOBML, OBMLtoOSI, validate_obml, validate_osi + +# OSI → OBML +converter = OSItoOBML(osi_dict) +obml = converter.convert() +result = validate_obml(obml) +assert result.valid + +# OBML → OSI +converter = OBMLtoOSI(obml_dict, model_name="my_model") +osi = converter.convert() +result = validate_osi(osi) +assert result.valid +``` + +## 6. Example: TPC-DS Roundtrip + +The converter is validated against the official [TPC-DS example](https://github.com/open-semantic-interchange/OSI/blob/main/examples/tpcds_semantic_model.yaml) from the OSI repository. That file is vendored at `tests/fixtures/tpcds_semantic_model.yaml` and exercised by `tests/test_osi_tpcds_baseline.py`, which runs the OSI converters guide's [conceptual conversion flow](https://github.com/open-semantic-interchange/OSI/blob/main/converters/index.md#example-conceptual-conversion-flow) end to end: OSI to OBML to OSI, asserting validity at each step and that the example's `SALESFORCE` and `DBT` custom extensions survive the round-trip (step 7). + +### OSI → OBML + +The TPC-DS OSI model with 5 datasets, 4 relationships, and 5 metrics converts cleanly to OBML: + +- 5 data objects with inline joins +- 16 dimensions (FK/PK join keys excluded) +- 5 measures (3 direct + 2 auto-generated for metric decomposition) +- 2 metrics (composite expressions referencing measures) +- All `ai_context` synonyms preserved via `customExtensions` + +### OBML → OSI (Roundtrip) + +Converting the OBML output back to OSI produces a valid OSI model where: + +- All `ai_context` synonyms are restored from `customExtensions` +- Measures are re-expanded into SQL metric expressions +- Inline joins are extracted back into global relationships +- OBML type information is preserved in `custom_extensions` + +### Files + +| File | Description | +|---|---| +| `tests/fixtures/tpcds_osi.yaml` | Official TPC-DS OSI example (from OSI repo) | +| `tests/fixtures/tpcds_as_obml.yaml` | Converted OBML output | +| `src/osi_orionbelt/schemas/osi-schema.json` | OSI JSON Schema (Draft 2020-12, from OSI repo) | +| `src/osi_orionbelt/converter.py` | Bidirectional converter with validation | + +## 7. Future Considerations + +- **MCP/API integration** — Expose OSI import/export as OrionBelt MCP tools or REST API endpoints +- **Multi-dialect support** — Preserve non-ANSI dialect expressions during roundtrip +- **Primary keys in OBML** — Add optional `primary_key` to data objects for richer metadata +- **Vendor enum** — Register `"ORIONBELT"` as an OSI vendor for OBML-specific extensions diff --git a/converters/orionbelt/osi_obml_ontology_mapping_analysis.md b/converters/orionbelt/osi_obml_ontology_mapping_analysis.md new file mode 100644 index 0000000..1887b85 --- /dev/null +++ b/converters/orionbelt/osi_obml_ontology_mapping_analysis.md @@ -0,0 +1,82 @@ +# OBML → OSI Ontology Mapping Analysis + +This pins the rules used by `OBMLtoOSIOntology` to derive an **OSI ontology +document** (validated against `src/osi_orionbelt/schemas/osi-ontology-schema.json`, OSI version +`0.2.0.dev0`) from an OBML semantic model. + +The OSI ontology is a **separate document** from the OSI core-spec semantic +model (different `$id`, different required root). It is produced alongside the +core export, never merged into it — OSI's own `validation/validate.py` validates +one document against one schema and reads only the first YAML document, so a +combined or multi-doc file is not portable. See the `include_ontology` flag on +the export endpoints. + +## Document shape produced + +```yaml +version: 0.2.0.dev0 +name: # required +description: # optional +ai_context: { instructions: ... } # optional, from ai_instructions / model +ontology: # required, minItems 1 + - concept: + name: # = OBML dataObject display name + type: EntityType + description: ... + relationships: # outgoing joins keyed by this entity + - name: _to_ + roles: [{ concept: }] # declaring concept (A) is the implicit first role + multiplicity: ManyToOne | OneToOne + verbalizes: ["{} relates to {}"] +ontology_mappings: + - name: _map + semantic_model: { ...full OSI core-spec model... } # reused from OBMLtoOSI + concept_mappings: + - concept: + object_mappings: [{ expression: "." }] + link_mappings: + - relationship: _to_ + object_mapping: { concept: , expression: "." } +``` + +## Mapping rules + +| OBML construct | OSI ontology target | Notes | +|----------------|---------------------|-------| +| `dataObject` | `EntityType` `Concept` (one per object) | name = display name (matches OSI dataset name for ref consistency) | +| `dataObject.description` / `.comment` | `concept.description` | first non-empty wins | +| `join` (A → B) | `Relationship` under A's component | declaring concept A is the implicit first role; B is an explicit `role` | +| `join.joinType` | `Relationship.multiplicity` | `many-to-one`→`ManyToOne`, `one-to-one`→`OneToOne` | +| `column.primaryKey` | entity `object_mappings[].expression` | `
.`; identifies the entity | +| `join.columnsFrom` (FK) | `link_mappings[].object_mapping.expression` | `.`; binds the relationship to its far role | +| whole core model | `ontology_mappings[].semantic_model` | embedded verbatim from `OBMLtoOSI.convert()` | + +`
` is the final identifier of the dataset `source` (e.g. `db.schema.t` → `t`), +falling back to the dataset name when `source` has no dotted physical table. + +## Gaps and warnings (emitted to `warnings`) + +| OBML construct | Handling | Reason | +|----------------|----------|--------| +| `joinType: many-to-many` | relationship **skipped** + warning | OSI `Multiplicity` enum is only `ManyToOne`/`OneToOne` | +| missing/unknown `joinType` | defaults to `ManyToOne` + warning | matches core-converter default | +| secondary / `pathName` joins | emitted as ordinary relationships + warning | OSI ontology has no named-alternate-path concept | +| composite primary/foreign keys | first column used + warning | `object_mapping.expression` is a single scalar SQL expression | +| measures / metrics | **not** in the ontology layer | live only in the embedded core `semantic_model`; ontology models entities/relationships | +| columns (non-key) as value concepts | not modeled (entities only) | keeps v1 valid and focused; ORM-style `ValueType` modeling deferred | +| `verbalizes` / `derived_by` / `requires` / `identify_by` | `verbalizes` emitted as a generated stub; others omitted | OBML has no native source for fact-based verbalization or derivation | + +## Validation + +`validate_osi_ontology()` runs: +1. JSON Schema (Draft 2020-12) against `osi-ontology-schema.json`. +2. Semantic checks: unique concept names; relationship `roles` reference defined + concepts; `concept_mappings` reference defined concepts. + +## Stability note + +OSI ontology is `0.2.0.dev0` (pre-1.0). This exporter is the supported, +schema-validated direction. An ontology **importer** (OSI ontology → OBML) is +intentionally deferred until OSI drops the `dev` pre-release suffix, because the +gaps above make the reverse direction lossy. Import the OSI **core spec** +instead (already supported). diff --git a/converters/orionbelt/pyproject.toml b/converters/orionbelt/pyproject.toml new file mode 100644 index 0000000..7aa309c --- /dev/null +++ b/converters/orionbelt/pyproject.toml @@ -0,0 +1,58 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "osi-orionbelt" +version = "0.1.0" +description = "Bidirectional OBML <-> OSI (Open Semantic Interchange) converter for OrionBelt semantic models" +license = { text = "Apache-2.0" } +readme = "README.md" +authors = [{ name = "Ralf Becher", email = "info@orionbelt.ai" }] +keywords = ["osi", "open-semantic-interchange", "obml", "semantic-layer", "orionbelt", "converter"] +requires-python = ">=3.12" +dependencies = [ + "pyyaml>=6.0", + "jsonschema>=4.18", + "referencing>=0.30", +] + +[project.optional-dependencies] +# Deep OBML semantic validation (cycles, duplicate names, invalid refs) via the +# full OrionBelt engine. Optional: validate_obml degrades to JSON-schema-only +# checks with a warning when this is not installed. +obml-validation = ["orionbelt-semantic-layer"] +dev = [ + "pytest>=8.0", + "mypy>=1.10", + "ruff>=0.4", + "types-jsonschema", + "types-PyYAML", +] + +[project.scripts] +osi-orionbelt = "osi_orionbelt.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/osi_orionbelt"] + +# All three schemas (the two OSI artefacts plus a vendored snapshot of the +# canonical OBML schema) live in src/osi_orionbelt/schemas/ as tracked files, +# so the sdist and wheel are self-contained and build in isolation. The OBML +# snapshot is kept in sync with the repo-root schema/obml-schema.json by a +# drift-guard test (tests/unit/test_osi_orionbelt_schema_sync.py). + +[tool.ruff] +line-length = 100 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "UP", "B", "A", "SIM"] + +[tool.mypy] +python_version = "3.12" +files = ["src/osi_orionbelt"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/converters/orionbelt/src/osi_orionbelt/__init__.py b/converters/orionbelt/src/osi_orionbelt/__init__.py new file mode 100644 index 0000000..9bccf42 --- /dev/null +++ b/converters/orionbelt/src/osi_orionbelt/__init__.py @@ -0,0 +1,41 @@ +"""osi-orionbelt: bidirectional OBML <-> OSI converter. + +Converts between OrionBelt Markup Language (OBML) semantic models and Open +Semantic Interchange (OSI) models, in both directions, plus an OSI ontology +emitter. Validation helpers check OBML and OSI documents against their JSON +schemas. + +Public API: + OSItoOBML - convert an OSI model dict to OBML + OBMLtoOSI - convert an OBML model dict to OSI core-spec + OBMLtoOSIOntology - emit an OSI ontology document from an OBML model + validate_obml - validate an OBML model dict + validate_osi - validate an OSI model dict + validate_osi_ontology - validate an OSI ontology document dict + ValidationResult - structured validation result +""" + +from __future__ import annotations + +from osi_orionbelt.converter import ( + OBMLtoOSI, + OBMLtoOSIOntology, + OSItoOBML, + ValidationResult, + validate_obml, + validate_osi, + validate_osi_ontology, +) + +__version__ = "0.1.0" + +__all__ = [ + "OBMLtoOSI", + "OBMLtoOSIOntology", + "OSItoOBML", + "ValidationResult", + "validate_obml", + "validate_osi", + "validate_osi_ontology", + "__version__", +] diff --git a/converters/orionbelt/src/osi_orionbelt/cli.py b/converters/orionbelt/src/osi_orionbelt/cli.py new file mode 100644 index 0000000..351d064 --- /dev/null +++ b/converters/orionbelt/src/osi_orionbelt/cli.py @@ -0,0 +1,155 @@ +"""Command-line entry point for the OBML <-> OSI converter. + +A single ``osi-orionbelt`` command with two format-named subcommands, mirroring +the OSI converter convention (e.g. ``osi-dbt msi-to-osi``): + + osi-orionbelt obml-to-osi -i model.obml.yaml -o model.osi.yaml + osi-orionbelt obml-to-osi --ontology -i model.obml.yaml -o model.ontology.yaml + osi-orionbelt osi-to-obml -i model.osi.yaml -o model.obml.yaml + +Both subcommands print conversion warnings and a validation summary to stderr, +and exit non-zero when the produced document fails schema validation (unless +``--no-validate``). +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import Any + +import yaml + +from osi_orionbelt.converter import ( + OBMLtoOSI, + OBMLtoOSIOntology, + OSItoOBML, + validate_obml, + validate_osi, + validate_osi_ontology, +) + + +def _load(input_path: str) -> dict[str, Any]: + data = yaml.safe_load(Path(input_path).read_text()) + if not isinstance(data, dict): + print(f"Error: {input_path} is not a YAML mapping", file=sys.stderr) + raise SystemExit(2) + return data + + +def _emit(result: dict[str, Any], output: str) -> None: + output_yaml = yaml.dump( + result, default_flow_style=False, allow_unicode=True, sort_keys=False, width=120 + ) + Path(output).write_text(output_yaml) + print(f"Written to {output}", file=sys.stderr) + + +def _print_warnings(warnings: list[str]) -> None: + if warnings: + print("\nConversion warnings:", file=sys.stderr) + for w in warnings: + print(f" - {w}", file=sys.stderr) + + +def _report_validation(label: str, result: dict[str, Any], validate_fn: Any) -> bool: + """Validate ``result`` and print a summary. Return True if there are errors.""" + print(f"\nValidating {label}...", file=sys.stderr) + vr = validate_fn(result) + for line in vr.summary_lines(): + print(line, file=sys.stderr) + if vr.valid: + print(f"{label} is valid", file=sys.stderr) + return False + print(f"{label} has validation errors", file=sys.stderr) + return True + + +def _cmd_obml_to_osi(args: argparse.Namespace) -> int: + """OBML -> OSI core-spec (or, with --ontology, OSI ontology).""" + data = _load(args.input) + + validate_fn: Any + if args.ontology: + converter: Any = OBMLtoOSIOntology( + data, args.model_name, args.description, args.ai_instructions + ) + result = converter.convert() + validate_fn, label = validate_osi_ontology, "OSI ontology output" + else: + converter = OBMLtoOSI(data, args.model_name, args.description, args.ai_instructions) + result = converter.convert() + validate_fn, label = validate_osi, "OSI output" + + _emit(result, args.output) + _print_warnings(converter.warnings) + + if args.no_validate: + return 0 + return 1 if _report_validation(label, result, validate_fn) else 0 + + +def _cmd_osi_to_obml(args: argparse.Namespace) -> int: + """OSI core-spec -> OBML.""" + data = _load(args.input) + + converter = OSItoOBML(data, args.database, args.schema) + result = converter.convert() + + _emit(result, args.output) + _print_warnings(converter.warnings) + + if args.no_validate: + return 0 + return 1 if _report_validation("OBML output", result, validate_obml) else 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="osi-orionbelt", + description="Convert between OrionBelt OBML and OSI YAML.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + o2s = subparsers.add_parser("obml-to-osi", help="Convert OBML YAML → OSI YAML") + o2s.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to OBML YAML") + o2s.add_argument( + "-o", "--output", required=True, metavar="FILE", help="Path for output OSI YAML" + ) + o2s.add_argument( + "--ontology", action="store_true", help="Emit an OSI ontology document instead of core-spec" + ) + o2s.add_argument( + "--model-name", default="semantic_model", metavar="NAME", help="OSI semantic model name" + ) + o2s.add_argument("--description", default="", metavar="TEXT", help="OSI model description") + o2s.add_argument( + "--ai-instructions", default="", metavar="TEXT", help="OSI ai_context instructions" + ) + o2s.add_argument("--no-validate", action="store_true", help="Skip output validation") + + s2o = subparsers.add_parser("osi-to-obml", help="Convert OSI YAML → OBML YAML") + s2o.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to OSI YAML") + s2o.add_argument( + "-o", "--output", required=True, metavar="FILE", help="Path for output OBML YAML" + ) + s2o.add_argument( + "--database", default="ANALYTICS", metavar="NAME", help="Default database for OBML output" + ) + s2o.add_argument( + "--schema", default="PUBLIC", metavar="NAME", help="Default schema for OBML output" + ) + s2o.add_argument("--no-validate", action="store_true", help="Skip output validation") + + args = parser.parse_args(argv) + if args.command == "obml-to-osi": + return _cmd_obml_to_osi(args) + if args.command == "osi-to-obml": + return _cmd_osi_to_obml(args) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/converters/orionbelt/src/osi_orionbelt/converter.py b/converters/orionbelt/src/osi_orionbelt/converter.py new file mode 100644 index 0000000..6ecf64d --- /dev/null +++ b/converters/orionbelt/src/osi_orionbelt/converter.py @@ -0,0 +1,2967 @@ +#!/usr/bin/env python3 +""" +OSI ↔ OBML Bidirectional Converter +=================================== +Converts between Open Semantic Interchange (OSI v0.2.0.dev0) YAML models +and OrionBelt Markup Language (OBML v1.0) YAML models. + +OSI v0.1.1 inputs are still accepted on read — the legacy shim +``_normalize_legacy_v01`` promotes pre-v0.2 custom_extensions into the +v0.2 first-class fields before regular parsing runs. + +Author: OrionBelt / RALFORION +""" + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any + +import yaml + +# ─── Spec version pin ─────────────────────────────────────────────────────── +# Single source of truth for the OSI spec we emit. Bump when upstream cuts +# a stable v0.2.0 (drop the ``.dev0`` suffix). All read paths accept both +# 0.1.x (via the legacy shim) and 0.2.x. +_OSI_VERSION = "0.2.0.dev0" + +# Dialect / vendor enum extras new in v0.2.0.dev0 +_OSI_KNOWN_DIALECTS = ("ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL") +# SQL dialects (of the OSI enum) whose aggregation expressions our regex-based +# metric parser can read, in preference order. ANSI_SQL first; SNOWFLAKE and +# DATABRICKS are SQL engines OrionBelt also targets, and their simple/expression +# aggregations (``SUM(t.c)``, ``SUM(t.a * t.b)``) are syntactically identical to +# ANSI. MDX / TABLEAU / MAQL are non-SQL languages and are never parsed as SQL. +_SQL_PARSEABLE_DIALECTS = ("ANSI_SQL", "SNOWFLAKE", "DATABRICKS") + +# Matches a ``dataset.column`` reference inside a SQL expression, where each +# side is a bare identifier or a quoted identifier (double quotes, backticks, or +# brackets). The leading lookbehind prevents matching the tail of a longer path +# (``a.b.c``) or a mid-token boundary; the bare form must start with a letter or +# underscore so numeric literals (``1.5``) are never treated as references. +_COLUMN_REF_RE = re.compile( + r'(?[A-Za-z_]\w*|"[^"]+"|`[^`]+`|\[[^\]]+\])' + r"\s*\.\s*" + r'(?P[A-Za-z_]\w*|"[^"]+"|`[^`]+`|\[[^\]]+\])' +) +_OSI_KNOWN_VENDORS = ( + "COMMON", + "ORIONBELT", + "SNOWFLAKE", + "SALESFORCE", + "DBT", + "DATABRICKS", + "GOODDATA", +) + +# Vendor identities for custom_extensions. +# ORIONBELT - OrionBelt/OBML-proprietary payloads we author on OBML -> OSI. +# OSI - OSI-native fields OBML can't hold (unique_keys, field label, +# ai_context leftovers), stashed into OBML on OSI -> OBML. +# Read paths also accept the legacy tags we emitted before this scheme so older +# documents still round-trip; foreign vendors (SNOWFLAKE, DBT, ...) are +# preserved verbatim, never relabelled. +_VENDOR_OBML = "ORIONBELT" +_VENDOR_OSI = "OSI" +_OBML_VENDOR_READ = ("ORIONBELT", "COMMON") +_OSI_VENDOR_READ = ("OSI", "OBSL") +# Vendors the converter handles internally (its own payloads + native-field +# stashes). Any custom_extension from a vendor outside this set is third-party +# and is carried through verbatim in both directions, never relabelled. +_INTERNAL_VENDORS = frozenset({"ORIONBELT", "COMMON", "OSI", "OBSL"}) + +# ─── Type mapping ─────────────────────────────────────────────────────────── + +OBML_TO_OSI_TYPE = { + "string": "string", + "json": "string", + "int": "integer", + "float": "number", + "date": "date", + "time": "time", + "time_tz": "time", + "timestamp": "timestamp", + "timestamp_tz": "timestamp", + "boolean": "boolean", +} + +OSI_TO_OBML_TYPE = { + "string": "string", + "integer": "int", + "number": "float", + "date": "date", + "time": "time", + "timestamp": "timestamp", + "boolean": "boolean", +} + + +# ═══════════════════════════════════════════════════════════════════════════ +# OSI → OBML Converter +# ═══════════════════════════════════════════════════════════════════════════ + + +class OSItoOBML: + """Convert an OSI semantic model YAML to OBML format.""" + + def __init__( + self, osi: dict, default_database: str = "ANALYTICS", default_schema: str = "PUBLIC" + ): + self.osi = osi + self.default_database = default_database + self.default_schema = default_schema + self.warnings: list[str] = [] + # OSI metrics that have no OBML representation (non-SQL dialect only, + # or an expression our parser cannot decompose). Preserved verbatim + # rather than dropped — see ``_preserve_unconverted_metric``. + self._unconverted_metrics: list[dict] = [] + + def _normalize_legacy_v01(self) -> None: + """Promote OSI v0.1.x payloads to the v0.2 shape, in place. + + The v0.2 spec promotes ``primary_key`` and ``unique_keys`` to + first-class dataset fields. v0.1.x serializers (including ours + pre-bump) stash both under ``custom_extensions`` with vendor + ``OBSL`` and keys ``obml_primary_key`` / ``obml_unique_keys``. + This shim runs before parsing so the rest of the converter can + assume v0.2 shape regardless of input version. + + No-op for documents that already declare ``version`` >= 0.2 or + that have nothing to migrate. + """ + version = str(self.osi.get("version", "")) + if version and not version.startswith(("0.1", "0.0")): + return # already v0.2+ (or future) — nothing to do + + models = self.osi.get("semantic_model", []) + if not isinstance(models, list): + return + + for model in models: + for ds in model.get("datasets", []) or []: + # Promote legacy primary_key / unique_keys from OBSL extras + # only if the dataset doesn't already declare them. + legacy = self._extract_obml_extras(ds) + if not legacy: + continue + if "primary_key" not in ds and legacy.get("obml_primary_key"): + pk = legacy["obml_primary_key"] + if isinstance(pk, list) and all(isinstance(c, str) for c in pk): + ds["primary_key"] = list(pk) + if "unique_keys" not in ds and legacy.get("obml_unique_keys"): + uk = legacy["obml_unique_keys"] + if isinstance(uk, list) and all( + isinstance(g, list) and all(isinstance(c, str) for c in g) for g in uk + ): + ds["unique_keys"] = [list(g) for g in uk] + + if version.startswith(("0.0", "0.1")): + self.warnings.append( + f"OSI input declares version '{version}'; legacy v0.1.x " + f"compatibility shim applied. Output target is v{_OSI_VERSION}." + ) + + def convert(self) -> dict: + # Reset per-conversion accumulators so calling convert() twice on the + # same instance is idempotent (no duplicated warnings or preserved + # metrics). Both are populated as a side effect of conversion below. + self.warnings = [] + self._unconverted_metrics = [] + + # v0.1.x inputs need the legacy shim to promote pre-v0.2 + # custom_extensions into v0.2 first-class fields before we parse. + self._normalize_legacy_v01() + + models = self.osi.get("semantic_model", []) + if not models: + raise ValueError("No semantic_model found in OSI input") + + # Take the first semantic model (OBML is a single-model format) + model = models[0] + if len(models) > 1: + self.warnings.append( + f"OSI contains {len(models)} semantic models; " + f"only the first ('{model.get('name')}') is converted." + ) + + obml: dict[str, Any] = {"version": 1.0} + + # ── Model description ───────────────────────────────────── + if model.get("description"): + obml["description"] = model["description"] + + # ── DataObjects ───────────────────────────────────────────── + datasets = model.get("datasets", []) + relationships = model.get("relationships", []) + + # Build lookup: dataset_name → dataset + ds_map = {ds["name"]: ds for ds in datasets} + + # Build relationship index: from_dataset → [relationship, ...] + rel_by_from: dict[str, list] = {} + for rel in relationships: + rel_by_from.setdefault(rel["from"], []).append(rel) + + # Collect join key columns: (dataset_name, field_name) pairs + # These should NOT become dimensions (they are FK/PK join keys) + self._join_key_columns: set[tuple[str, str]] = set() + for rel in relationships: + for col in rel.get("from_columns", []): + self._join_key_columns.add((rel["from"], col)) + for col in rel.get("to_columns", []): + self._join_key_columns.add((rel["to"], col)) + + data_objects: dict[str, Any] = {} + for ds in datasets: + do_name, do_obj = self._convert_dataset(ds, rel_by_from) + data_objects[do_name] = do_obj + + obml["dataObjects"] = data_objects + + # ── Dimensions (extracted from OSI fields with dimension metadata) ── + dimensions = self._extract_dimensions(datasets) + if dimensions: + obml["dimensions"] = dimensions + + # ── Measures & Metrics ────────────────────────────────────── + osi_metrics = model.get("metrics", []) + measures, metrics = self._convert_metrics(osi_metrics, ds_map) + if measures: + obml["measures"] = measures + if metrics: + obml["metrics"] = metrics + + # Metrics that have no OBML representation are not dropped: stash the + # original OSI metric verbatim under the OSI vendor so the reverse + # (OBML -> OSI) direction re-emits them and a full OSI -> OBML -> OSI + # roundtrip stays lossless. They are not queryable in OBML; a LOSSY + # warning was already recorded per metric. + if self._unconverted_metrics: + obml.setdefault("customExtensions", []).append( + { + "vendor": _VENDOR_OSI, + "data": json.dumps({"obml_unconverted_metrics": self._unconverted_metrics}), + } + ) + + # ── Restore model-level properties from custom_extensions ──── + for ext in model.get("custom_extensions", []): + if ext.get("vendor_name") in _OBML_VENDOR_READ: + try: + ext_data = json.loads(ext.get("data", "{}")) + if ext_data.get("obml_filters"): + obml["filters"] = ext_data["obml_filters"] + if ext_data.get("obml_settings"): + obml["settings"] = ext_data["obml_settings"] + if ext_data.get("obml_owner"): + obml["owner"] = ext_data["obml_owner"] + except (json.JSONDecodeError, TypeError): + pass + break + + # Preserve third-party vendor extensions verbatim + self._carry_foreign_extensions(model.get("custom_extensions"), obml) + + return obml + + @staticmethod + def _carry_foreign_extensions(osi_exts: list[dict] | None, obml_target: dict[str, Any]) -> None: + """Carry third-party OSI custom_extensions verbatim into OBML. + + Our own payloads and OSI-native stashes are reconstructed elsewhere; + any other vendor's extension is preserved unchanged on the OBML side + so a full OSI -> OBML -> OSI roundtrip keeps the original vendor. + """ + for ext in osi_exts or []: + vendor = ext.get("vendor_name") + if vendor and vendor not in _INTERNAL_VENDORS: + obml_target.setdefault("customExtensions", []).append( + {"vendor": vendor, "data": ext.get("data", "")} + ) + + def _parse_source(self, source: str) -> tuple[str, str, str]: + """Parse 'database.schema.table' into parts.""" + parts = source.split(".") + if len(parts) == 3: + return parts[0], parts[1], parts[2] + elif len(parts) == 2: + return self.default_database, parts[0], parts[1] + else: + return self.default_database, self.default_schema, parts[0] + + def _convert_dataset(self, ds: dict, rel_by_from: dict) -> tuple[str, dict]: + """Convert an OSI dataset to an OBML dataObject. + + Uses the exact OSI dataset name as the OBML data object key. + """ + name = ds["name"] + + source = ds.get("source", name) + database, schema, table = self._parse_source(source) + + do: dict[str, Any] = { + "code": table, + "database": database, + "schema": schema, + } + + # ── Columns ───────────────────────────────────────────────── + columns: dict[str, Any] = {} + fields = ds.get("fields", []) + for field in fields: + col_name, col_obj = self._convert_field(field) + columns[col_name] = col_obj + + # ── Primary key flag propagation (OSI v0.2 first-class) ── + # ``primary_key`` lists physical column codes; mark every matching + # column with ``primaryKey: true``. Unknown PK columns surface as + # a warning (the spec couples PK to relationship cardinality, so + # silently dropping is unsafe). + pk_codes = ds.get("primary_key") or [] + if pk_codes: + code_to_col = {col.get("code"): (cname, col) for cname, col in columns.items()} + unknown_pks: list[str] = [] + for pk_code in pk_codes: + hit = code_to_col.get(pk_code) + if hit is None: + unknown_pks.append(pk_code) + continue + _, col = hit + col["primaryKey"] = True + if unknown_pks: + self.warnings.append( + f"Dataset '{name}' primary_key references unknown columns: " + f"{unknown_pks}. Ignored." + ) + + if columns: + do["columns"] = columns + else: + self.warnings.append(f"Dataset '{name}' has no fields; adding placeholder column.") + do["columns"] = {f"{name}_id": {"code": f"{table}_id", "abstractType": "string"}} + + # ── Joins (from relationships where this dataset is on 'from' side) ── + joins = [] + for rel in rel_by_from.get(name, []): + join_obj = self._convert_relationship_to_join(rel) + joins.append(join_obj) + + if joins: + do["joins"] = joins + + # ── Description (semantic, from OSI) ───────────────────────── + if ds.get("description"): + do["description"] = ds["description"] + + # ── Extract ai_context: synonyms → native, rest → customExtensions ─ + ai_ctx = ds.get("ai_context") + if ai_ctx: + ai_data = ai_ctx if isinstance(ai_ctx, dict) else {"instructions": ai_ctx} + # Extract synonyms directly into OBML synonyms property + if "synonyms" in ai_data: + do["synonyms"] = list(ai_data["synonyms"]) + # Store remaining ai_context keys in customExtensions + remaining = {k: v for k, v in ai_data.items() if k != "synonyms"} + if remaining: + do["customExtensions"] = [ + { + "vendor": "OSI", + "data": json.dumps(remaining), + } + ] + + # Restore DataObject owner / comment / refresh from custom_extensions + for ext in ds.get("custom_extensions", []): + if ext.get("vendor_name") in _OBML_VENDOR_READ: + try: + ext_data = json.loads(ext.get("data", "{}")) + if ext_data.get("obml_owner"): + do["owner"] = ext_data["obml_owner"] + if ext_data.get("obml_comment"): + do["comment"] = ext_data["obml_comment"] + if ext_data.get("obml_refresh"): + do["refresh"] = ext_data["obml_refresh"] + except (json.JSONDecodeError, TypeError): + pass + break + + # ── Unique keys roundtrip (OBML has no native concept) ── + # Persist the OSI ``unique_keys`` array into the OBSL-vendor + # customExtensions so the OBML → OSI direction can emit it back. + unique_keys = ds.get("unique_keys") or [] + if unique_keys: + do.setdefault("customExtensions", []).append( + { + "vendor": _VENDOR_OSI, + "data": json.dumps({"obml_unique_keys": [list(g) for g in unique_keys]}), + } + ) + + # Preserve third-party vendor extensions verbatim + self._carry_foreign_extensions(ds.get("custom_extensions"), do) + + return name, do + + def _convert_field(self, field: dict) -> tuple[str, dict]: + """Convert an OSI field to an OBML column. + + Uses the exact OSI field name as the OBML column key. + """ + name = field["name"] + + # Get expression (prefer ANSI_SQL dialect) + expr_obj = field.get("expression", {}) + code = name # fallback + if isinstance(expr_obj, dict): + dialects = expr_obj.get("dialects", []) + for d in dialects: + if d.get("dialect") == "ANSI_SQL": + code = d.get("expression", name) + break + if not dialects: + code = name + elif code == name and dialects: + code = dialects[0].get("expression", name) + + # Determine abstract type: prefer explicit data_type, fall back to heuristic + osi_type = field.get("data_type", "") + if osi_type and osi_type in OSI_TO_OBML_TYPE: + abstract_type = OSI_TO_OBML_TYPE[osi_type] + else: + abstract_type = self._infer_obml_type(field) + + col: dict[str, Any] = { + "code": code, + "abstractType": abstract_type, + } + + if field.get("description"): + col["description"] = field["description"] + + # Extract field-level ai_context: synonyms → native, rest → customExtensions + ai_ctx = field.get("ai_context") + if ai_ctx: + ai_data = ai_ctx if isinstance(ai_ctx, dict) else {"instructions": ai_ctx} + # Extract synonyms directly into OBML synonyms property + if "synonyms" in ai_data: + col["synonyms"] = list(ai_data["synonyms"]) + # Store remaining ai_context keys in customExtensions + remaining = {k: v for k, v in ai_data.items() if k != "synonyms"} + if remaining: + col["customExtensions"] = [ + { + "vendor": "OSI", + "data": json.dumps(remaining), + } + ] + + # Restore OBML-only column properties from custom_extensions + for ext in field.get("custom_extensions", []): + if ext.get("vendor_name") in _OBML_VENDOR_READ: + try: + ext_data = json.loads(ext.get("data", "{}")) + if ext_data.get("obml_sql_type"): + col["sqlType"] = ext_data["obml_sql_type"] + if ext_data.get("obml_sql_precision") is not None: + col["sqlPrecision"] = ext_data["obml_sql_precision"] + if ext_data.get("obml_sql_scale") is not None: + col["sqlScale"] = ext_data["obml_sql_scale"] + if ext_data.get("obml_num_class"): + col["numClass"] = ext_data["obml_num_class"] + if ext_data.get("obml_comment"): + col["comment"] = ext_data["obml_comment"] + if ext_data.get("obml_owner"): + col["owner"] = ext_data["obml_owner"] + except (json.JSONDecodeError, TypeError): + pass + break + + # ── Field label roundtrip (OSI v0.2 first-class) ── + # OBML has no native column label today; preserve via OBSL-vendor + # customExtensions so the reverse direction can emit it back. + if field.get("label"): + col.setdefault("customExtensions", []).append( + { + "vendor": _VENDOR_OSI, + "data": json.dumps({"obml_field_label": field["label"]}), + } + ) + + # Preserve third-party vendor extensions verbatim + self._carry_foreign_extensions(field.get("custom_extensions"), col) + + return name, col + + def _infer_obml_type(self, field: dict) -> str: + """Infer OBML abstractType from OSI field metadata.""" + + dim = field.get("dimension", {}) + if isinstance(dim, dict) and dim.get("is_time"): + return "date" + + name_lower = field.get("name", "").lower() + + # Helper: match keywords at word boundaries to avoid false positives + # (e.g. "country" should NOT match "count") + def _has_keyword(keywords: tuple[str, ...]) -> bool: + for kw in keywords: + if kw.startswith("_") or kw.endswith("_"): + # Substring match for prefix/suffix patterns like "_sk", "is_" + if kw in name_lower: + return True + else: + # Word-boundary match for standalone keywords + if re.search(r"(?:^|_)" + re.escape(kw) + r"(?:$|_)", name_lower): + return True + return False + + if _has_keyword( + ( + "_sk", + "_id", + "_key", + "name", + "desc", + "email", + "address", + "city", + "state", + "zip", + "phone", + "status", + "type", + "category", + "class", + ) + ): + return "string" + if _has_keyword( + ( + "price", + "cost", + "amount", + "sales", + "profit", + "revenue", + "tax", + "discount", + "rate", + "percent", + "ratio", + "margin", + ) + ): + return "float" + if _has_keyword(("qty", "quantity", "count", "num", "number", "cnt")): + return "int" + if _has_keyword(("date", "time", "year", "month", "day", "week")): + return "date" + if _has_keyword(("flag", "is_", "has_")): + return "boolean" + + return "string" + + # OSI relationship type → OBML joinType mapping + _REL_TYPE_MAP: dict[str, str] = { + "many_to_one": "many-to-one", + "many-to-one": "many-to-one", + "one_to_many": "one-to-many", + "one-to-many": "one-to-many", + "one_to_one": "one-to-one", + "one-to-one": "one-to-one", + "many_to_many": "many-to-many", + "many-to-many": "many-to-many", + } + + def _convert_relationship_to_join(self, rel: dict) -> dict: + """Convert an OSI relationship to an OBML join. + + Uses exact OSI names for joinTo and column references. + Maps OSI relationship 'type' to OBML joinType if present, + defaults to many-to-one with a warning otherwise. + """ + rel_type = rel.get("type", "") + join_type = self._REL_TYPE_MAP.get(rel_type.lower(), "") if rel_type else "" + if not join_type: + join_type = "many-to-one" + if rel_type: + self.warnings.append( + f"Relationship '{rel.get('name', '?')}': unknown type " + f"'{rel_type}', defaulting to many-to-one." + ) + else: + self.warnings.append( + f"Relationship '{rel.get('name', '?')}': no type specified, " + f"defaulting to many-to-one." + ) + + join: dict[str, Any] = { + "joinType": join_type, + "joinTo": rel["to"], + "columnsFrom": list(rel["from_columns"]), + "columnsTo": list(rel["to_columns"]), + } + return join + + def _extract_dimensions(self, datasets: list) -> dict: + """Extract dimension definitions from OSI fields marked as dimensions. + + Skips fields that are join keys (FK/PK columns used in relationships), + since those are structural and not analytical dimensions. + """ + dimensions: dict[str, Any] = {} + for ds in datasets: + ds_name = ds["name"] + for field in ds.get("fields", []): + dim = field.get("dimension") + if dim is None: + continue + field_name = field["name"] + # Skip join key columns — they are FK/PK, not analytical dims + if (ds_name, field_name) in self._join_key_columns: + continue + abstract_type = self._infer_obml_type(field) + dim_def: dict[str, Any] = { + "dataObject": ds_name, + "column": field_name, + "resultType": abstract_type, + } + # Extract synonyms from field-level ai_context + ai_ctx = field.get("ai_context") + if isinstance(ai_ctx, dict) and ai_ctx.get("synonyms"): + dim_def["synonyms"] = list(ai_ctx["synonyms"]) + # Restore OBML-only dimension properties from custom_extensions + for ext in field.get("custom_extensions", []): + if ext.get("vendor_name") in _OBML_VENDOR_READ: + try: + ext_data = json.loads(ext.get("data", "{}")) + if ext_data.get("obml_time_grain"): + dim_def["timeGrain"] = ext_data["obml_time_grain"] + if ext_data.get("obml_dimension_format"): + dim_def["format"] = ext_data["obml_dimension_format"] + if ext_data.get("obml_dimension_result_type"): + dim_def["resultType"] = ext_data["obml_dimension_result_type"] + if ext_data.get("obml_dimension_description"): + dim_def["description"] = ext_data["obml_dimension_description"] + if ext_data.get("obml_dimension_owner"): + dim_def["owner"] = ext_data["obml_dimension_owner"] + if ext_data.get("obml_dimension_via"): + dim_def["via"] = ext_data["obml_dimension_via"] + except (json.JSONDecodeError, TypeError): + pass + break + dimensions[field_name] = dim_def + return dimensions + + def _convert_metrics(self, osi_metrics: list, ds_map: dict) -> tuple[dict, dict]: + """ + Convert OSI metrics to OBML measures and metrics. + + OSI has a single 'metrics' concept with SQL expressions. + OBML separates 'measures' (simple aggregations on single columns) + from 'metrics' (cross-fact expressions referencing measures). + + Strategy: + - Simple single-aggregation metrics → OBML measures + - Aggregation over expression (e.g. SUM(a.x * a.y)) → expression measure + - Complex/multi-aggregation metrics → OBML metrics referencing auto-measures + """ + + measures: dict[str, Any] = {} + metrics: dict[str, Any] = {} + + # Case-insensitive dataset/field index for resolving SQL identifiers + # back to their canonical OSI names (Snowflake/Databricks expressions + # commonly upper-case or quote them). + ds_lc = {name.lower(): name for name in ds_map} + fields_lc = { + name: { + f["name"].lower(): f["name"] + for f in ds.get("fields", []) or [] + if isinstance(f, dict) and f.get("name") + } + for name, ds in ds_map.items() + } + + for m in osi_metrics: + name = m["name"] + + osi_description = m.get("description") + + # Extract synonyms from OSI ai_context + osi_ai_ctx = m.get("ai_context") + osi_synonyms: list[str] = [] + if isinstance(osi_ai_ctx, dict) and osi_ai_ctx.get("synonyms"): + osi_synonyms = list(osi_ai_ctx["synonyms"]) + + # Restore OBML-only properties from custom_extensions + obml_extras = self._extract_obml_extras(m) + + # Check for cumulative metric stored in custom_extensions + if obml_extras.get("obml_metric_type") == "cumulative": + cum_metric = self._reconstruct_cumulative_metric( + name, obml_extras, osi_description, osi_synonyms + ) + metrics[name] = cum_metric + continue + + # Check for period-over-period metric stored in custom_extensions + if obml_extras.get("obml_metric_type") == "period_over_period": + pop_metric = self._reconstruct_pop_metric( + name, obml_extras, osi_description, osi_synonyms + ) + metrics[name] = pop_metric + continue + + # Check for window metric (rank/lag/lead/ntile/first_value/last_value) + if obml_extras.get("obml_metric_type") == "window": + window_metric = self._reconstruct_window_metric( + name, obml_extras, osi_description, osi_synonyms + ) + metrics[name] = window_metric + continue + + # Engine-delegated aggregation (Databricks Metric View). Round-trip + # marker comes from the OBML → OSI direction; on input we restore + # ``aggregation: measure`` without touching the OSI expression + # (which is a literal ``MEASURE("[A-Za-z_]\w*|"[^"]+"|`[^`]+`|\[[^\]]+\])' +) +_OSI_KNOWN_VENDORS = ( + "COMMON", + "ORIONBELT", + "SNOWFLAKE", + "SALESFORCE", + "DBT", + "DATABRICKS", + "GOODDATA", +) + +# Vendor identities for custom_extensions. +# ORIONBELT - OrionBelt/OBML-proprietary payloads we author on OBML -> OSI. +# OSI - OSI-native fields OBML can't hold (unique_keys, field label, +# ai_context leftovers), stashed into OBML on OSI -> OBML. +# Read paths also accept the legacy tags we emitted before this scheme so older +# documents still round-trip; foreign vendors (SNOWFLAKE, DBT, ...) are +# preserved verbatim, never relabelled. +_VENDOR_OBML = "ORIONBELT" +_VENDOR_OSI = "OSI" +_OBML_VENDOR_READ = ("ORIONBELT", "COMMON") +_OSI_VENDOR_READ = ("OSI", "OBSL") +# Vendors the converter handles internally (its own payloads + native-field +# stashes). Any custom_extension from a vendor outside this set is third-party +# and is carried through verbatim in both directions, never relabelled. +_INTERNAL_VENDORS = frozenset({"ORIONBELT", "COMMON", "OSI", "OBSL"}) + +# ─── Type mapping ─────────────────────────────────────────────────────────── + +OBML_TO_OSI_TYPE = { + "string": "string", + "json": "string", + "int": "integer", + "float": "number", + "date": "date", + "time": "time", + "time_tz": "time", + "timestamp": "timestamp", + "timestamp_tz": "timestamp", + "boolean": "boolean", +} + +OSI_TO_OBML_TYPE = { + "string": "string", + "integer": "int", + "number": "float", + "date": "date", + "time": "time", + "timestamp": "timestamp", + "boolean": "boolean", +} diff --git a/converters/orionbelt/src/osi_orionbelt/converter.py b/converters/orionbelt/src/osi_orionbelt/converter.py index 6ecf64d..46a4aa0 100644 --- a/converters/orionbelt/src/osi_orionbelt/converter.py +++ b/converters/orionbelt/src/osi_orionbelt/converter.py @@ -10,2854 +10,120 @@ v0.2 first-class fields before regular parsing runs. Author: OrionBelt / RALFORION + +This module is a thin **facade**. The converter implementation is split across +sibling modules to keep each file focused: + +* :mod:`osi_orionbelt._common` — shared constants and mapping tables +* :mod:`osi_orionbelt.osi_to_obml` — :class:`OSItoOBML` +* :mod:`osi_orionbelt.obml_to_osi` — :class:`OBMLtoOSI` +* :mod:`osi_orionbelt.ontology` — :class:`OBMLtoOSIOntology` +* :mod:`osi_orionbelt.validation` — :class:`ValidationResult` + ``validate_*`` + +Every public name is re-exported here so ``osi_orionbelt.converter.`` +continues to work unchanged. """ +from __future__ import annotations + import argparse -import json -import re import sys from pathlib import Path -from typing import Any import yaml -# ─── Spec version pin ─────────────────────────────────────────────────────── -# Single source of truth for the OSI spec we emit. Bump when upstream cuts -# a stable v0.2.0 (drop the ``.dev0`` suffix). All read paths accept both -# 0.1.x (via the legacy shim) and 0.2.x. -_OSI_VERSION = "0.2.0.dev0" - -# Dialect / vendor enum extras new in v0.2.0.dev0 -_OSI_KNOWN_DIALECTS = ("ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL") -# SQL dialects (of the OSI enum) whose aggregation expressions our regex-based -# metric parser can read, in preference order. ANSI_SQL first; SNOWFLAKE and -# DATABRICKS are SQL engines OrionBelt also targets, and their simple/expression -# aggregations (``SUM(t.c)``, ``SUM(t.a * t.b)``) are syntactically identical to -# ANSI. MDX / TABLEAU / MAQL are non-SQL languages and are never parsed as SQL. -_SQL_PARSEABLE_DIALECTS = ("ANSI_SQL", "SNOWFLAKE", "DATABRICKS") - -# Matches a ``dataset.column`` reference inside a SQL expression, where each -# side is a bare identifier or a quoted identifier (double quotes, backticks, or -# brackets). The leading lookbehind prevents matching the tail of a longer path -# (``a.b.c``) or a mid-token boundary; the bare form must start with a letter or -# underscore so numeric literals (``1.5``) are never treated as references. -_COLUMN_REF_RE = re.compile( - r'(?[A-Za-z_]\w*|"[^"]+"|`[^`]+`|\[[^\]]+\])' - r"\s*\.\s*" - r'(?P[A-Za-z_]\w*|"[^"]+"|`[^`]+`|\[[^\]]+\])' +from osi_orionbelt._common import ( + _COLUMN_REF_RE as _COLUMN_REF_RE, ) -_OSI_KNOWN_VENDORS = ( - "COMMON", - "ORIONBELT", - "SNOWFLAKE", - "SALESFORCE", - "DBT", - "DATABRICKS", - "GOODDATA", +from osi_orionbelt._common import ( + _INTERNAL_VENDORS as _INTERNAL_VENDORS, ) - -# Vendor identities for custom_extensions. -# ORIONBELT - OrionBelt/OBML-proprietary payloads we author on OBML -> OSI. -# OSI - OSI-native fields OBML can't hold (unique_keys, field label, -# ai_context leftovers), stashed into OBML on OSI -> OBML. -# Read paths also accept the legacy tags we emitted before this scheme so older -# documents still round-trip; foreign vendors (SNOWFLAKE, DBT, ...) are -# preserved verbatim, never relabelled. -_VENDOR_OBML = "ORIONBELT" -_VENDOR_OSI = "OSI" -_OBML_VENDOR_READ = ("ORIONBELT", "COMMON") -_OSI_VENDOR_READ = ("OSI", "OBSL") -# Vendors the converter handles internally (its own payloads + native-field -# stashes). Any custom_extension from a vendor outside this set is third-party -# and is carried through verbatim in both directions, never relabelled. -_INTERNAL_VENDORS = frozenset({"ORIONBELT", "COMMON", "OSI", "OBSL"}) - -# ─── Type mapping ─────────────────────────────────────────────────────────── - -OBML_TO_OSI_TYPE = { - "string": "string", - "json": "string", - "int": "integer", - "float": "number", - "date": "date", - "time": "time", - "time_tz": "time", - "timestamp": "timestamp", - "timestamp_tz": "timestamp", - "boolean": "boolean", -} - -OSI_TO_OBML_TYPE = { - "string": "string", - "integer": "int", - "number": "float", - "date": "date", - "time": "time", - "timestamp": "timestamp", - "boolean": "boolean", -} - - -# ═══════════════════════════════════════════════════════════════════════════ -# OSI → OBML Converter -# ═══════════════════════════════════════════════════════════════════════════ - - -class OSItoOBML: - """Convert an OSI semantic model YAML to OBML format.""" - - def __init__( - self, osi: dict, default_database: str = "ANALYTICS", default_schema: str = "PUBLIC" - ): - self.osi = osi - self.default_database = default_database - self.default_schema = default_schema - self.warnings: list[str] = [] - # OSI metrics that have no OBML representation (non-SQL dialect only, - # or an expression our parser cannot decompose). Preserved verbatim - # rather than dropped — see ``_preserve_unconverted_metric``. - self._unconverted_metrics: list[dict] = [] - - def _normalize_legacy_v01(self) -> None: - """Promote OSI v0.1.x payloads to the v0.2 shape, in place. - - The v0.2 spec promotes ``primary_key`` and ``unique_keys`` to - first-class dataset fields. v0.1.x serializers (including ours - pre-bump) stash both under ``custom_extensions`` with vendor - ``OBSL`` and keys ``obml_primary_key`` / ``obml_unique_keys``. - This shim runs before parsing so the rest of the converter can - assume v0.2 shape regardless of input version. - - No-op for documents that already declare ``version`` >= 0.2 or - that have nothing to migrate. - """ - version = str(self.osi.get("version", "")) - if version and not version.startswith(("0.1", "0.0")): - return # already v0.2+ (or future) — nothing to do - - models = self.osi.get("semantic_model", []) - if not isinstance(models, list): - return - - for model in models: - for ds in model.get("datasets", []) or []: - # Promote legacy primary_key / unique_keys from OBSL extras - # only if the dataset doesn't already declare them. - legacy = self._extract_obml_extras(ds) - if not legacy: - continue - if "primary_key" not in ds and legacy.get("obml_primary_key"): - pk = legacy["obml_primary_key"] - if isinstance(pk, list) and all(isinstance(c, str) for c in pk): - ds["primary_key"] = list(pk) - if "unique_keys" not in ds and legacy.get("obml_unique_keys"): - uk = legacy["obml_unique_keys"] - if isinstance(uk, list) and all( - isinstance(g, list) and all(isinstance(c, str) for c in g) for g in uk - ): - ds["unique_keys"] = [list(g) for g in uk] - - if version.startswith(("0.0", "0.1")): - self.warnings.append( - f"OSI input declares version '{version}'; legacy v0.1.x " - f"compatibility shim applied. Output target is v{_OSI_VERSION}." - ) - - def convert(self) -> dict: - # Reset per-conversion accumulators so calling convert() twice on the - # same instance is idempotent (no duplicated warnings or preserved - # metrics). Both are populated as a side effect of conversion below. - self.warnings = [] - self._unconverted_metrics = [] - - # v0.1.x inputs need the legacy shim to promote pre-v0.2 - # custom_extensions into v0.2 first-class fields before we parse. - self._normalize_legacy_v01() - - models = self.osi.get("semantic_model", []) - if not models: - raise ValueError("No semantic_model found in OSI input") - - # Take the first semantic model (OBML is a single-model format) - model = models[0] - if len(models) > 1: - self.warnings.append( - f"OSI contains {len(models)} semantic models; " - f"only the first ('{model.get('name')}') is converted." - ) - - obml: dict[str, Any] = {"version": 1.0} - - # ── Model description ───────────────────────────────────── - if model.get("description"): - obml["description"] = model["description"] - - # ── DataObjects ───────────────────────────────────────────── - datasets = model.get("datasets", []) - relationships = model.get("relationships", []) - - # Build lookup: dataset_name → dataset - ds_map = {ds["name"]: ds for ds in datasets} - - # Build relationship index: from_dataset → [relationship, ...] - rel_by_from: dict[str, list] = {} - for rel in relationships: - rel_by_from.setdefault(rel["from"], []).append(rel) - - # Collect join key columns: (dataset_name, field_name) pairs - # These should NOT become dimensions (they are FK/PK join keys) - self._join_key_columns: set[tuple[str, str]] = set() - for rel in relationships: - for col in rel.get("from_columns", []): - self._join_key_columns.add((rel["from"], col)) - for col in rel.get("to_columns", []): - self._join_key_columns.add((rel["to"], col)) - - data_objects: dict[str, Any] = {} - for ds in datasets: - do_name, do_obj = self._convert_dataset(ds, rel_by_from) - data_objects[do_name] = do_obj - - obml["dataObjects"] = data_objects - - # ── Dimensions (extracted from OSI fields with dimension metadata) ── - dimensions = self._extract_dimensions(datasets) - if dimensions: - obml["dimensions"] = dimensions - - # ── Measures & Metrics ────────────────────────────────────── - osi_metrics = model.get("metrics", []) - measures, metrics = self._convert_metrics(osi_metrics, ds_map) - if measures: - obml["measures"] = measures - if metrics: - obml["metrics"] = metrics - - # Metrics that have no OBML representation are not dropped: stash the - # original OSI metric verbatim under the OSI vendor so the reverse - # (OBML -> OSI) direction re-emits them and a full OSI -> OBML -> OSI - # roundtrip stays lossless. They are not queryable in OBML; a LOSSY - # warning was already recorded per metric. - if self._unconverted_metrics: - obml.setdefault("customExtensions", []).append( - { - "vendor": _VENDOR_OSI, - "data": json.dumps({"obml_unconverted_metrics": self._unconverted_metrics}), - } - ) - - # ── Restore model-level properties from custom_extensions ──── - for ext in model.get("custom_extensions", []): - if ext.get("vendor_name") in _OBML_VENDOR_READ: - try: - ext_data = json.loads(ext.get("data", "{}")) - if ext_data.get("obml_filters"): - obml["filters"] = ext_data["obml_filters"] - if ext_data.get("obml_settings"): - obml["settings"] = ext_data["obml_settings"] - if ext_data.get("obml_owner"): - obml["owner"] = ext_data["obml_owner"] - except (json.JSONDecodeError, TypeError): - pass - break - - # Preserve third-party vendor extensions verbatim - self._carry_foreign_extensions(model.get("custom_extensions"), obml) - - return obml - - @staticmethod - def _carry_foreign_extensions(osi_exts: list[dict] | None, obml_target: dict[str, Any]) -> None: - """Carry third-party OSI custom_extensions verbatim into OBML. - - Our own payloads and OSI-native stashes are reconstructed elsewhere; - any other vendor's extension is preserved unchanged on the OBML side - so a full OSI -> OBML -> OSI roundtrip keeps the original vendor. - """ - for ext in osi_exts or []: - vendor = ext.get("vendor_name") - if vendor and vendor not in _INTERNAL_VENDORS: - obml_target.setdefault("customExtensions", []).append( - {"vendor": vendor, "data": ext.get("data", "")} - ) - - def _parse_source(self, source: str) -> tuple[str, str, str]: - """Parse 'database.schema.table' into parts.""" - parts = source.split(".") - if len(parts) == 3: - return parts[0], parts[1], parts[2] - elif len(parts) == 2: - return self.default_database, parts[0], parts[1] - else: - return self.default_database, self.default_schema, parts[0] - - def _convert_dataset(self, ds: dict, rel_by_from: dict) -> tuple[str, dict]: - """Convert an OSI dataset to an OBML dataObject. - - Uses the exact OSI dataset name as the OBML data object key. - """ - name = ds["name"] - - source = ds.get("source", name) - database, schema, table = self._parse_source(source) - - do: dict[str, Any] = { - "code": table, - "database": database, - "schema": schema, - } - - # ── Columns ───────────────────────────────────────────────── - columns: dict[str, Any] = {} - fields = ds.get("fields", []) - for field in fields: - col_name, col_obj = self._convert_field(field) - columns[col_name] = col_obj - - # ── Primary key flag propagation (OSI v0.2 first-class) ── - # ``primary_key`` lists physical column codes; mark every matching - # column with ``primaryKey: true``. Unknown PK columns surface as - # a warning (the spec couples PK to relationship cardinality, so - # silently dropping is unsafe). - pk_codes = ds.get("primary_key") or [] - if pk_codes: - code_to_col = {col.get("code"): (cname, col) for cname, col in columns.items()} - unknown_pks: list[str] = [] - for pk_code in pk_codes: - hit = code_to_col.get(pk_code) - if hit is None: - unknown_pks.append(pk_code) - continue - _, col = hit - col["primaryKey"] = True - if unknown_pks: - self.warnings.append( - f"Dataset '{name}' primary_key references unknown columns: " - f"{unknown_pks}. Ignored." - ) - - if columns: - do["columns"] = columns - else: - self.warnings.append(f"Dataset '{name}' has no fields; adding placeholder column.") - do["columns"] = {f"{name}_id": {"code": f"{table}_id", "abstractType": "string"}} - - # ── Joins (from relationships where this dataset is on 'from' side) ── - joins = [] - for rel in rel_by_from.get(name, []): - join_obj = self._convert_relationship_to_join(rel) - joins.append(join_obj) - - if joins: - do["joins"] = joins - - # ── Description (semantic, from OSI) ───────────────────────── - if ds.get("description"): - do["description"] = ds["description"] - - # ── Extract ai_context: synonyms → native, rest → customExtensions ─ - ai_ctx = ds.get("ai_context") - if ai_ctx: - ai_data = ai_ctx if isinstance(ai_ctx, dict) else {"instructions": ai_ctx} - # Extract synonyms directly into OBML synonyms property - if "synonyms" in ai_data: - do["synonyms"] = list(ai_data["synonyms"]) - # Store remaining ai_context keys in customExtensions - remaining = {k: v for k, v in ai_data.items() if k != "synonyms"} - if remaining: - do["customExtensions"] = [ - { - "vendor": "OSI", - "data": json.dumps(remaining), - } - ] - - # Restore DataObject owner / comment / refresh from custom_extensions - for ext in ds.get("custom_extensions", []): - if ext.get("vendor_name") in _OBML_VENDOR_READ: - try: - ext_data = json.loads(ext.get("data", "{}")) - if ext_data.get("obml_owner"): - do["owner"] = ext_data["obml_owner"] - if ext_data.get("obml_comment"): - do["comment"] = ext_data["obml_comment"] - if ext_data.get("obml_refresh"): - do["refresh"] = ext_data["obml_refresh"] - except (json.JSONDecodeError, TypeError): - pass - break - - # ── Unique keys roundtrip (OBML has no native concept) ── - # Persist the OSI ``unique_keys`` array into the OBSL-vendor - # customExtensions so the OBML → OSI direction can emit it back. - unique_keys = ds.get("unique_keys") or [] - if unique_keys: - do.setdefault("customExtensions", []).append( - { - "vendor": _VENDOR_OSI, - "data": json.dumps({"obml_unique_keys": [list(g) for g in unique_keys]}), - } - ) - - # Preserve third-party vendor extensions verbatim - self._carry_foreign_extensions(ds.get("custom_extensions"), do) - - return name, do - - def _convert_field(self, field: dict) -> tuple[str, dict]: - """Convert an OSI field to an OBML column. - - Uses the exact OSI field name as the OBML column key. - """ - name = field["name"] - - # Get expression (prefer ANSI_SQL dialect) - expr_obj = field.get("expression", {}) - code = name # fallback - if isinstance(expr_obj, dict): - dialects = expr_obj.get("dialects", []) - for d in dialects: - if d.get("dialect") == "ANSI_SQL": - code = d.get("expression", name) - break - if not dialects: - code = name - elif code == name and dialects: - code = dialects[0].get("expression", name) - - # Determine abstract type: prefer explicit data_type, fall back to heuristic - osi_type = field.get("data_type", "") - if osi_type and osi_type in OSI_TO_OBML_TYPE: - abstract_type = OSI_TO_OBML_TYPE[osi_type] - else: - abstract_type = self._infer_obml_type(field) - - col: dict[str, Any] = { - "code": code, - "abstractType": abstract_type, - } - - if field.get("description"): - col["description"] = field["description"] - - # Extract field-level ai_context: synonyms → native, rest → customExtensions - ai_ctx = field.get("ai_context") - if ai_ctx: - ai_data = ai_ctx if isinstance(ai_ctx, dict) else {"instructions": ai_ctx} - # Extract synonyms directly into OBML synonyms property - if "synonyms" in ai_data: - col["synonyms"] = list(ai_data["synonyms"]) - # Store remaining ai_context keys in customExtensions - remaining = {k: v for k, v in ai_data.items() if k != "synonyms"} - if remaining: - col["customExtensions"] = [ - { - "vendor": "OSI", - "data": json.dumps(remaining), - } - ] - - # Restore OBML-only column properties from custom_extensions - for ext in field.get("custom_extensions", []): - if ext.get("vendor_name") in _OBML_VENDOR_READ: - try: - ext_data = json.loads(ext.get("data", "{}")) - if ext_data.get("obml_sql_type"): - col["sqlType"] = ext_data["obml_sql_type"] - if ext_data.get("obml_sql_precision") is not None: - col["sqlPrecision"] = ext_data["obml_sql_precision"] - if ext_data.get("obml_sql_scale") is not None: - col["sqlScale"] = ext_data["obml_sql_scale"] - if ext_data.get("obml_num_class"): - col["numClass"] = ext_data["obml_num_class"] - if ext_data.get("obml_comment"): - col["comment"] = ext_data["obml_comment"] - if ext_data.get("obml_owner"): - col["owner"] = ext_data["obml_owner"] - except (json.JSONDecodeError, TypeError): - pass - break - - # ── Field label roundtrip (OSI v0.2 first-class) ── - # OBML has no native column label today; preserve via OBSL-vendor - # customExtensions so the reverse direction can emit it back. - if field.get("label"): - col.setdefault("customExtensions", []).append( - { - "vendor": _VENDOR_OSI, - "data": json.dumps({"obml_field_label": field["label"]}), - } - ) - - # Preserve third-party vendor extensions verbatim - self._carry_foreign_extensions(field.get("custom_extensions"), col) - - return name, col - - def _infer_obml_type(self, field: dict) -> str: - """Infer OBML abstractType from OSI field metadata.""" - - dim = field.get("dimension", {}) - if isinstance(dim, dict) and dim.get("is_time"): - return "date" - - name_lower = field.get("name", "").lower() - - # Helper: match keywords at word boundaries to avoid false positives - # (e.g. "country" should NOT match "count") - def _has_keyword(keywords: tuple[str, ...]) -> bool: - for kw in keywords: - if kw.startswith("_") or kw.endswith("_"): - # Substring match for prefix/suffix patterns like "_sk", "is_" - if kw in name_lower: - return True - else: - # Word-boundary match for standalone keywords - if re.search(r"(?:^|_)" + re.escape(kw) + r"(?:$|_)", name_lower): - return True - return False - - if _has_keyword( - ( - "_sk", - "_id", - "_key", - "name", - "desc", - "email", - "address", - "city", - "state", - "zip", - "phone", - "status", - "type", - "category", - "class", - ) - ): - return "string" - if _has_keyword( - ( - "price", - "cost", - "amount", - "sales", - "profit", - "revenue", - "tax", - "discount", - "rate", - "percent", - "ratio", - "margin", - ) - ): - return "float" - if _has_keyword(("qty", "quantity", "count", "num", "number", "cnt")): - return "int" - if _has_keyword(("date", "time", "year", "month", "day", "week")): - return "date" - if _has_keyword(("flag", "is_", "has_")): - return "boolean" - - return "string" - - # OSI relationship type → OBML joinType mapping - _REL_TYPE_MAP: dict[str, str] = { - "many_to_one": "many-to-one", - "many-to-one": "many-to-one", - "one_to_many": "one-to-many", - "one-to-many": "one-to-many", - "one_to_one": "one-to-one", - "one-to-one": "one-to-one", - "many_to_many": "many-to-many", - "many-to-many": "many-to-many", - } - - def _convert_relationship_to_join(self, rel: dict) -> dict: - """Convert an OSI relationship to an OBML join. - - Uses exact OSI names for joinTo and column references. - Maps OSI relationship 'type' to OBML joinType if present, - defaults to many-to-one with a warning otherwise. - """ - rel_type = rel.get("type", "") - join_type = self._REL_TYPE_MAP.get(rel_type.lower(), "") if rel_type else "" - if not join_type: - join_type = "many-to-one" - if rel_type: - self.warnings.append( - f"Relationship '{rel.get('name', '?')}': unknown type " - f"'{rel_type}', defaulting to many-to-one." - ) - else: - self.warnings.append( - f"Relationship '{rel.get('name', '?')}': no type specified, " - f"defaulting to many-to-one." - ) - - join: dict[str, Any] = { - "joinType": join_type, - "joinTo": rel["to"], - "columnsFrom": list(rel["from_columns"]), - "columnsTo": list(rel["to_columns"]), - } - return join - - def _extract_dimensions(self, datasets: list) -> dict: - """Extract dimension definitions from OSI fields marked as dimensions. - - Skips fields that are join keys (FK/PK columns used in relationships), - since those are structural and not analytical dimensions. - """ - dimensions: dict[str, Any] = {} - for ds in datasets: - ds_name = ds["name"] - for field in ds.get("fields", []): - dim = field.get("dimension") - if dim is None: - continue - field_name = field["name"] - # Skip join key columns — they are FK/PK, not analytical dims - if (ds_name, field_name) in self._join_key_columns: - continue - abstract_type = self._infer_obml_type(field) - dim_def: dict[str, Any] = { - "dataObject": ds_name, - "column": field_name, - "resultType": abstract_type, - } - # Extract synonyms from field-level ai_context - ai_ctx = field.get("ai_context") - if isinstance(ai_ctx, dict) and ai_ctx.get("synonyms"): - dim_def["synonyms"] = list(ai_ctx["synonyms"]) - # Restore OBML-only dimension properties from custom_extensions - for ext in field.get("custom_extensions", []): - if ext.get("vendor_name") in _OBML_VENDOR_READ: - try: - ext_data = json.loads(ext.get("data", "{}")) - if ext_data.get("obml_time_grain"): - dim_def["timeGrain"] = ext_data["obml_time_grain"] - if ext_data.get("obml_dimension_format"): - dim_def["format"] = ext_data["obml_dimension_format"] - if ext_data.get("obml_dimension_result_type"): - dim_def["resultType"] = ext_data["obml_dimension_result_type"] - if ext_data.get("obml_dimension_description"): - dim_def["description"] = ext_data["obml_dimension_description"] - if ext_data.get("obml_dimension_owner"): - dim_def["owner"] = ext_data["obml_dimension_owner"] - if ext_data.get("obml_dimension_via"): - dim_def["via"] = ext_data["obml_dimension_via"] - except (json.JSONDecodeError, TypeError): - pass - break - dimensions[field_name] = dim_def - return dimensions - - def _convert_metrics(self, osi_metrics: list, ds_map: dict) -> tuple[dict, dict]: - """ - Convert OSI metrics to OBML measures and metrics. - - OSI has a single 'metrics' concept with SQL expressions. - OBML separates 'measures' (simple aggregations on single columns) - from 'metrics' (cross-fact expressions referencing measures). - - Strategy: - - Simple single-aggregation metrics → OBML measures - - Aggregation over expression (e.g. SUM(a.x * a.y)) → expression measure - - Complex/multi-aggregation metrics → OBML metrics referencing auto-measures - """ - - measures: dict[str, Any] = {} - metrics: dict[str, Any] = {} - - # Case-insensitive dataset/field index for resolving SQL identifiers - # back to their canonical OSI names (Snowflake/Databricks expressions - # commonly upper-case or quote them). - ds_lc = {name.lower(): name for name in ds_map} - fields_lc = { - name: { - f["name"].lower(): f["name"] - for f in ds.get("fields", []) or [] - if isinstance(f, dict) and f.get("name") - } - for name, ds in ds_map.items() - } - - for m in osi_metrics: - name = m["name"] - - osi_description = m.get("description") - - # Extract synonyms from OSI ai_context - osi_ai_ctx = m.get("ai_context") - osi_synonyms: list[str] = [] - if isinstance(osi_ai_ctx, dict) and osi_ai_ctx.get("synonyms"): - osi_synonyms = list(osi_ai_ctx["synonyms"]) - - # Restore OBML-only properties from custom_extensions - obml_extras = self._extract_obml_extras(m) - - # Check for cumulative metric stored in custom_extensions - if obml_extras.get("obml_metric_type") == "cumulative": - cum_metric = self._reconstruct_cumulative_metric( - name, obml_extras, osi_description, osi_synonyms - ) - metrics[name] = cum_metric - continue - - # Check for period-over-period metric stored in custom_extensions - if obml_extras.get("obml_metric_type") == "period_over_period": - pop_metric = self._reconstruct_pop_metric( - name, obml_extras, osi_description, osi_synonyms - ) - metrics[name] = pop_metric - continue - - # Check for window metric (rank/lag/lead/ntile/first_value/last_value) - if obml_extras.get("obml_metric_type") == "window": - window_metric = self._reconstruct_window_metric( - name, obml_extras, osi_description, osi_synonyms - ) - metrics[name] = window_metric - continue - - # Engine-delegated aggregation (Databricks Metric View). Round-trip - # marker comes from the OBML → OSI direction; on input we restore - # ``aggregation: measure`` without touching the OSI expression - # (which is a literal ``MEASURE(" *
CalculatedN/AMODEL.semanticCalculatedDimensions
* - * @param osiDataset The OSI dataset + * @param osiDataset The Ossie dataset * @param sfDataObject The Salesforce data object to add direct fields to * @param outputData The Salesforce model for adding calculated dimensions */ @@ -267,7 +286,7 @@ private void processFieldsForDataset( for (Object osiFieldObj : osiFields) { Map osiField = asMap(osiFieldObj); - // Determine field type based on OSI structure + // Determine field type based on Ossie structure boolean hasDimension = osiField.containsKey(DIMENSION); ExpressionInfo expressionInfo = unwrapExpression(osiField); @@ -316,7 +335,7 @@ private void processFieldsForDataset( * Maps field properties based on whether the field is calculated. * Includes common properties plus type-specific properties. * - * @param osiField The OSI field + * @param osiField The Ossie field * @param expression The extracted expression string * @return A map with Salesforce field properties */ @@ -343,10 +362,10 @@ private Map mapFieldProperties( } /** - * Creates a Salesforce semanticCalculatedDimension from an OSI field with a calculated expression. + * Creates a Salesforce semanticCalculatedDimension from an Ossie field with a calculated expression. * Per schema: required properties are apiName and expression. * - * @param osiField The OSI field + * @param osiField The Ossie field * @param expression The calculated expression * @return A map representing a semanticCalculatedDimension */ @@ -422,10 +441,10 @@ private record RoutingResult(List dataObjectDimensions, List dat private record ExpressionInfo(String expression, String dialect) {} /** - * Extracts the expression value and dialect from OSI field's expression.dialects[0].expression. + * Extracts the expression value and dialect from Ossie field's expression.dialects[0].expression. * This unwraps the nested structure to get the simple column reference and its dialect. * - * @param osiField The OSI field containing expression structure + * @param osiField The Ossie field containing expression structure * @return ExpressionInfo containing the expression string and dialect type, or null if not found */ private ExpressionInfo unwrapExpression(Map osiField) { @@ -532,7 +551,7 @@ private void applyFieldDefaults(Map sfField) { * * * @param sourceData The source Salesforce data (will be modified to remove converted dimensions) - * @param outputData The output OSI data containing datasets + * @param outputData The output Ossie data containing datasets */ private void processModelLevelCalculatedDimensions( Map sourceData, Map outputData) { @@ -615,11 +634,11 @@ private String getSingleDataObjectFromDependencies(List dependencies) { } /** - * Converts a Salesforce semanticCalculatedDimension to an OSI field with dimension property. + * Converts a Salesforce semanticCalculatedDimension to an Ossie field with dimension property. * Similar to convertDimensionToOsiField but handles expression (not dataObjectFieldName). * * @param calcDim The Salesforce calculated dimension - * @return An OSI field map with dimension property and wrapped expression + * @return An Ossie field map with dimension property and wrapped expression */ private Map convertCalculatedDimensionToField(Map calcDim) { Map osiField = new LinkedHashMap<>(); diff --git a/converters/salesforce/src/main/java/org/osi/converter/GenericMappingEngine.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/GenericMappingEngine.java similarity index 82% rename from converters/salesforce/src/main/java/org/osi/converter/GenericMappingEngine.java rename to converters/salesforce/src/main/java/org/apache/ossie/converter/GenericMappingEngine.java index 86abc7d..2c71b6e 100644 --- a/converters/salesforce/src/main/java/org/osi/converter/GenericMappingEngine.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/GenericMappingEngine.java @@ -1,9 +1,28 @@ -package org.osi.converter; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; -import static org.osi.util.DataStructureUtils.*; +import static org.apache.ossie.util.DataStructureUtils.*; -import org.osi.util.MappingUtils; -import org.osi.util.PathUtils; +import org.apache.ossie.util.MappingUtils; +import org.apache.ossie.util.PathUtils; import java.util.LinkedHashMap; import java.util.List; diff --git a/converters/salesforce/src/main/java/org/osi/converter/MetricMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java similarity index 70% rename from converters/salesforce/src/main/java/org/osi/converter/MetricMappingHandler.java rename to converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java index 4d1a9da..c54d46a 100644 --- a/converters/salesforce/src/main/java/org/osi/converter/MetricMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java @@ -1,23 +1,42 @@ -package org.osi.converter; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; -import static org.osi.converter.ConverterConstants.*; -import static org.osi.util.DataStructureUtils.*; +import static org.apache.ossie.converter.ConverterConstants.*; +import static org.apache.ossie.util.DataStructureUtils.*; -import org.osi.converter.ConverterConstants.Level; -import org.osi.converter.pipeline.PipelineStep; +import org.apache.ossie.converter.ConverterConstants.Level; +import org.apache.ossie.converter.pipeline.PipelineStep; import java.util.*; -import org.osi.util.MappingUtils; +import org.apache.ossie.util.MappingUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Bidirectional handler for mapping metrics between OSI and Salesforce formats. + * Bidirectional handler for mapping metrics between Ossie and Salesforce formats. * *

Supports both conversion directions: *

    - *
  • OSI → Salesforce: unwrap expression from dialects structure
    • - *
  • Salesforce → OSI: wrap expression in dialects structure
    • + *
  • Ossie → Salesforce: unwrap expression from dialects structure
    • + *
  • Salesforce → Ossie: wrap expression in dialects structure
    • *
* */ @@ -47,7 +66,7 @@ public void execute(Map sourceData, Map outputDa } /** - * Maps OSI metrics to Salesforce semanticCalculatedMeasurements. + * Maps Ossie metrics to Salesforce semanticCalculatedMeasurements. */ private void mapOsiToSalesforce( Map sourceData, Map outputData, Map mappings) { @@ -61,11 +80,11 @@ private void mapOsiToSalesforce( Map metricMappings = MappingUtils.filterMappingsByPrefix(mappings, METRICS); metricMappings.keySet().forEach(mappings::remove); - logger.debug("Metrics are not mapped in OSI to Salesforce direction"); + logger.debug("Metrics are not mapped in Ossie to Salesforce direction"); } /** - * Maps Salesforce semanticCalculatedMeasurements to OSI metrics. + * Maps Salesforce semanticCalculatedMeasurements to Ossie metrics. */ private void mapSalesforceToOsi( Map sourceData, Map outputData, Map mappings) { @@ -100,7 +119,7 @@ private void mapSalesforceToOsi( /** - * Wraps expressions for SF→OSI conversion. + * Wraps expressions for SF→Ossie conversion. */ private void wrapExpressions(List sfMetrics, List osiMetrics) { for (int i = 0; i < sfMetrics.size() && i < osiMetrics.size(); i++) { @@ -110,14 +129,14 @@ private void wrapExpressions(List sfMetrics, List osiMetrics) { // Get expression from SF metric String expressionValue = getString(sfMetric, EXPRESSION); if (expressionValue != null) { - // Wrap in OSI dialect structure + // Wrap in Ossie dialect structure osiMetric.put(EXPRESSION, wrapExpression(expressionValue)); } } } /** - * Wraps a simple expression string in OSI's expression.dialects structure. + * Wraps a simple expression string in Ossie's expression.dialects structure. * Tags expressions with TABLEAU dialect as they come from Salesforce (Tableau CRM). */ private Map wrapExpression(String expressionValue) { diff --git a/converters/salesforce/src/main/java/org/osi/converter/RelationshipMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/RelationshipMappingHandler.java similarity index 90% rename from converters/salesforce/src/main/java/org/osi/converter/RelationshipMappingHandler.java rename to converters/salesforce/src/main/java/org/apache/ossie/converter/RelationshipMappingHandler.java index cd669a6..92f67d0 100644 --- a/converters/salesforce/src/main/java/org/osi/converter/RelationshipMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/RelationshipMappingHandler.java @@ -1,23 +1,42 @@ -package org.osi.converter; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; -import static org.osi.converter.ConverterConstants.*; -import static org.osi.util.DataStructureUtils.*; +import static org.apache.ossie.converter.ConverterConstants.*; +import static org.apache.ossie.util.DataStructureUtils.*; -import org.osi.converter.ConverterConstants.Level; -import org.osi.converter.pipeline.PipelineStep; +import org.apache.ossie.converter.ConverterConstants.Level; +import org.apache.ossie.converter.pipeline.PipelineStep; import java.util.*; -import org.osi.util.MappingUtils; +import org.apache.ossie.util.MappingUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Bidirectional handler for mapping relationships between OSI and Salesforce formats. + * Bidirectional handler for mapping relationships between Ossie and Salesforce formats. * *

Supports both conversion directions: *

    - *
  • OSI → Salesforce: from/to/from_columns/to_columns → criteria array
    • - *
  • Salesforce → OSI: criteria array → from/to/from_columns/to_columns
    • + *
  • Ossie → Salesforce: from/to/from_columns/to_columns → criteria array
    • + *
  • Salesforce → Ossie: criteria array → from/to/from_columns/to_columns
    • *
* */ @@ -44,7 +63,7 @@ public void execute(Map sourceData, Map outputDa } /** - * Maps OSI relationships to Salesforce semanticRelationships. + * Maps Ossie relationships to Salesforce semanticRelationships. */ private void mapOsiToSalesforce( Map sourceData, Map outputData, Map mappings) { @@ -84,7 +103,7 @@ private void mapOsiToSalesforce( } /** - * Maps Salesforce semanticRelationships to OSI relationships. + * Maps Salesforce semanticRelationships to Ossie relationships. */ private void mapSalesforceToOsi( Map sourceData, Map outputData, Map mappings) { @@ -147,7 +166,7 @@ private void mapSalesforceToOsi( } /** - * Reconstructs criteria for OSI→SF conversion. + * Reconstructs criteria for Ossie→SF conversion. */ private void reconstructCriteria(List osiRelationships, List sfRelationships) { for (int i = 0; i < osiRelationships.size() && i < sfRelationships.size(); i++) { @@ -200,7 +219,7 @@ private void reconstructCriteria(List osiRelationships, List sfR } /** - * Deconstructs criteria for SF→OSI conversion. + * Deconstructs criteria for SF→Ossie conversion. * Note: This method only processes supported relationships (TableField types). * Unsupported relationships are filtered out earlier and stored in custom_extensions. */ @@ -267,7 +286,7 @@ private void applyDefaults(List sfRelationships) { /** * Validates and filters relationships, removing those that reference non-existent fields. (Calculated fields that are not supported) * - * @param osiRelationships List of OSI relationships to validate + * @param osiRelationships List of Ossie relationships to validate * @param outputData The output data containing semanticDataObjects with their fields * @return Filtered list of valid relationships */ diff --git a/converters/salesforce/src/main/java/org/osi/converter/SemanticModelMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/SemanticModelMappingHandler.java similarity index 68% rename from converters/salesforce/src/main/java/org/osi/converter/SemanticModelMappingHandler.java rename to converters/salesforce/src/main/java/org/apache/ossie/converter/SemanticModelMappingHandler.java index 5bbfbbd..d6c997c 100644 --- a/converters/salesforce/src/main/java/org/osi/converter/SemanticModelMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/SemanticModelMappingHandler.java @@ -1,27 +1,46 @@ -package org.osi.converter; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import org.osi.converter.ConverterConstants.Level; -import org.osi.converter.pipeline.PipelineStep; -import org.osi.exception.ConversionException; -import org.osi.util.MappingUtils; +import org.apache.ossie.converter.ConverterConstants.Level; +import org.apache.ossie.converter.pipeline.PipelineStep; +import org.apache.ossie.exception.ConversionException; +import org.apache.ossie.util.MappingUtils; import java.util.Map; import java.util.Set; -import static org.osi.converter.ConverterConstants.AI_CONTEXT; -import static org.osi.converter.ConverterConstants.API_NAME; -import static org.osi.converter.ConverterConstants.BUSINESS_PREFERENCES; -import static org.osi.converter.ConverterConstants.LABEL; +import static org.apache.ossie.converter.ConverterConstants.AI_CONTEXT; +import static org.apache.ossie.converter.ConverterConstants.API_NAME; +import static org.apache.ossie.converter.ConverterConstants.BUSINESS_PREFERENCES; +import static org.apache.ossie.converter.ConverterConstants.LABEL; /** * Bidirectional handler for mapping top-level semantic model properties. * *

Supports both conversion directions: *

    - *
  • OSI → Salesforce: name → apiName, apply SF defaults, restore custom_extensions
    • - *
  • Salesforce → OSI: apiName → name, strip SF defaults, store custom_extensions
    • + *
  • Ossie → Salesforce: name → apiName, apply SF defaults, restore custom_extensions
    • + *
  • Salesforce → Ossie: apiName → name, strip SF defaults, store custom_extensions
    • *
* *

Handles only top-level scalar properties (not arrays). @@ -49,7 +68,7 @@ public void execute(Map sourceData, Map outputDa } /** - * Maps OSI top-level properties to Salesforce format. + * Maps Ossie top-level properties to Salesforce format. * Steps: generic mappings → manual conversions → restore custom extensions */ private void mapOsiToSalesforce( @@ -66,7 +85,7 @@ private void mapOsiToSalesforce( } /** - * Maps Salesforce top-level properties to OSI format. + * Maps Salesforce top-level properties to Ossie format. * Steps: generic mappings → manual conversions → store custom extensions */ private void mapSalesforceToOsi( @@ -84,10 +103,10 @@ private void mapSalesforceToOsi( } /** - * Converts Salesforce businessPreferences to OSI ai_context. + * Converts Salesforce businessPreferences to Ossie ai_context. * * @param sourceData Salesforce data containing businessPreferences - * @param outputData OSI data to populate with ai_context + * @param outputData Ossie data to populate with ai_context */ private void convertBusinessPreferencesToAiContext(Map sourceData, Map outputData) { Object businessPreferences = sourceData.get(BUSINESS_PREFERENCES); @@ -97,7 +116,7 @@ private void convertBusinessPreferencesToAiContext(Map sourceDat } /** - * Converts OSI ai_context (string or object) to Salesforce businessPreferences (string). + * Converts Ossie ai_context (string or object) to Salesforce businessPreferences (string). * *

ai_context can be: *

    @@ -105,7 +124,7 @@ private void convertBusinessPreferencesToAiContext(Map sourceDat *
  • An object - serialized to JSON string
    • *
* - * @param sourceData OSI data containing ai_context + * @param sourceData Ossie data containing ai_context * @param outputData Salesforce data to populate with businessPreferences */ private void convertAiContextToBusinessPreferences(Map sourceData, Map outputData) { @@ -129,7 +148,7 @@ private void convertAiContextToBusinessPreferences(Map sourceDat /** * Applies default values for required Salesforce semantic model properties. - * Used when converting OSI → Salesforce. + * Used when converting Ossie → Salesforce. */ private void applyDefaults(Map outputData) { if (!outputData.containsKey(LABEL)) { diff --git a/converters/salesforce/src/main/java/org/osi/converter/pipeline/DirectionConfig.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/DirectionConfig.java similarity index 58% rename from converters/salesforce/src/main/java/org/osi/converter/pipeline/DirectionConfig.java rename to converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/DirectionConfig.java index b7df1db..0978f91 100644 --- a/converters/salesforce/src/main/java/org/osi/converter/pipeline/DirectionConfig.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/DirectionConfig.java @@ -1,6 +1,25 @@ -package org.osi.converter.pipeline; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter.pipeline; -import static org.osi.converter.ConverterConstants.*; +import static org.apache.ossie.converter.ConverterConstants.*; /** * Configuration for a specific conversion direction. diff --git a/converters/salesforce/src/main/java/org/osi/converter/pipeline/HandlerFactory.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/HandlerFactory.java similarity index 62% rename from converters/salesforce/src/main/java/org/osi/converter/pipeline/HandlerFactory.java rename to converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/HandlerFactory.java index 4b98824..f4f7591 100644 --- a/converters/salesforce/src/main/java/org/osi/converter/pipeline/HandlerFactory.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/HandlerFactory.java @@ -1,7 +1,26 @@ -package org.osi.converter.pipeline; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter.pipeline; -import org.osi.converter.*; -import org.osi.exception.ConversionException; +import org.apache.ossie.converter.*; +import org.apache.ossie.exception.ConversionException; /** * Factory for creating handler instances using a hardcoded registry. diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineConfig.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineConfig.java new file mode 100644 index 0000000..d1c3cc7 --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineConfig.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter.pipeline; + +import java.util.List; +import java.util.Map; + +/** + * Root configuration model matching pipeline-config.yaml structure. + * + */ +public class PipelineConfig { + private Map> pipelines; // Direction -> handler names + private Map directionConfigs; // Direction -> config + + public Map> getPipelines() { + return pipelines; + } + + public void setPipelines(Map> pipelines) { + this.pipelines = pipelines; + } + + public Map getDirectionConfigs() { + return directionConfigs; + } + + public void setDirectionConfigs(Map directionConfigs) { + this.directionConfigs = directionConfigs; + } +} diff --git a/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineConfigLoader.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineConfigLoader.java similarity index 71% rename from converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineConfigLoader.java rename to converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineConfigLoader.java index 6805a54..6cf1913 100644 --- a/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineConfigLoader.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineConfigLoader.java @@ -1,9 +1,28 @@ -package org.osi.converter.pipeline; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter.pipeline; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.osi.exception.ConversionException; +import org.apache.ossie.exception.ConversionException; import java.io.IOException; import java.io.InputStream; diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineStep.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineStep.java new file mode 100644 index 0000000..95fef55 --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineStep.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter.pipeline; + +import java.util.Map; + +/** + * Base interface for pipeline steps. + * Both mapping handlers and special wrapper steps implement this interface. + * + */ +public interface PipelineStep { + /** + * Execute this pipeline step. + * + * @param sourceData The source data (may be modified by handler) + * @param outputData The output data being built + * @param mappings Property mappings + */ + void execute(Map sourceData, Map outputData, Map mappings); +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/exception/ConversionException.java b/converters/salesforce/src/main/java/org/apache/ossie/exception/ConversionException.java new file mode 100644 index 0000000..704622d --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/exception/ConversionException.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.exception; + +/** + * Exception thrown when a conversion operation fails. + * This can happen during YAML to JSON or JSON to YAML conversion. + * + */ +public class ConversionException extends RuntimeException { + + /** + * Constructs a new ConversionException with the specified detail message. + * + * @param message the detail message + */ + public ConversionException(String message) { + super(message); + } + + /** + * Constructs a new ConversionException with the specified detail message and cause. + * + * @param message the detail message + * @param cause the cause of the exception + */ + public ConversionException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/exception/InvalidInputException.java b/converters/salesforce/src/main/java/org/apache/ossie/exception/InvalidInputException.java new file mode 100644 index 0000000..0c52a8e --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/exception/InvalidInputException.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.exception; + +/** + * Exception thrown when the input is invalid. + * This includes cases where the file does not exist, is not readable, + * or if the file/String contains invalid YAML/JSON content. + * + */ +public class InvalidInputException extends RuntimeException { + + /** + * Constructs a new InvalidInputException with the specified detail message. + * + * @param message the detail message + */ + public InvalidInputException(String message) { + super(message); + } + + /** + * Constructs a new InvalidInputException with the specified detail message and cause. + * + * @param message the detail message + * @param cause the cause of the exception + */ + public InvalidInputException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/exception/ValidationException.java b/converters/salesforce/src/main/java/org/apache/ossie/exception/ValidationException.java new file mode 100644 index 0000000..f6c33b3 --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/exception/ValidationException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.exception; + +/** + * Exception thrown when schema validation fails. + * + */ +public class ValidationException extends RuntimeException { + + public ValidationException(String message) { + super(message); + } + + public ValidationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/converters/salesforce/src/main/java/org/osi/mapper/FileBasedPropertyMapper.java b/converters/salesforce/src/main/java/org/apache/ossie/mapper/FileBasedPropertyMapper.java similarity index 73% rename from converters/salesforce/src/main/java/org/osi/mapper/FileBasedPropertyMapper.java rename to converters/salesforce/src/main/java/org/apache/ossie/mapper/FileBasedPropertyMapper.java index 40d4783..a40e6e0 100644 --- a/converters/salesforce/src/main/java/org/osi/mapper/FileBasedPropertyMapper.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/mapper/FileBasedPropertyMapper.java @@ -1,9 +1,28 @@ -package org.osi.mapper; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.mapper; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.osi.exception.InvalidInputException; +import org.apache.ossie.exception.InvalidInputException; import java.io.IOException; import java.io.InputStream; import java.util.Collections; @@ -14,7 +33,7 @@ /** * Property mapper that loads mappings from a YAML configuration file. * - *

This mapper reads the bundled mappings.yaml which defines OSI-Salesforce + *

This mapper reads the bundled mappings.yaml which defines Ossie-Salesforce * property mappings. The transformation is performed by the converter classes.

* */ @@ -28,7 +47,7 @@ public class FileBasedPropertyMapper implements PropertyMapper { /** * Constructs a FileBasedPropertyMapper with the specified mappings. * - * @param mappings the OSI-Salesforce mappings + * @param mappings the Ossie-Salesforce mappings */ public FileBasedPropertyMapper(Map mappings) { this.mappings = mappings != null ? new LinkedHashMap<>(mappings) : new LinkedHashMap<>(); @@ -36,9 +55,9 @@ public FileBasedPropertyMapper(Map mappings) { } /** - * Generates reverse mappings (Salesforce to OSI) from the OSI to Salesforce mappings. + * Generates reverse mappings (Salesforce to Ossie) from the Ossie to Salesforce mappings. * - * @return the Salesforce-to-OSI mappings + * @return the Salesforce-to-Ossie mappings */ private Map generateReverseMappings() { return mappings.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); diff --git a/converters/salesforce/src/main/java/org/apache/ossie/mapper/PropertyMapper.java b/converters/salesforce/src/main/java/org/apache/ossie/mapper/PropertyMapper.java new file mode 100644 index 0000000..1c6ccf8 --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/mapper/PropertyMapper.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.mapper; + +import java.util.Map; + +/** + * Interface for mapping properties between Ossie and Salesforce formats during conversion. + * + *

Implementations of this interface define bidirectional mappings between Ossie YAML format + * and Salesforce JSON format. This supports nested paths using dot notation + * (e.g., "datasets.name" ↔ "semanticDataObjects.apiName").

+ * + * + */ +public interface PropertyMapper { + + /** + * Returns the mapping from Ossie property paths to Salesforce property paths. + * + * @return a map where keys are Ossie property paths and values are Salesforce property paths + */ + Map getOsiToSalesforceMappings(); + + /** + * Returns the mapping from Salesforce property paths to Ossie property paths. + *

This is the reverse mapping of {@link #getOsiToSalesforceMappings()}. + * + * @return a map where keys are Salesforce property paths and values are Ossie property paths + */ + Map getSalesforceToOsiMappings(); +} diff --git a/converters/salesforce/src/main/java/org/osi/util/DataStructureUtils.java b/converters/salesforce/src/main/java/org/apache/ossie/util/DataStructureUtils.java similarity index 81% rename from converters/salesforce/src/main/java/org/osi/util/DataStructureUtils.java rename to converters/salesforce/src/main/java/org/apache/ossie/util/DataStructureUtils.java index 87f3a02..4b40928 100644 --- a/converters/salesforce/src/main/java/org/osi/util/DataStructureUtils.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/util/DataStructureUtils.java @@ -1,4 +1,23 @@ -package org.osi.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.util; import java.util.ArrayList; import java.util.List; diff --git a/converters/salesforce/src/main/java/org/osi/util/MappingUtils.java b/converters/salesforce/src/main/java/org/apache/ossie/util/MappingUtils.java similarity index 81% rename from converters/salesforce/src/main/java/org/osi/util/MappingUtils.java rename to converters/salesforce/src/main/java/org/apache/ossie/util/MappingUtils.java index cb17380..014a9e8 100644 --- a/converters/salesforce/src/main/java/org/osi/util/MappingUtils.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/util/MappingUtils.java @@ -1,4 +1,23 @@ -package org.osi.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.util; import java.util.*; diff --git a/converters/salesforce/src/main/java/org/osi/util/PathUtils.java b/converters/salesforce/src/main/java/org/apache/ossie/util/PathUtils.java similarity index 77% rename from converters/salesforce/src/main/java/org/osi/util/PathUtils.java rename to converters/salesforce/src/main/java/org/apache/ossie/util/PathUtils.java index 4ed4861..461dd10 100644 --- a/converters/salesforce/src/main/java/org/osi/util/PathUtils.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/util/PathUtils.java @@ -1,7 +1,26 @@ -package org.osi.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.util; -import static org.osi.util.DataStructureUtils.asMap; -import static org.osi.util.DataStructureUtils.asList; +import static org.apache.ossie.util.DataStructureUtils.asMap; +import static org.apache.ossie.util.DataStructureUtils.asList; import java.util.ArrayList; import java.util.LinkedHashMap; diff --git a/converters/salesforce/src/main/java/org/osi/validator/SchemaValidator.java b/converters/salesforce/src/main/java/org/apache/ossie/validator/SchemaValidator.java similarity index 77% rename from converters/salesforce/src/main/java/org/osi/validator/SchemaValidator.java rename to converters/salesforce/src/main/java/org/apache/ossie/validator/SchemaValidator.java index 9cd0ada..b1bea2c 100644 --- a/converters/salesforce/src/main/java/org/osi/validator/SchemaValidator.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/validator/SchemaValidator.java @@ -1,4 +1,23 @@ -package org.osi.validator; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.validator; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -11,12 +30,12 @@ import java.util.Set; import java.util.stream.Collectors; -import org.osi.exception.ValidationException; +import org.apache.ossie.exception.ValidationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Validates OSI semantic model data against JSON Schema. + * Validates Ossie semantic model data against JSON Schema. * *

Uses networknt/json-schema-validator for schema validation. * Collects all validation errors for better developer experience. @@ -65,7 +84,7 @@ private JsonSchema loadSchema() { } /** - * Validates OSI semantic model data against the schema. + * Validates Ossie semantic model data against the schema. * * @param data The semantic model data to validate * @throws ValidationException if validation fails with details of all errors diff --git a/converters/salesforce/src/main/java/org/osi/converter/ConversionDirection.java b/converters/salesforce/src/main/java/org/osi/converter/ConversionDirection.java deleted file mode 100644 index e87a1b5..0000000 --- a/converters/salesforce/src/main/java/org/osi/converter/ConversionDirection.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.osi.converter; - -/** - * Enum representing the direction of conversion. - * - */ -public enum ConversionDirection { - /** - * Converting from OSI YAML format to Salesforce JSON format. - */ - OSI_TO_SALESFORCE, - - /** - * Converting from Salesforce JSON format to OSI YAML format. - */ - SALESFORCE_TO_OSI; - - /** - * Converts enum name to pipeline configuration key. - * Maps OSI_TO_SALESFORCE -> "osiToSalesforce" and SALESFORCE_TO_OSI -> "salesforceToOsi" - * - * @return The pipeline key used in YAML configuration - */ - public String toPipelineKey() { - return switch (this) { - case OSI_TO_SALESFORCE -> "osiToSalesforce"; - case SALESFORCE_TO_OSI -> "salesforceToOsi"; - }; - } -} diff --git a/converters/salesforce/src/main/java/org/osi/converter/Converter.java b/converters/salesforce/src/main/java/org/osi/converter/Converter.java deleted file mode 100644 index 97d0848..0000000 --- a/converters/salesforce/src/main/java/org/osi/converter/Converter.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.osi.converter; - -import java.nio.file.Path; -import java.util.List; - -/** - * Interface for converting between data formats (YAML and JSON). - * - *

This is the external API for conversion. Implementations handle the conversion - * of data from one format to another, with support for property mapping.

- * - */ -public interface Converter { - - /** - * Converts the input file and writes results to the specified output directory. - * - *

Each semantic model is written to a separate file named after its apiName. - * Example: "Sales_Model.json", "Marketing_Model.json" - * - * @param inputPath the path to the input file - * @param outputDir the directory where output files will be written - */ - void convert(Path inputPath, Path outputDir); - - /** - * Converts string content from the source format to the target format. - * - *

For OSI to Salesforce: returns one Salesforce model per OSI semantic_model entry. - *

For Salesforce to OSI: returns one OSI document with one semantic_model entry. - * - * @param content the content to convert - * @return list of converted content strings (one per semantic model) - */ - List convert(String content); -} diff --git a/converters/salesforce/src/main/java/org/osi/converter/ConverterFactory.java b/converters/salesforce/src/main/java/org/osi/converter/ConverterFactory.java deleted file mode 100644 index 9e78e9b..0000000 --- a/converters/salesforce/src/main/java/org/osi/converter/ConverterFactory.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.osi.converter; - -/** - * Factory for creating converters based on conversion direction. - * - */ -public class ConverterFactory { - - /** - * Creates a converter for the specified direction. - * - * @param direction The conversion direction - * @return A Converter instance configured for the specified direction - */ - public static Converter getConverter(ConversionDirection direction) { - return new ConverterImpl(direction); - } -} diff --git a/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineConfig.java b/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineConfig.java deleted file mode 100644 index ec9f142..0000000 --- a/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineConfig.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.osi.converter.pipeline; - -import java.util.List; -import java.util.Map; - -/** - * Root configuration model matching pipeline-config.yaml structure. - * - */ -public class PipelineConfig { - private Map> pipelines; // Direction -> handler names - private Map directionConfigs; // Direction -> config - - public Map> getPipelines() { - return pipelines; - } - - public void setPipelines(Map> pipelines) { - this.pipelines = pipelines; - } - - public Map getDirectionConfigs() { - return directionConfigs; - } - - public void setDirectionConfigs(Map directionConfigs) { - this.directionConfigs = directionConfigs; - } -} diff --git a/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineStep.java b/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineStep.java deleted file mode 100644 index bec154d..0000000 --- a/converters/salesforce/src/main/java/org/osi/converter/pipeline/PipelineStep.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.osi.converter.pipeline; - -import java.util.Map; - -/** - * Base interface for pipeline steps. - * Both mapping handlers and special wrapper steps implement this interface. - * - */ -public interface PipelineStep { - /** - * Execute this pipeline step. - * - * @param sourceData The source data (may be modified by handler) - * @param outputData The output data being built - * @param mappings Property mappings - */ - void execute(Map sourceData, Map outputData, Map mappings); -} diff --git a/converters/salesforce/src/main/java/org/osi/exception/ConversionException.java b/converters/salesforce/src/main/java/org/osi/exception/ConversionException.java deleted file mode 100644 index f9d8fdd..0000000 --- a/converters/salesforce/src/main/java/org/osi/exception/ConversionException.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.osi.exception; - -/** - * Exception thrown when a conversion operation fails. - * This can happen during YAML to JSON or JSON to YAML conversion. - * - */ -public class ConversionException extends RuntimeException { - - /** - * Constructs a new ConversionException with the specified detail message. - * - * @param message the detail message - */ - public ConversionException(String message) { - super(message); - } - - /** - * Constructs a new ConversionException with the specified detail message and cause. - * - * @param message the detail message - * @param cause the cause of the exception - */ - public ConversionException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/converters/salesforce/src/main/java/org/osi/exception/InvalidInputException.java b/converters/salesforce/src/main/java/org/osi/exception/InvalidInputException.java deleted file mode 100644 index 079dc63..0000000 --- a/converters/salesforce/src/main/java/org/osi/exception/InvalidInputException.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.osi.exception; - -/** - * Exception thrown when the input is invalid. - * This includes cases where the file does not exist, is not readable, - * or if the file/String contains invalid YAML/JSON content. - * - */ -public class InvalidInputException extends RuntimeException { - - /** - * Constructs a new InvalidInputException with the specified detail message. - * - * @param message the detail message - */ - public InvalidInputException(String message) { - super(message); - } - - /** - * Constructs a new InvalidInputException with the specified detail message and cause. - * - * @param message the detail message - * @param cause the cause of the exception - */ - public InvalidInputException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/converters/salesforce/src/main/java/org/osi/exception/ValidationException.java b/converters/salesforce/src/main/java/org/osi/exception/ValidationException.java deleted file mode 100644 index 68d6268..0000000 --- a/converters/salesforce/src/main/java/org/osi/exception/ValidationException.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.osi.exception; - -/** - * Exception thrown when schema validation fails. - * - */ -public class ValidationException extends RuntimeException { - - public ValidationException(String message) { - super(message); - } - - public ValidationException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/converters/salesforce/src/main/java/org/osi/mapper/PropertyMapper.java b/converters/salesforce/src/main/java/org/osi/mapper/PropertyMapper.java deleted file mode 100644 index af8789b..0000000 --- a/converters/salesforce/src/main/java/org/osi/mapper/PropertyMapper.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.osi.mapper; - -import java.util.Map; - -/** - * Interface for mapping properties between OSI and Salesforce formats during conversion. - * - *

Implementations of this interface define bidirectional mappings between OSI YAML format - * and Salesforce JSON format. This supports nested paths using dot notation - * (e.g., "datasets.name" ↔ "semanticDataObjects.apiName").

- * - * - */ -public interface PropertyMapper { - - /** - * Returns the mapping from OSI property paths to Salesforce property paths. - * - * @return a map where keys are OSI property paths and values are Salesforce property paths - */ - Map getOsiToSalesforceMappings(); - - /** - * Returns the mapping from Salesforce property paths to OSI property paths. - *

This is the reverse mapping of {@link #getOsiToSalesforceMappings()}. - * - * @return a map where keys are Salesforce property paths and values are OSI property paths - */ - Map getSalesforceToOsiMappings(); -} diff --git a/converters/salesforce/src/main/resources/mappings.yaml b/converters/salesforce/src/main/resources/mappings.yaml index fddc085..57552cf 100644 --- a/converters/salesforce/src/main/resources/mappings.yaml +++ b/converters/salesforce/src/main/resources/mappings.yaml @@ -1,8 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # ============================================================================= -# OSI-Salesforce Converter - Mapping Configuration +# Ossie-Salesforce Converter - Mapping Configuration # ============================================================================= # -# This file defines ONLY straightforward property mappings between OSI YAML +# This file defines ONLY straightforward property mappings between Ossie YAML # format and Salesforce Semantic Model JSON format. # # Complex mappings (arrays, routing, transformations) are handled programmatically diff --git a/converters/salesforce/src/main/resources/osi-salesforce-converter-config.yaml b/converters/salesforce/src/main/resources/osi-salesforce-converter-config.yaml index fea5fea..371c90d 100644 --- a/converters/salesforce/src/main/resources/osi-salesforce-converter-config.yaml +++ b/converters/salesforce/src/main/resources/osi-salesforce-converter-config.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Pipeline definitions - each direction lists handlers to execute in order pipelines: diff --git a/converters/salesforce/src/test/java/org/osi/OsiToSalesforceConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/OsiToSalesforceConverterTest.java similarity index 89% rename from converters/salesforce/src/test/java/org/osi/OsiToSalesforceConverterTest.java rename to converters/salesforce/src/test/java/org/apache/ossie/OsiToSalesforceConverterTest.java index defdd36..94fa6b2 100644 --- a/converters/salesforce/src/test/java/org/osi/OsiToSalesforceConverterTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/OsiToSalesforceConverterTest.java @@ -1,19 +1,38 @@ -package org.osi; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import org.osi.converter.Converter; -import org.osi.converter.ConverterFactory; -import org.osi.converter.ConversionDirection; -import org.osi.converter.CustomExtensionHandler; -import org.osi.validator.SchemaValidator; +import org.apache.ossie.converter.Converter; +import org.apache.ossie.converter.ConverterFactory; +import org.apache.ossie.converter.ConversionDirection; +import org.apache.ossie.converter.CustomExtensionHandler; +import org.apache.ossie.validator.SchemaValidator; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.osi.converter.pipeline.DirectionConfig; -import org.osi.converter.pipeline.HandlerFactory; -import org.osi.converter.pipeline.PipelineConfig; -import org.osi.converter.pipeline.PipelineConfigLoader; +import org.apache.ossie.converter.pipeline.DirectionConfig; +import org.apache.ossie.converter.pipeline.HandlerFactory; +import org.apache.ossie.converter.pipeline.PipelineConfig; +import org.apache.ossie.converter.pipeline.PipelineConfigLoader; import java.io.IOException; import java.nio.file.Files; @@ -25,7 +44,7 @@ import static org.junit.jupiter.api.Assumptions.assumeTrue; /** - * Comprehensive integration test for OSI → Salesforce conversion. + * Comprehensive integration test for Ossie → Salesforce conversion. * Uses real example file: osiToSalesforce.yaml */ class OsiToSalesforceConverterTest { @@ -55,10 +74,10 @@ static void checkSchemaAvailability() { System.err.println(" and save it to: src/main/resources/schemas/salesforce-semantic-model-schema.json\n"); } if (!osiSchemaExists) { - System.err.println("\n WARNING: OSI schema not found at " + SchemaValidator.OSI_SCHEMA_PATH); + System.err.println("\n WARNING: Ossie schema not found at " + SchemaValidator.OSI_SCHEMA_PATH); System.err.println(" Skipping OsiToSalesforceConverterTest tests."); System.err.println(" To run these tests, download the schema from:"); - System.err.println(" https://github.com/open-semantic-interchange/OSI/blob/main/core-spec/osi-schema.json"); + System.err.println(" https://github.com/apache/ossie/blob/main/core-spec/osi-schema.json"); System.err.println(" and save it to: src/main/resources/schemas/osi-schema.json\n"); } warningPrinted = true; @@ -67,7 +86,7 @@ static void checkSchemaAvailability() { @BeforeEach void setUp() throws IOException { - assumeTrue(osiSchemaExists, "OSI schema file is required but not found. See README for setup instructions."); + assumeTrue(osiSchemaExists, "Ossie schema file is required but not found. See README for setup instructions."); converter = ConverterFactory.getConverter(ConversionDirection.OSI_TO_SALESFORCE); jsonMapper = new ObjectMapper(); @@ -255,7 +274,7 @@ void testMetricsNotConvertedInOsiToSalesforce() throws Exception { Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); List> calcMeasurements = (List>) sfModel.get("semanticCalculatedMeasurements"); - assertNull(calcMeasurements, "Metrics from OSI are not converted to semanticCalculatedMeasurements in OSI->SF direction"); + assertNull(calcMeasurements, "Metrics from Ossie are not converted to semanticCalculatedMeasurements in Ossie->SF direction"); } @Test diff --git a/converters/salesforce/src/test/java/org/osi/SalesforceToOsiConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOsiConverterTest.java similarity index 90% rename from converters/salesforce/src/test/java/org/osi/SalesforceToOsiConverterTest.java rename to converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOsiConverterTest.java index 9a11af7..1926843 100644 --- a/converters/salesforce/src/test/java/org/osi/SalesforceToOsiConverterTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOsiConverterTest.java @@ -1,16 +1,35 @@ -package org.osi; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.osi.converter.Converter; -import org.osi.converter.ConverterFactory; -import org.osi.converter.ConversionDirection; -import org.osi.converter.CustomExtensionHandler; -import org.osi.validator.SchemaValidator; +import org.apache.ossie.converter.Converter; +import org.apache.ossie.converter.ConverterFactory; +import org.apache.ossie.converter.ConversionDirection; +import org.apache.ossie.converter.CustomExtensionHandler; +import org.apache.ossie.validator.SchemaValidator; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.osi.converter.pipeline.*; +import org.apache.ossie.converter.pipeline.*; import java.io.IOException; import java.nio.file.Files; @@ -22,7 +41,7 @@ import static org.junit.jupiter.api.Assumptions.assumeTrue; /** - * Comprehensive integration test for Salesforce → OSI conversion. + * Comprehensive integration test for Salesforce → Ossie conversion. * Uses real example file: salesforceToOsi.json */ class SalesforceToOsiConverterTest { @@ -51,10 +70,10 @@ static void checkSchemaAvailability() { System.err.println(" and save it to: src/main/resources/schemas/salesforce-semantic-model-schema.json\n"); } if (!osiSchemaExists) { - System.err.println("\n WARNING: OSI schema not found at " + SchemaValidator.OSI_SCHEMA_PATH); + System.err.println("\n WARNING: Ossie schema not found at " + SchemaValidator.OSI_SCHEMA_PATH); System.err.println(" Skipping SalesforceToOsiConverterTest tests."); System.err.println(" To run these tests, download the schema from:"); - System.err.println(" https://github.com/open-semantic-interchange/OSI/blob/main/core-spec/osi-schema.json"); + System.err.println(" https://github.com/apache/ossie/blob/main/core-spec/osi-schema.json"); System.err.println(" and save it to: src/main/resources/schemas/osi-schema.json\n"); } warningPrinted = true; @@ -64,7 +83,7 @@ static void checkSchemaAvailability() { @BeforeEach void setUp() throws IOException { assumeTrue(schemaExists, "Salesforce schema file is required but not found. See README for setup instructions."); - assumeTrue(osiSchemaExists, "OSI schema file is required but not found. See README for setup instructions."); + assumeTrue(osiSchemaExists, "Ossie schema file is required but not found. See README for setup instructions."); converter = ConverterFactory.getConverter(ConversionDirection.SALESFORCE_TO_OSI); yamlMapper = new ObjectMapper(new YAMLFactory()); @@ -303,7 +322,7 @@ void testOutputCompilesWithOsiSchema() throws Exception { Map osiRoot = yamlMapper.readValue(osiYaml, Map.class); SchemaValidator validator = new SchemaValidator(new ObjectMapper(), SchemaValidator.OSI_SCHEMA_PATH); - assertDoesNotThrow(() -> validator.validate(osiRoot), "Output should comply with OSI schema"); + assertDoesNotThrow(() -> validator.validate(osiRoot), "Output should comply with Ossie schema"); } @Test @@ -368,11 +387,11 @@ void testHandlerFactoryForSalesforceToOsi() { @Test void testConverterImplExtractModelNameFromOsiFormat() throws Exception { - // Test extractModelName specifically handles OSI wrapped format + // Test extractModelName specifically handles Ossie wrapped format List results = converter.convert(salesforceJson); String osiYaml = results.get(0); - // The result is wrapped OSI format - extractModelName should handle this + // The result is wrapped Ossie format - extractModelName should handle this Map osiRoot = yamlMapper.readValue(osiYaml, Map.class); assertTrue(osiRoot.containsKey("semantic_model")); diff --git a/converters/salesforce/src/test/resources/examples/osiToSalesforce.yaml b/converters/salesforce/src/test/resources/examples/osiToSalesforce.yaml index 46e578b..edb1859 100644 --- a/converters/salesforce/src/test/resources/examples/osiToSalesforce.yaml +++ b/converters/salesforce/src/test/resources/examples/osiToSalesforce.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + version: 0.2.0.dev0 semantic_model: - name: Customer_Orders_Model diff --git a/converters/snowflake/README.md b/converters/snowflake/README.md index 93ee961..249624e 100644 --- a/converters/snowflake/README.md +++ b/converters/snowflake/README.md @@ -1,6 +1,25 @@ -# OSI to Snowflake Converter + + +# Apache Ossie to Snowflake Converter + +Converts Ossie YAML semantic models to [Snowflake Cortex Analyst](https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-analyst) semantic model YAML. Pure offline conversion — no Snowflake connection required. > **Note:** This converter is under active development. It handles common cases but has not been thoroughly tested against all edge cases — use with caution in production. @@ -24,4 +43,4 @@ python3 -m pytest tests/ ## Limitations -Some OSI concepts (e.g., `ai_context` on relationships) do not have a native counterpart in the Snowflake semantic model. These are dropped during conversion and the converter will emit warnings so you know what was left behind. +Some Ossie concepts (e.g., `ai_context` on relationships) do not have a native counterpart in the Snowflake semantic model. These are dropped during conversion and the converter will emit warnings so you know what was left behind. diff --git a/converters/snowflake/requirements.txt b/converters/snowflake/requirements.txt index 8d5fd98..c5dcb69 100644 --- a/converters/snowflake/requirements.txt +++ b/converters/snowflake/requirements.txt @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + PyYAML>=5.0 diff --git a/converters/snowflake/src/osi_to_snowflake_yaml_converter.py b/converters/snowflake/src/osi_to_snowflake_yaml_converter.py index f2a8520..17ef387 100644 --- a/converters/snowflake/src/osi_to_snowflake_yaml_converter.py +++ b/converters/snowflake/src/osi_to_snowflake_yaml_converter.py @@ -1,5 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ -Converts an OSI (Open Semantic Interchange) YAML semantic model to a Snowflake +Converts an Ossie (Open Semantic Interchange) YAML semantic model to a Snowflake Cortex Analyst semantic model YAML. Pure offline conversion — no Snowflake connection required. @@ -18,21 +35,21 @@ class OsiConversionError(Exception): - """Raised when an OSI YAML cannot be converted to Snowflake format.""" + """Raised when an Ossie YAML cannot be converted to Snowflake format.""" def convert_osi_to_snowflake(osi_yaml_str): - """Top-level entry point. Parses OSI YAML, validates, converts, returns + """Top-level entry point. Parses Ossie YAML, validates, converts, returns Snowflake YAML string. - Expects the standard OSI wrapped format:: + Expects the standard Ossie wrapped format:: version: "0.2.0.dev0" semantic_model: - name: ... Args: - osi_yaml_str: OSI YAML as a string. + osi_yaml_str: Ossie YAML as a string. Returns: Snowflake Cortex Analyst semantic model YAML string. @@ -42,34 +59,34 @@ def convert_osi_to_snowflake(osi_yaml_str): """ root = yaml.safe_load(osi_yaml_str) if not isinstance(root, dict): - raise OsiConversionError("Invalid OSI YAML: expected a mapping at the root") + raise OsiConversionError("Invalid Ossie YAML: expected a mapping at the root") version_str = str(root.get("version", "")) if version_str != SUPPORTED_VERSION: raise OsiConversionError( - f"Unsupported OSI specification version '{version_str}'. " + f"Unsupported Ossie specification version '{version_str}'. " f"Supported: {SUPPORTED_VERSION}" ) semantic_model = root.get("semantic_model") if not isinstance(semantic_model, list) or len(semantic_model) == 0: raise OsiConversionError( - "Invalid OSI YAML: 'semantic_model' must be a non-empty list" + "Invalid Ossie YAML: 'semantic_model' must be a non-empty list" ) if len(semantic_model) > 1: warnings.warn( - f"OSI YAML contains {len(semantic_model)} semantic models; " + f"Ossie YAML contains {len(semantic_model)} semantic models; " f"only the first will be converted" ) - osi = semantic_model[0] - if not isinstance(osi, dict): + ossie = semantic_model[0] + if not isinstance(ossie, dict): raise OsiConversionError( - "Invalid OSI YAML: 'semantic_model' entries must be mappings" + "Invalid Ossie YAML: 'semantic_model' entries must be mappings" ) - snowflake_model = _convert_model(osi) + snowflake_model = _convert_model(ossie) return yaml.dump( snowflake_model, @@ -79,38 +96,38 @@ def convert_osi_to_snowflake(osi_yaml_str): ) -def _convert_model(osi): - """Converts the root OSI model dict to a Snowflake semantic model dict.""" - name = osi.get("name") +def _convert_model(ossie): + """Converts the root Ossie model dict to a Snowflake semantic model dict.""" + name = ossie.get("name") if not name: raise OsiConversionError("Missing required 'name' field in semantic model") result = {} result["name"] = name - description = osi.get("description") - ai_context = osi.get("ai_context") + description = ossie.get("description") + ai_context = ossie.get("ai_context") if isinstance(ai_context, str) and ai_context: description = f"{description}\n{ai_context}" if description else ai_context if description: result["description"] = description # datasets -> tables - datasets = osi.get("datasets") + datasets = ossie.get("datasets") if datasets: tables = [_convert_dataset(ds) for ds in datasets] if tables: result["tables"] = tables # relationships - relationships = osi.get("relationships") + relationships = ossie.get("relationships") if relationships: converted_rels = [_convert_relationship(rel) for rel in relationships] if converted_rels: result["relationships"] = converted_rels # metrics - metrics = osi.get("metrics") + metrics = ossie.get("metrics") if metrics: converted_metrics = [] for m in metrics: @@ -121,13 +138,13 @@ def _convert_model(osi): result["metrics"] = converted_metrics dropped_ai = ["ai_context"] if isinstance(ai_context, dict) and ai_context else [] - _warn_dropped_fields(osi, "model", extra_dropped=dropped_ai) + _warn_dropped_fields(ossie, "model", extra_dropped=dropped_ai) return result def _convert_dataset(dataset): - """Converts an OSI dataset dict to a Snowflake table dict.""" + """Converts an Ossie dataset dict to a Snowflake table dict.""" result = {} name = dataset.get("name") if not name: @@ -209,11 +226,11 @@ def _classify_field(field): def _convert_named_expr(entry, kind): - """Converts an OSI field or metric dict to a Snowflake entry with name, expr, + """Converts an Ossie field or metric dict to a Snowflake entry with name, expr, description, and synonyms. Args: - entry: The OSI field or metric dict. + entry: The Ossie field or metric dict. kind: Human-readable type for error messages (e.g., "field", "metric"). """ name = entry.get("name") @@ -250,7 +267,7 @@ def _convert_named_expr(entry, kind): def _convert_relationship(rel): - """Converts an OSI relationship dict to a Snowflake relationship dict.""" + """Converts an Ossie relationship dict to a Snowflake relationship dict.""" result = {} rel_name = rel.get("name") if not rel_name: @@ -342,7 +359,7 @@ def _normalize_identifier(identifier): def _parse_source(source): - """Parses an OSI dataset source string into a Snowflake base_table dict. + """Parses an Ossie dataset source string into a Snowflake base_table dict. Returns None if source is empty/None. Returns {"definition": source} for subqueries. Otherwise splits into 3-part db.schema.table. @@ -393,13 +410,13 @@ def _extract_synonyms(ai_context): def _warn_dropped_fields(source, context, extra_dropped=None): - """Warns about OSI fields that have no Snowflake counterpart and are dropped. + """Warns about Ossie fields that have no Snowflake counterpart and are dropped. Checks for universally-dropped fields (custom_extensions, label, version). Callers handle ai_context warnings themselves since consumption logic varies. Args: - source: The OSI dict being converted. + source: The Ossie dict being converted. context: Human-readable description (e.g., "field 'col1'"). extra_dropped: Optional list of additional field descriptions to report as dropped (e.g., ai_context details computed by the caller). @@ -424,10 +441,10 @@ def _warn_dropped_fields(source, context, extra_dropped=None): def main(): parser = argparse.ArgumentParser( - description="Convert OSI YAML semantic model to Snowflake Cortex Analyst YAML" + description="Convert Ossie YAML semantic model to Snowflake Cortex Analyst YAML" ) parser.add_argument( - "-i", "--input", required=True, help="Path to the OSI YAML input file" + "-i", "--input", required=True, help="Path to the Ossie YAML input file" ) parser.add_argument( "-o", "--output", required=True, help="Path to write the Snowflake YAML output" diff --git a/converters/snowflake/tests/example_converted_tpcds_semantic_model.yaml b/converters/snowflake/tests/example_converted_tpcds_semantic_model.yaml index 9220ab6..b2d4aff 100644 --- a/converters/snowflake/tests/example_converted_tpcds_semantic_model.yaml +++ b/converters/snowflake/tests/example_converted_tpcds_semantic_model.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + name: tpcds_retail_model description: TPC-DS retail semantic model for sales and customer analytics tables: diff --git a/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py b/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py index 7a8de1c..0c732e8 100644 --- a/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py +++ b/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py @@ -1,4 +1,21 @@ -"""Tests for the OSI to Snowflake YAML converter.""" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the Ossie to Snowflake YAML converter.""" import sys import warnings @@ -29,7 +46,7 @@ # --------------------------------------------------------------------------- def _wrap_osi(model_dict): - """Wrap a model dict in the standard OSI envelope.""" + """Wrap a model dict in the standard Ossie envelope.""" return yaml.dump( {"version": "0.2.0.dev0", "semantic_model": [model_dict]}, default_flow_style=False, @@ -37,7 +54,7 @@ def _wrap_osi(model_dict): def _minimal_model(**overrides): - """Return a minimal valid OSI model dict.""" + """Return a minimal valid Ossie model dict.""" base = { "name": "test_model", "datasets": [ @@ -478,7 +495,7 @@ def test_invalid_yaml_root_raises(self): def test_wrong_version_raises(self): bad = yaml.dump({"version": "9.9.9", "semantic_model": [{"name": "m"}]}) - with pytest.raises(OsiConversionError, match="Unsupported OSI specification"): + with pytest.raises(OsiConversionError, match="Unsupported Ossie specification"): convert_osi_to_snowflake(bad) def test_missing_semantic_model_raises(self): @@ -605,7 +622,7 @@ def test_subquery_source(self): # --------------------------------------------------------------------------- -# _warn_dropped_fields (OSI concepts with no Snowflake counterpart) +# _warn_dropped_fields (Ossie concepts with no Snowflake counterpart) # --------------------------------------------------------------------------- class TestWarnDroppedFields: diff --git a/core-spec/osi-schema.json b/core-spec/osi-schema.json index 72cb164..1732d4d 100644 --- a/core-spec/osi-schema.json +++ b/core-spec/osi-schema.json @@ -1,14 +1,14 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/open-semantic-interchange/OSI/core-spec/osi-schema.json", - "title": "OSI Core Metadata Specification", - "description": "JSON Schema for validating OSI (Open Semantic Interoperability) semantic model definitions", + "$id": "https://github.com/apache/ossie/core-spec/osi-schema.json", + "title": "Apache Ossie Core Metadata Specification", + "description": "JSON Schema for validating Apache Ossie semantic model definitions", "type": "object", "properties": { "version": { "type": "string", "const": "0.2.0.dev0", - "description": "OSI specification version" + "description": "Apache Ossie specification version" }, "semantic_model": { "type": "array", diff --git a/core-spec/spec.md b/core-spec/spec.md index ea7e775..b544042 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -1,4 +1,23 @@ -# OSI - Core Metadata Specification + + +# Apache Ossie - Core Metadata Specification > **DRAFT version** — in development, schema may change before 0.2.0 is released. diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index ed46203..316d784 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -1,4 +1,21 @@ -# OSI - Core Metadata Spec (YAML Schema) +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Apache Ossie - Core Metadata Spec (YAML Schema) # DRAFT version - in development, schema may change before 0.2.0 is released version: 0.2.0.dev0 # diff --git a/docs/index.md b/docs/index.md index fa9b564..4365166 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,10 +1,31 @@ -# Open Semantic Interchange (OSI) + + +# Apache Ossie ## Overview -The [Open Semantic Interchange (OSI)](https://open-semantic-interchange.org/) initiative is a collaborative, open-source effort dedicated to standardizing and streamlining semantic model exchange and utilization across the data analytics, AI, and BI ecosystem. Our shared vision is to establish a common, vendor-agnostic semantic model specification, promoting interoperability, efficiency, and collaboration among all participants. +The [Apache Ossie](https://ossie.apache.org/) initiative is a collaborative, open-source effort dedicated to standardizing and streamlining semantic model exchange and utilization across the data analytics, AI, and BI ecosystem. Our shared vision is to establish a common, vendor-agnostic semantic model specification, promoting interoperability, efficiency, and collaboration among all participants. -By providing a single, consistent source of truth, the OSI standard ensures that your data's definitions and value remain consistent as they are interchanged between AI agents, BI platforms, and all other tools in your ecosystem — eliminating inconsistencies across your different tools. +Apache Ossie was formerly known as **Open Semantic Interchange (OSI)**. + +By providing a single, consistent source of truth, the Ossie standard ensures that your data's definitions and value remain consistent as they are interchanged between AI agents, BI platforms, and all other tools in your ecosystem — eliminating inconsistencies across your different tools. ### The Problem: Semantic Fragmentation @@ -15,18 +36,18 @@ Today's data ecosystem is fragmented. Organizations rely on a wide array of tool - **AI Hallucinations**: When AI agents encounter conflicting or incomplete business logic across tools, they produce unreliable outputs grounded in inconsistent data definitions. - **Integration Debt**: Every new tool added to the stack requires custom integration work, creating a web of brittle, point-to-point connectors that are costly to maintain. -### How OSI Solves It +### How Apache Ossie Solves It -OSI addresses semantic fragmentation by providing: +Ossie addresses semantic fragmentation by providing: - **Single Source of Truth**: A unified specification for semantic and metric definitions that all tools can read and write, ensuring consistency across the entire data stack. -- **Native Interoperability**: A hub-and-spoke model where tools exchange semantic models through OSI as a common format — enabling direct platform-to-platform exchange without custom connectors. +- **Native Interoperability**: A hub-and-spoke model where tools exchange semantic models through Ossie as a common format — enabling direct platform-to-platform exchange without custom connectors. - **Trusted AI Grounding**: Consistent business logic and rich AI context annotations ensure that AI agents and LLMs can reliably interpret and query data. - **Reduced Total Cost of Ownership**: Automated model exchange eliminates manual reconciliation work and reduces the engineering effort needed to integrate new tools. ### Specification at a Glance -The OSI core specification (current version: **0.2.0.dev0**, latest released: **0.1.1**) defines a YAML-based format for describing semantic models. The key constructs are: +The Ossie core specification (current version: **0.2.0.dev0**, latest released: **0.1.1**) defines a YAML-based format for describing semantic models. The key constructs are: | Construct | Description | |-----------|-------------| @@ -44,13 +65,13 @@ For the full specification, see [core-spec/spec.md](../core-spec/spec.md). For v ### Participating Organizations -OSI is supported by a broad coalition of 50+ organizations across the data ecosystem, including: +Ossie is supported by a broad coalition of 50+ organizations across the data ecosystem, including: Alation, Anomalo, Atlan, AtScale, Bigeye, BlackRock, Blue Yonder, Carto, Cloudera, Coalesce, Collate, Collibra, Cogniti, Count, Credible, Cube, Databricks, DataHub, Denodo, dbt Labs, Dremio, Domo, Elementum AI, Firebolt, GoodData, Hex, Honeydew, Informatica, Instacart, JetBrains, Lightdash, Mistral AI, Omni, Oracle, Preset, Qlik, RelationalAI, Salesforce, Select Star, Sigma, Snowflake, Starburst Data, Strategy, Sundial, ThoughtSpot, and more. ### Converters -OSI converters follow a **hub-and-spoke** architecture: the OSI core specification acts as the central, vendor-neutral format, and each converter handles translation to or from a specific vendor format (e.g., Snowflake, dbt, Salesforce, Databricks). This avoids the need for point-to-point converters between every pair of vendors. +Ossie converters follow a **hub-and-spoke** architecture: the Ossie core specification acts as the central, vendor-neutral format, and each converter handles translation to or from a specific vendor format (e.g., Snowflake, dbt, Salesforce, Databricks). This avoids the need for point-to-point converters between every pair of vendors. For details on implementing a converter, see the [Converters Guide](../converters/index.md). @@ -58,8 +79,8 @@ For details on implementing a converter, see the [Converters Guide](../converter ## Project Governance -We wanted the OSI project to be a collaborative effort from the start. The purpose is to grow a community of developers, contributors, and users who are actively involved in shaping the OSI Specification as it moves along. -Open Semantic Interchange (OSI) project governance is inspired by the governance model from [The ASF](https://www.apache.org). +We wanted the Ossie project to be a collaborative effort from the start. The purpose is to grow a community of developers, contributors, and users who are actively involved in shaping the Ossie Specification as it moves along. +Apache Ossie project governance is inspired by the governance model from [The ASF](https://www.apache.org). ### The Apache Way @@ -82,47 +103,43 @@ All technical discussions, design decisions, and specification changes happen in The community collectively ensures that the specification and its associated tooling remain high quality, secure, and aligned with the project's mission of vendor-agnostic semantic interoperability. 6. **Independence** -The OSI project operates independently of any single vendor or organization. While contributors may be employed by companies that have a stake in semantic interoperability, the project's direction is determined by the community as a whole. +The Ossie project operates independently of any single vendor or organization. While contributors may be employed by companies that have a stake in semantic interoperability, the project's direction is determined by the community as a whole. + +### Governing Bodies -### Technical Governing Bodies +As an incubating project, Apache Ossie is governed by its **Podling Project Management Committee (PPMC)**, working alongside the project's **Mentors** and under the oversight of the **Apache Incubator PMC (IPMC)**. See [CONTRIBUTING.md](../CONTRIBUTING.md) for the full description of roles and process. -The **Technical Steering Committee (TSC)** is responsible for the overall technical direction of the OSI Specification. The TSC: +The **PPMC** is responsible for the overall direction and health of the podling. It: - Reviews and approves significant changes to the core specification -- Resolves technical disputes that cannot be settled through normal consensus +- Guides the roadmap for new specification features and extensions - Ensures backward compatibility and coherence across specification versions -- Oversees the roadmap for new specification features and extensions -- Has binding vote on the Specification Change vote - -New TSC members are elected by the TSC members. +- Casts binding votes and votes on committer/PPMC nominations and releases +- Resolves technical disputes that cannot be settled through normal consensus -The TSC members are: +New committers and PPMC members are nominated and voted on by the PPMC following the standard [ASF process](https://www.apache.org/dev/pmc.html). During incubation, releases are additionally approved by the Incubator PMC. -| Name | Affiliation | -|------|-------------| -| Khushboo Bhatia | Snowflake | -| Lior Ebel | Salesforce | -| Quigley Malcom | dbt Labs | -| JB Onofré | The ASF | +Current PPMC members and Mentors are listed on the [podling status page](https://incubator.apache.org/projects/ossie.html). -### Non-Governing Bodies +### Roles -- **Contributors**: Anyone who contributes to the project in any form — code, documentation, bug reports, specification feedback, or community support. Contributors do not have binding vote rights but are encouraged to participate in all discussions. -- **Committers**: Contributors who have earned write access to the repository through sustained contributions. Committers can merge pull requests and have binding votes on non-specification matters. -- **Specification Reviewers**: Community members with domain expertise (e.g., in BI, AI, data engineering) who review proposed specification changes for correctness, completeness, and practical applicability. +- **Contributors**: Anyone who contributes to the project in any form — code, documentation, bug reports, specification feedback, or community support. Contributor votes are non-binding, but everyone is encouraged to participate in all discussions and votes. +- **Committers**: Contributors who have earned write access to the repository through sustained contributions. Committers merge pull requests and have binding votes on the project's technical decisions. All committers have an ICLA on file. +- **PPMC Members**: Committers who also help steer the podling — growing the community, overseeing releases, and mentoring contributors. PPMC votes are binding. +- **Mentors**: Experienced ASF members assigned by the Incubator to guide the podling and shepherd release votes to the IPMC. -### Vote on Specification Changes +### Voting on Specification Changes -Changes to the OSI Specification follow a structured voting process: +Changes to the Apache Ossie specification follow the ASF voting model, held on the `dev@ossie.apache.org` mailing list: -1. **Proposal**: A specification change is submitted as a GitHub pull request with a clear description of the motivation, the change itself, and its impact on existing implementations. -2. **Discussion period**: The community has a minimum of 7 days to review and discuss the proposed change. Complex changes may require a longer review window. -3. **Vote**: Once the discussion period has elapsed, a vote is called. TSC members and committers cast votes using the following convention: - - **+1** (Yes): In favor of the change - - **0** (Abstain): No opinion or willing to go with the majority - - **-1** (Veto): Against the change — must be accompanied by a technical justification - The votes from the TSC members are considered as binding votes. -4. **Resolution**: A specification change passes with at least 2 binding +1 votes and no vetoes. A veto can only be overridden by addressing the stated concern or by a supermajority (two-thirds) vote of the TSC. +1. **Proposal**: Announce the change on `dev@ossie.apache.org` and open a GitHub pull request describing the motivation, the change, and its impact on existing implementations. +2. **Discussion period**: The community has a minimum of 7 days to review and discuss. Complex changes may require a longer window. +3. **Vote**: A `[VOTE]` thread is called. Voters cast: + - **+1**: In favor + - **0**: Abstain / no strong opinion + - **-1**: Veto — must include a technical justification; a valid veto is resolved only by addressing the stated concern + PPMC member votes are binding. +4. **Resolution**: A specification change passes with at least **three binding +1 votes** and no vetoes. ### Working Groups @@ -137,7 +154,7 @@ Community meetings are open to all participants and provide a forum for discussi - **Global Community Meetings**: Regular meetings open to all community members for cross-cutting discussions, roadmap reviews, and community announcements. - **Working Group Meetings**: Each working group holds its own meetings to drive focused progress on their specific area of the specification. -Meeting schedules, agendas, and notes are published on the [OSI website](https://open-semantic-interchange.org/) and the project's GitHub repository. All community members are welcome to attend and participate. +Meeting schedules, agendas, and notes are published on the [Ossie website](https://ossie.apache.org/) and the project's GitHub repository. All community members are welcome to attend and participate. **Google Calendar**: _link TBD_ @@ -147,7 +164,7 @@ Meeting schedules, agendas, and notes are published on the [OSI website](https:/ ### Hub-and-Spoke Model -OSI is designed around a hub-and-spoke architecture that dramatically simplifies the integration landscape. Instead of requiring every tool to build custom connectors to every other tool, OSI acts as the universal interchange format at the center. +Ossie is designed around a hub-and-spoke architecture that dramatically simplifies the integration landscape. Instead of requiring every tool to build custom connectors to every other tool, Ossie acts as the universal interchange format at the center. ``` ┌─────────────┐ @@ -155,7 +172,7 @@ OSI is designed around a hub-and-spoke architecture that dramatically simplifies └──────┬──────┘ │ ┌─────────────┐ ┌─────┴─────┐ ┌─────────────┐ - │ dbt ├────┤ OSI ├────┤ Salesforce │ + │ dbt ├────┤ Ossie ├────┤ Salesforce │ └─────────────┘ └─────┬─────┘ └─────────────┘ │ ┌──────┴──────┐ @@ -163,22 +180,22 @@ OSI is designed around a hub-and-spoke architecture that dramatically simplifies └─────────────┘ ``` -With N vendors, a point-to-point strategy would require **N×(N-1)** converters. With OSI as the hub, only **2×N** converters are needed (one import and one export per vendor), and interoperability with all other vendors comes for free. +With N vendors, a point-to-point strategy would require **N×(N-1)** converters. With Ossie as the hub, only **2×N** converters are needed (one import and one export per vendor), and interoperability with all other vendors comes for free. ### How It Flows -A typical OSI-based workflow looks like this: +A typical Ossie-based workflow looks like this: -1. **Author**: A semantic model is authored in one tool (e.g., dbt) or directly in the OSI YAML format. -2. **Import**: If authored in a vendor tool, the vendor's import converter translates it into an OSI model, preserving vendor-specific metadata in `custom_extensions`. -3. **Validate**: The OSI model is validated against the [JSON Schema](../core-spec/osi-schema.json) and the [validation script](../validation/validate.py) to ensure correctness. -4. **Exchange**: The OSI model is shared — via Git, a data catalog, or a sync API — with other teams and tools. -5. **Export**: Each consuming tool's export converter translates the OSI model into its native format, selecting the appropriate SQL dialect and applying vendor-specific extensions. -6. **Round-Trip**: When changes are made in a downstream tool, they can be imported back into the OSI model, preserving all metadata for lossless round-tripping. +1. **Author**: A semantic model is authored in one tool (e.g., dbt) or directly in the Ossie YAML format. +2. **Import**: If authored in a vendor tool, the vendor's import converter translates it into an Ossie model, preserving vendor-specific metadata in `custom_extensions`. +3. **Validate**: The Ossie model is validated against the [JSON Schema](../core-spec/osi-schema.json) and the [validation script](../validation/validate.py) to ensure correctness. +4. **Exchange**: The Ossie model is shared — via Git, a data catalog, or a sync API — with other teams and tools. +5. **Export**: Each consuming tool's export converter translates the Ossie model into its native format, selecting the appropriate SQL dialect and applying vendor-specific extensions. +6. **Round-Trip**: When changes are made in a downstream tool, they can be imported back into the Ossie model, preserving all metadata for lossless round-tripping. ### Multi-Dialect Expression System -A key architectural feature of OSI is its multi-dialect expression system. Fields and metrics can carry expressions in multiple SQL dialects simultaneously: +A key architectural feature of Ossie is its multi-dialect expression system. Fields and metrics can carry expressions in multiple SQL dialects simultaneously: ```yaml expression: @@ -202,32 +219,32 @@ This allows a single semantic model to be consumed natively by different platfor **What is a semantic model?** A semantic model is a structured description of business data that defines what datasets exist, what their fields mean, how datasets relate to each other, and what metrics (KPIs) can be computed from the data. It serves as a shared vocabulary between data producers and consumers — whether those consumers are humans using BI tools or AI agents generating queries. -**How is OSI different from existing standards?** -Most existing standards focus on data formats (e.g., Parquet, Arrow), query interfaces (e.g., ODBC, JDBC), or catalog metadata (e.g., Hive Metastore, OpenMetadata). OSI focuses specifically on the *semantic layer* — the business meaning, metric definitions, and relationships that sit on top of raw data. It is complementary to these other standards rather than a replacement. +**How is Ossie different from existing standards?** +Most existing standards focus on data formats (e.g., Parquet, Arrow), query interfaces (e.g., ODBC, JDBC), or catalog metadata (e.g., Hive Metastore, OpenMetadata). Ossie focuses specifically on the *semantic layer* — the business meaning, metric definitions, and relationships that sit on top of raw data. It is complementary to these other standards rather than a replacement. -**Is OSI tied to any specific vendor?** -No. OSI is vendor-agnostic by design. The specification is developed and governed by a community of contributors from many organizations. While vendor-specific metadata can be carried via `custom_extensions`, the core specification is neutral. +**Is Ossie tied to any specific vendor?** +No. Ossie is vendor-agnostic by design. The specification is developed and governed by a community of contributors from many organizations. While vendor-specific metadata can be carried via `custom_extensions`, the core specification is neutral. ### Adoption -**Can I use OSI with my existing BI tool?** -Yes, as long as a converter exists (or is built) for your tool. The hub-and-spoke model means that adding OSI support to a single tool gives it interoperability with every other OSI-compatible tool. Check the [Converters Guide](../converters/index.md) for currently supported vendors. +**Can I use Ossie with my existing BI tool?** +Yes, as long as a converter exists (or is built) for your tool. The hub-and-spoke model means that adding Ossie support to a single tool gives it interoperability with every other Ossie-compatible tool. Check the [Converters Guide](../converters/index.md) for currently supported vendors. **What if my vendor isn't supported yet?** You can contribute a converter. The [Converters Guide](../converters/index.md) provides a step-by-step guide for implementing import and export converters for new vendors. The community is happy to help with design reviews and testing. **Do I need to rewrite my existing semantic models?** -No. Import converters translate existing vendor-specific models into the OSI format automatically. Your existing models remain intact — OSI provides an additional interchange layer on top of them. +No. Import converters translate existing vendor-specific models into the Ossie format automatically. Your existing models remain intact — Ossie provides an additional interchange layer on top of them. -**How do I validate an OSI model?** +**How do I validate an Ossie model?** Use the [validation script](../validation/validate.py) included in the repository. It checks your model against the [JSON Schema](../core-spec/osi-schema.json), validates SQL expressions across dialects, and ensures referential integrity between datasets and relationships. ### Technical **Why YAML and not JSON?** -YAML is more human-readable and easier to author by hand, which is important for a specification that teams may edit directly. The OSI JSON Schema is available for programmatic validation, and converters can work with either format. +YAML is more human-readable and easier to author by hand, which is important for a specification that teams may edit directly. The Ossie JSON Schema is available for programmatic validation, and converters can work with either format. -**How does OSI handle vendor-specific features?** +**How does Ossie handle vendor-specific features?** Through `custom_extensions`. Each vendor can store arbitrary JSON metadata in extension blocks tagged with their vendor name. This metadata is preserved during round-trip conversions and ignored by tools that don't understand it — ensuring that no information is lost. **Can metrics reference multiple datasets?** @@ -242,7 +259,7 @@ The current specification supports `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`, `MDX`, ### Specification Versioning -The OSI specification follows [Semantic Versioning](https://semver.org/) (SemVer): +The Ossie specification follows [Semantic Versioning](https://semver.org/) (SemVer): - **Major version** (e.g., 1.0.0 → 2.0.0): Breaking changes that are not backward compatible. Existing valid models may not be valid under the new version. Major version bumps are rare and go through an extended review and migration period. - **Minor version** (e.g., 0.1.0 → 0.2.0): New features or constructs added in a backward-compatible way. Existing valid models remain valid. @@ -264,7 +281,7 @@ Custom extensions (`custom_extensions`) are explicitly outside the scope of core ## Adoption Guide -A practical guide for organizations looking to adopt OSI. +A practical guide for organizations looking to adopt Ossie. ### Phase 1: Evaluate @@ -274,22 +291,22 @@ A practical guide for organizations looking to adopt OSI. ### Phase 2: Pilot -- **Start with one model**: Choose a well-understood semantic model (e.g., a core sales or finance model) and express it in the OSI format. +- **Start with one model**: Choose a well-understood semantic model (e.g., a core sales or finance model) and express it in the Ossie format. - **Validate**: Run the [validation script](../validation/validate.py) to ensure the model conforms to the specification. -- **Test round-tripping**: If converters are available, export the OSI model to your vendor format and compare with the original. Identify any gaps or lossy conversions. +- **Test round-tripping**: If converters are available, export the Ossie model to your vendor format and compare with the original. Identify any gaps or lossy conversions. - **Gather feedback**: Share the pilot results with your data team and collect feedback on the experience. ### Phase 3: Expand -- **Convert additional models**: Gradually bring more semantic models into the OSI format, prioritizing those shared across multiple tools. -- **Integrate into workflows**: Add OSI validation to your CI/CD pipeline. Store OSI models in version control alongside your data code. +- **Convert additional models**: Gradually bring more semantic models into the Ossie format, prioritizing those shared across multiple tools. +- **Integrate into workflows**: Add Ossie validation to your CI/CD pipeline. Store Ossie models in version control alongside your data code. - **Automate synchronization**: Use converters (and eventually the Sync API) to automate the propagation of semantic model changes across your tool ecosystem. ### Phase 4: Govern - **Establish ownership**: Define who owns each semantic model and who is responsible for approving changes. - **Implement review processes**: Use the voting and review processes described in this document (or your own governance model) to manage specification changes. -- **Monitor consistency**: Regularly validate that the semantic models consumed by each tool are in sync with the authoritative OSI model. +- **Monitor consistency**: Regularly validate that the semantic models consumed by each tool are in sync with the authoritative Ossie model. --- @@ -300,15 +317,15 @@ A practical guide for organizations looking to adopt OSI. | **Semantic Model** | A structured description of business data that defines datasets, fields, relationships, and metrics. It provides a shared vocabulary for interpreting data across tools and teams. | | **Dataset** | A logical representation of a business entity, typically corresponding to a fact table or dimension table in a data warehouse. | | **Field** | A row-level attribute within a dataset, used for grouping, filtering, or as part of metric expressions. Fields can be simple column references or computed expressions. | -| **Dimension** | A categorical attribute used to slice and filter data (e.g., region, product category, date). In OSI, dimensions are represented as fields with optional metadata such as `is_time`. | +| **Dimension** | A categorical attribute used to slice and filter data (e.g., region, product category, date). In Ossie, dimensions are represented as fields with optional metadata such as `is_time`. | | **Metric** | A quantitative measure computed by aggregating data across one or more datasets (e.g., total revenue, average order value). Metrics are defined at the semantic model level. | | **Relationship** | A foreign key connection between two datasets, defining how they can be joined. Relationships are always many-to-one (from the referencing dataset to the referenced dataset). | -| **Dialect** | A specific SQL or expression language variant (e.g., `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`). OSI supports multiple dialects so expressions can be tailored to each platform. | -| **Custom Extension** | Vendor-specific metadata attached to any OSI construct as a JSON string. Extensions allow platforms to carry additional information without modifying the core specification. | -| **AI Context** | Optional annotations on any OSI construct (model, dataset, field, relationship, metric) that provide additional context for AI tools — including natural language instructions, synonyms, and example queries. | -| **Converter** | A tool that translates between the OSI format and a specific vendor's semantic model format. Converters come in pairs: import (vendor → OSI) and export (OSI → vendor). | -| **Hub-and-Spoke** | The architectural pattern used by OSI, where the specification acts as the central format (hub) and vendor converters act as spokes, avoiding the need for point-to-point integrations. | -| **Round-Trip Fidelity** | The ability to convert a model from one format to OSI and back without losing information. Achieved by preserving vendor-specific metadata in `custom_extensions`. | +| **Dialect** | A specific SQL or expression language variant (e.g., `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`). Ossie supports multiple dialects so expressions can be tailored to each platform. | +| **Custom Extension** | Vendor-specific metadata attached to any Ossie construct as a JSON string. Extensions allow platforms to carry additional information without modifying the core specification. | +| **AI Context** | Optional annotations on any Ossie construct (model, dataset, field, relationship, metric) that provide additional context for AI tools — including natural language instructions, synonyms, and example queries. | +| **Converter** | A tool that translates between the Ossie format and a specific vendor's semantic model format. Converters come in pairs: import (vendor → Ossie) and export (Ossie → vendor). | +| **Hub-and-Spoke** | The architectural pattern used by Ossie, where the specification acts as the central format (hub) and vendor converters act as spokes, avoiding the need for point-to-point integrations. | +| **Round-Trip Fidelity** | The ability to convert a model from one format to Ossie and back without losing information. Achieved by preserving vendor-specific metadata in `custom_extensions`. | | **Fact Table** | A dataset that records business events or transactions (e.g., sales, clicks, shipments). Fact tables typically contain numeric measures and foreign keys to dimension tables. | | **Dimension Table** | A dataset that describes business entities referenced by fact tables (e.g., customers, products, dates). Dimension tables provide the context for analyzing facts. | | **Semantic Layer** | The abstraction layer between raw data and business users/tools. It defines the business meaning of data, standardizes metric calculations, and provides a consistent query interface. | @@ -317,9 +334,9 @@ A practical guide for organizations looking to adopt OSI. ## Related Resources -- **Website**: [open-semantic-interchange.org](https://open-semantic-interchange.org/) -- **GitHub**: [github.com/open-semantic-interchange](https://github.com/open-semantic-interchange) -- **Slack**: [join slack](https://join.slack.com/t/apache-ossie/shared_invite/zt-42i1xkgy8-7YQtKEDq7v~mceFmdiLhkA) +- **Website**: [ossie.apache.org](https://ossie.apache.org/) +- **GitHub**: [github.com/apache/ossie](https://github.com/apache/ossie) +- **Slack**: [join slack](https://join.slack.com/t/opensemanticx/shared_invite/zt-3yuad6c0h-MaoPgVSD1g9MEOf1_QeaiQ) - **Core Specification**: [core-spec/spec.md](../core-spec/spec.md) - **JSON Schema**: [core-spec/osi-schema.json](../core-spec/osi-schema.json) - **YAML Schema**: [core-spec/spec.yaml](../core-spec/spec.yaml) @@ -345,13 +362,13 @@ We welcome contributions from everyone — whether you are a developer, a data e 1. **Read the Specification**: Familiarize yourself with the [core specification](../core-spec/spec.md) to understand the semantic model format. 2. **Explore the Examples**: Review the [TPC-DS example](../examples/tpcds_semantic_model.yaml) to see a complete semantic model in practice. -3. **Join the Conversation**: Open or participate in [GitHub Issues](https://github.com/open-semantic-interchange) and Discussions to share ideas and feedback. +3. **Join the Conversation**: Open or participate in [GitHub Issues](https://github.com/apache/ossie) and Discussions to share ideas and feedback. 4. **Submit a Pull Request**: Fork the repository, make your changes, and submit a pull request. All contributions go through the standard review process described in the governance section above. 5. **Join the Slack workspace**: You can chat directly with the community by [joining Slack](https://join.slack.com/t/apache-ossie/shared_invite/zt-42i1xkgy8-7YQtKEDq7v~mceFmdiLhkA). ### Code of Conduct -All participants in the OSI community are expected to treat each other with respect and professionalism. We are committed to providing a welcoming and inclusive environment for everyone, regardless of background or experience level. +All participants in the Ossie community are expected to treat each other with respect and professionalism. We are committed to providing a welcoming and inclusive environment for everyone, regardless of background or experience level. --- diff --git a/docs/working_groups.md b/docs/working_groups.md index 21e3a7b..4ed0256 100644 --- a/docs/working_groups.md +++ b/docs/working_groups.md @@ -1,4 +1,23 @@ -# OSI Working Groups + + +# Apache Ossie Working Groups Join the community on Slack: [join.slack.com/t/opensemanticx](https://join.slack.com/t/apache-ossie/shared_invite/zt-42i1xkgy8-7YQtKEDq7v~mceFmdiLhkA) diff --git a/examples/flights.yaml b/examples/flights.yaml index 8c9d4a1..775709c 100644 --- a/examples/flights.yaml +++ b/examples/flights.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + version: 0.2.0.dev0 name: flights description: Ontology for flight-related concepts and relationships diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index 7dd2785..c20f288 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -1,7 +1,23 @@ # yaml-language-server: $schema=../core-spec/osi-schema.json +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # TPC-DS Semantic Model Example -# This example demonstrates the OSI Core Metadata Spec using the TPC-DS benchmark schema +# This example demonstrates the Ossie Core Metadata Spec using the TPC-DS benchmark schema # TPC-DS is a decision support benchmark with a realistic retail business model version: "0.2.0.dev0" diff --git a/ontology/ontology.json b/ontology/ontology.json index 473f69b..6488a89 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/open-semantic-interchange/OSI/core-spec/osi-schema.json", - "title": "OSI Ontology Metadata Specification", - "description": "JSON Schema for validating OSI (Open Semantic Interoperability) ontology definitions", + "$id": "https://github.com/apache/ossie/core-spec/osi-schema.json", + "title": "Apache Ossie Ontology Metadata Specification", + "description": "JSON Schema for validating Apache Ossie ontology definitions", "type": "object", "properties": { "version": { @@ -19,7 +19,7 @@ "description": "Human-readable description" }, "ai_context": { - "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/AIContext" + "$ref": "https://raw.githubusercontent.com/apache/ossie/main/core-spec/osi-schema.json#/$defs/AIContext" }, "ontology": { "type": "array", @@ -282,7 +282,7 @@ "description": "Human-readable description of this ontology map" }, "semantic_model": { - "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/SemanticModel" + "$ref": "https://raw.githubusercontent.com/apache/ossie/main/core-spec/osi-schema.json#/$defs/SemanticModel" }, "concept_mappings": { "type": "array", diff --git a/ontology/ontology.md b/ontology/ontology.md index 921e061..9e8aa7a 100644 --- a/ontology/ontology.md +++ b/ontology/ontology.md @@ -1,4 +1,23 @@ -# OSI - Ontology Specification + + +# Apache Ossie - Ontology Specification **Version:** 0.2.0.dev0 @@ -382,7 +401,7 @@ mappings that group by some concept. Each concept mapping declares how to populate a concept with objects and how to populate the relationships that group under that concept with links. These declarations are formed from patterns of expressions that -reference fields in a logical model that is declared using the OSI core semantic model spec. +reference fields in a logical model that is declared using the Ossie core semantic model spec. Concept mappings have the following schema: diff --git a/python/pyproject.toml b/python/pyproject.toml index baf099b..d209859 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,12 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" [project] -name = "osi-python" +name = "apache-ossie" version = "0.2.0.dev0" -description = "Python types for the Open Semantic Interchange (OSI) specification" +description = "Python types for the Apache Ossie semantic model specification" requires-python = ">=3.11" +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", +] dependencies = [ "pydantic>=2.0", "PyYAML>=6.0", @@ -16,7 +37,7 @@ dependencies = [ text = "Apache-2.0" [tool.hatch.build.targets.wheel] -packages = ["src/osi"] +packages = ["src/ossie"] [tool.uv] dev-dependencies = [ diff --git a/python/src/osi/__init__.py b/python/src/osi/__init__.py deleted file mode 100644 index 00970fd..0000000 --- a/python/src/osi/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -from osi.models import ( - OSIAIContext, - OSIAIContextObject, - OSICustomExtension, - OSIDataset, - OSIDialect, - OSIDialectExpression, - OSIDimension, - OSIDocument, - OSIExpression, - OSIField, - OSIMetric, - OSIRelationship, - OSISemanticModel, - OSIVendor, -) - -__all__ = [ - "OSIAIContext", - "OSIAIContextObject", - "OSICustomExtension", - "OSIDataset", - "OSIDialect", - "OSIDialectExpression", - "OSIDimension", - "OSIDocument", - "OSIExpression", - "OSIField", - "OSIMetric", - "OSIRelationship", - "OSISemanticModel", - "OSIVendor", -] diff --git a/python/src/ossie/__init__.py b/python/src/ossie/__init__.py new file mode 100644 index 0000000..cb105fa --- /dev/null +++ b/python/src/ossie/__init__.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from ossie.models import ( + OSIAIContext, + OSIAIContextObject, + OSICustomExtension, + OSIDataset, + OSIDialect, + OSIDialectExpression, + OSIDimension, + OSIDocument, + OSIExpression, + OSIField, + OSIMetric, + OSIRelationship, + OSISemanticModel, + OSIVendor, +) + +__all__ = [ + "OSIAIContext", + "OSIAIContextObject", + "OSICustomExtension", + "OSIDataset", + "OSIDialect", + "OSIDialectExpression", + "OSIDimension", + "OSIDocument", + "OSIExpression", + "OSIField", + "OSIMetric", + "OSIRelationship", + "OSISemanticModel", + "OSIVendor", +] diff --git a/python/src/osi/models.py b/python/src/ossie/models.py similarity index 81% rename from python/src/osi/models.py rename to python/src/ossie/models.py index 766b6e0..cc4aad5 100644 --- a/python/src/osi/models.py +++ b/python/src/ossie/models.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from enum import Enum from typing import Any, Optional, Union @@ -144,7 +161,7 @@ class OSISemanticModel(BaseModel): class OSIDocument(BaseModel): - """Root OSI document.""" + """Root Ossie document.""" model_config = ConfigDict(frozen=True) @@ -154,10 +171,10 @@ class OSIDocument(BaseModel): semantic_model: list[OSISemanticModel] def to_osi_yaml(self, **kwargs: Any) -> str: - """Serialize to OSI-compliant YAML (uses field aliases and excludes None values).""" + """Serialize to Ossie-compliant YAML (uses field aliases and excludes None values).""" data = self.model_dump(by_alias=True, exclude_none=True, mode="json", **kwargs) return yaml.dump(data, default_flow_style=False, sort_keys=False, allow_unicode=True) def to_osi_json(self, **kwargs: Any) -> str: - """Serialize to OSI-compliant JSON (uses field aliases and excludes None values).""" + """Serialize to Ossie-compliant JSON (uses field aliases and excludes None values).""" return self.model_dump_json(by_alias=True, exclude_none=True, **kwargs) diff --git a/validation/validate.py b/validation/validate.py index a33cf76..ad78d30 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -1,8 +1,25 @@ #!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ -OSI Semantic Model Validator +Ossie Semantic Model Validator -Validates OSI YAML files against: +Validates Ossie YAML files against: 1. JSON Schema (structure, types, enums) 2. Unique names (datasets, fields, metrics, relationships) 3. Valid relationship references @@ -33,7 +50,7 @@ except ImportError: SQLGLOT_AVAILABLE = False -# Map OSI dialects to sqlglot dialects +# Map Ossie dialects to sqlglot dialects DIALECT_MAP = { "ANSI_SQL": None, # sqlglot default "SNOWFLAKE": "snowflake", @@ -199,7 +216,7 @@ def main(): args = sys.argv[1:] yaml_path = Path(args[0]) - schema_path = Path(__file__).parent.parent / "core-spec" / "osi-schema.json" + schema_path = Path(__file__).parent.parent / "core-spec" / "ossie-schema.json" if len(args) > 1: if len(args) == 3 and args[1] == "--schema": schema_path = Path(args[2]) From 4af9fa4d1d989a2825fa70acc4dcc01bfb6df7b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 6 Jul 2026 08:58:24 +0200 Subject: [PATCH 47/89] Fix .asf.yaml --- .asf.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 354410e..b5e72a1 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -60,8 +60,6 @@ github: ghp_branch: gh-pages ghp_path: / - collaborators: - notifications: commits: commits@ossie.apache.org issues: issues@ossie.apache.org From 65e71545bca7fdff567ab52dbd33ac71e3c89d01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Tue, 7 Jul 2026 16:16:13 +0200 Subject: [PATCH 48/89] Remove branch protection for now --- .asf.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index b5e72a1..40fa7e7 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -41,16 +41,6 @@ github: review_drafts: false review_on_push: false - protected_branches: - main: - required_pull_request_reviews: - require_code_owner_reviews: false - dismiss_stale_reviews: true - require_last_push_approval: false - required_approving_review_count: 1 - - required_linear_history: true - features: wiki: false issues: true From 4e38091cb8ba6bf64bd3e013cd49436d1908ea0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Tue, 7 Jul 2026 16:17:06 +0200 Subject: [PATCH 49/89] Remove gh-pages here as it's managed by ossie-website --- .asf.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 40fa7e7..462fcf8 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -47,15 +47,9 @@ github: projects: true discussions: true - ghp_branch: gh-pages - ghp_path: / - notifications: commits: commits@ossie.apache.org issues: issues@ossie.apache.org pullrequests: issues@ossie.apache.org discussions: dev@ossie.apache.org jira_options: link label - -publish: - whoami: asf-site From 12a69efc7061509416b9690b50bd31b41d1f209f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 13 Jul 2026 17:34:28 +0200 Subject: [PATCH 50/89] Update the invite --- CONTRIBUTING.md | 1 - docs/working_groups.md | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8df5b49..9307ff3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,7 +38,6 @@ communities. If you are new to the ASF, the [Apache Incubator](https://incubator.apache.org/) and the [ASF New Committers guide](https://www.apache.org/dev/new-committers-guide.html) are good starting points. ->>>>>>> 8268b7b (Welcome Apache Ossie (incubating)) ## Ways to Contribute diff --git a/docs/working_groups.md b/docs/working_groups.md index 4ed0256..3e8a3a9 100644 --- a/docs/working_groups.md +++ b/docs/working_groups.md @@ -19,7 +19,7 @@ # Apache Ossie Working Groups -Join the community on Slack: [join.slack.com/t/opensemanticx](https://join.slack.com/t/apache-ossie/shared_invite/zt-42i1xkgy8-7YQtKEDq7v~mceFmdiLhkA) +Join the community on Slack: [join.slack.com/t/opensemanticx](https://join.slack.com/t/opensemanticx/shared_invite/zt-3yuad6c0h-MaoPgVSD1g9MEOf1_QeaiQ) ## 1. Metric Language and Relationships From eee6b2f871590a07166b1478344f82df953e1969 Mon Sep 17 00:00:00 2001 From: Ralf Becher Date: Mon, 13 Jul 2026 19:44:54 +0200 Subject: [PATCH 51/89] Rename orionbelt converter package to ossie_orionbelt for Apache naming consistency Align with the OSI->Ossie incubation rename applied to the other converters: package dir osi_orionbelt -> ossie_orionbelt, dist name apache-ossie-orionbelt, CLI ossie-orionbelt. Keeps osi in spec-format file/class names (osi_to_obml.py, OSItoOBML) as upstream did for dbt/gooddata. --- converters/orionbelt/README.md | 20 +++--- .../orionbelt/osi_obml_mapping_analysis.md | 16 ++--- .../osi_obml_ontology_mapping_analysis.md | 2 +- converters/orionbelt/pyproject.toml | 12 ++-- .../__init__.py | 4 +- .../_common.py | 0 .../{osi_orionbelt => ossie_orionbelt}/cli.py | 12 ++-- .../converter.py | 68 +++++++++--------- .../obml_to_osi.py | 6 +- .../ontology.py | 4 +- .../osi_to_obml.py | 4 +- .../schemas/obml-schema.json | 0 .../schemas/osi-ontology-schema.json | 0 .../schemas/osi-schema.json | 0 .../validation.py | 0 .../orionbelt/tests/fixtures/obml_as_osi.yaml | 2 +- .../tests/test_osi_converter_cumulative.py | 2 +- .../tests/test_osi_converter_filters.py | 2 +- .../test_osi_converter_measure_overrides.py | 2 +- .../tests/test_osi_converter_ontology.py | 2 +- .../orionbelt/tests/test_osi_converter_pop.py | 2 +- .../tests/test_osi_converter_properties.py | 2 +- .../tests/test_osi_converter_trend_v26.py | 2 +- .../tests/test_osi_converter_vendors.py | 2 +- .../tests/test_osi_metric_no_silent_loss.py | 2 +- .../tests/test_osi_tpcds_baseline.py | 2 +- .../orionbelt/tests/test_osi_v02_compat.py | 4 +- converters/orionbelt/uv.lock | 72 +++++++++---------- 28 files changed, 123 insertions(+), 123 deletions(-) rename converters/orionbelt/src/{osi_orionbelt => ossie_orionbelt}/__init__.py (91%) rename converters/orionbelt/src/{osi_orionbelt => ossie_orionbelt}/_common.py (100%) rename converters/orionbelt/src/{osi_orionbelt => ossie_orionbelt}/cli.py (92%) rename converters/orionbelt/src/{osi_orionbelt => ossie_orionbelt}/converter.py (79%) rename converters/orionbelt/src/{osi_orionbelt => ossie_orionbelt}/obml_to_osi.py (99%) rename converters/orionbelt/src/{osi_orionbelt => ossie_orionbelt}/ontology.py (98%) rename converters/orionbelt/src/{osi_orionbelt => ossie_orionbelt}/osi_to_obml.py (99%) rename converters/orionbelt/src/{osi_orionbelt => ossie_orionbelt}/schemas/obml-schema.json (100%) rename converters/orionbelt/src/{osi_orionbelt => ossie_orionbelt}/schemas/osi-ontology-schema.json (100%) rename converters/orionbelt/src/{osi_orionbelt => ossie_orionbelt}/schemas/osi-schema.json (100%) rename converters/orionbelt/src/{osi_orionbelt => ossie_orionbelt}/validation.py (100%) diff --git a/converters/orionbelt/README.md b/converters/orionbelt/README.md index 94b3ae0..bcc96bb 100644 --- a/converters/orionbelt/README.md +++ b/converters/orionbelt/README.md @@ -1,4 +1,4 @@ -# osi-orionbelt +# apache-ossie-orionbelt Bidirectional converter between **OBML** (OrionBelt Markup Language) semantic models and **OSI** ([Open Semantic Interchange](https://open-semantic-interchange.org/)), @@ -20,14 +20,14 @@ file issues and contributions upstream. ## Install ```bash -pip install osi-orionbelt +pip install apache-ossie-orionbelt ``` Optional deep OBML semantic validation (cycles, duplicate names, invalid refs) via the full OrionBelt engine: ```bash -pip install "osi-orionbelt[obml-validation]" +pip install "apache-ossie-orionbelt[obml-validation]" ``` Without that extra, OBML validation runs JSON-schema checks only and emits a @@ -35,7 +35,7 @@ warning for the deeper semantic pass. ## CLI -A single `osi-orionbelt` command with two subcommands (mirroring `osi-dbt`): +A single `ossie-orionbelt` command with two subcommands (mirroring `ossie-dbt`): | Subcommand | Direction | In | Out | |---------|-----------|----|----| @@ -44,22 +44,22 @@ A single `osi-orionbelt` command with two subcommands (mirroring `osi-dbt`): | `osi-to-obml` | OSI core-spec -> OBML | OSI YAML | OBML YAML | ```bash -osi-orionbelt obml-to-osi -i model.obml.yaml -o model.osi.yaml -osi-orionbelt obml-to-osi --ontology -i model.obml.yaml -o model.ontology.yaml -osi-orionbelt osi-to-obml -i model.osi.yaml -o model.obml.yaml +ossie-orionbelt obml-to-osi -i model.obml.yaml -o model.osi.yaml +ossie-orionbelt obml-to-osi --ontology -i model.obml.yaml -o model.ontology.yaml +ossie-orionbelt osi-to-obml -i model.osi.yaml -o model.obml.yaml ``` `-i/--input` and `-o/--output` are required. Each subcommand prints conversion warnings and a validation summary to stderr, and exits non-zero when the produced document fails schema validation (unless `--no-validate`). Run -`osi-orionbelt --help` or `osi-orionbelt obml-to-osi --help` for the full +`ossie-orionbelt --help` or `ossie-orionbelt obml-to-osi --help` for the full option list. ## Python API ```python import yaml -from osi_orionbelt import OBMLtoOSI, OSItoOBML, validate_osi +from ossie_orionbelt import OBMLtoOSI, OSItoOBML, validate_osi obml = yaml.safe_load(open("model.obml.yaml")) osi = OBMLtoOSI(obml, "sales", "Sales model").convert() @@ -121,5 +121,5 @@ for the ontology-layer mapping and its documented gaps. ```bash uv sync # install uv run pytest # run the test suite (includes a TPC-DS baseline) -uv run ruff check && uv run mypy src/osi_orionbelt +uv run ruff check && uv run mypy src/ossie_orionbelt ``` diff --git a/converters/orionbelt/osi_obml_mapping_analysis.md b/converters/orionbelt/osi_obml_mapping_analysis.md index 9c3f6ee..dc62018 100644 --- a/converters/orionbelt/osi_obml_mapping_analysis.md +++ b/converters/orionbelt/osi_obml_mapping_analysis.md @@ -177,22 +177,22 @@ Validation runs automatically after each conversion. Use `--no-validate` to skip ### CLI -A single `osi-orionbelt` command with two subcommands is installed with the package: +A single `ossie-orionbelt` command with two subcommands is installed with the package: ```bash # OSI → OBML -osi-orionbelt osi-to-obml -i tpcds_osi.yaml -o tpcds_as_obml.yaml +ossie-orionbelt osi-to-obml -i tpcds_osi.yaml -o tpcds_as_obml.yaml # OBML → OSI -osi-orionbelt obml-to-osi -i tpcds_as_obml.yaml -o tpcds_obml_as_osi.yaml \ +ossie-orionbelt obml-to-osi -i tpcds_as_obml.yaml -o tpcds_obml_as_osi.yaml \ --model-name tpcds_retail_model \ --description "TPC-DS retail semantic model" # OBML → OSI ontology document -osi-orionbelt obml-to-osi --ontology -i tpcds_as_obml.yaml -o tpcds_ontology.yaml +ossie-orionbelt obml-to-osi --ontology -i tpcds_as_obml.yaml -o tpcds_ontology.yaml # Skip validation -osi-orionbelt osi-to-obml -i input.yaml -o output.yaml --no-validate +ossie-orionbelt osi-to-obml -i input.yaml -o output.yaml --no-validate ``` ### CLI Options @@ -214,7 +214,7 @@ osi-orionbelt osi-to-obml -i input.yaml -o output.yaml --no-validate ### Python API ```python -from osi_orionbelt import OSItoOBML, OBMLtoOSI, validate_obml, validate_osi +from ossie_orionbelt import OSItoOBML, OBMLtoOSI, validate_obml, validate_osi # OSI → OBML converter = OSItoOBML(osi_dict) @@ -258,8 +258,8 @@ Converting the OBML output back to OSI produces a valid OSI model where: |---|---| | `tests/fixtures/tpcds_osi.yaml` | Official TPC-DS OSI example (from OSI repo) | | `tests/fixtures/tpcds_as_obml.yaml` | Converted OBML output | -| `src/osi_orionbelt/schemas/osi-schema.json` | OSI JSON Schema (Draft 2020-12, from OSI repo) | -| `src/osi_orionbelt/converter.py` | Bidirectional converter with validation | +| `src/ossie_orionbelt/schemas/osi-schema.json` | OSI JSON Schema (Draft 2020-12, from OSI repo) | +| `src/ossie_orionbelt/converter.py` | Bidirectional converter with validation | ## 7. Future Considerations diff --git a/converters/orionbelt/osi_obml_ontology_mapping_analysis.md b/converters/orionbelt/osi_obml_ontology_mapping_analysis.md index 1887b85..6a7bc95 100644 --- a/converters/orionbelt/osi_obml_ontology_mapping_analysis.md +++ b/converters/orionbelt/osi_obml_ontology_mapping_analysis.md @@ -1,7 +1,7 @@ # OBML → OSI Ontology Mapping Analysis This pins the rules used by `OBMLtoOSIOntology` to derive an **OSI ontology -document** (validated against `src/osi_orionbelt/schemas/osi-ontology-schema.json`, OSI version +document** (validated against `src/ossie_orionbelt/schemas/osi-ontology-schema.json`, OSI version `0.2.0.dev0`) from an OBML semantic model. The OSI ontology is a **separate document** from the OSI core-spec semantic diff --git a/converters/orionbelt/pyproject.toml b/converters/orionbelt/pyproject.toml index 7aa309c..f580cdb 100644 --- a/converters/orionbelt/pyproject.toml +++ b/converters/orionbelt/pyproject.toml @@ -3,7 +3,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [project] -name = "osi-orionbelt" +name = "apache-ossie-orionbelt" version = "0.1.0" description = "Bidirectional OBML <-> OSI (Open Semantic Interchange) converter for OrionBelt semantic models" license = { text = "Apache-2.0" } @@ -31,16 +31,16 @@ dev = [ ] [project.scripts] -osi-orionbelt = "osi_orionbelt.cli:main" +ossie-orionbelt = "ossie_orionbelt.cli:main" [tool.hatch.build.targets.wheel] -packages = ["src/osi_orionbelt"] +packages = ["src/ossie_orionbelt"] # All three schemas (the two OSI artefacts plus a vendored snapshot of the -# canonical OBML schema) live in src/osi_orionbelt/schemas/ as tracked files, +# canonical OBML schema) live in src/ossie_orionbelt/schemas/ as tracked files, # so the sdist and wheel are self-contained and build in isolation. The OBML # snapshot is kept in sync with the repo-root schema/obml-schema.json by a -# drift-guard test (tests/unit/test_osi_orionbelt_schema_sync.py). +# drift-guard test (tests/unit/test_ossie_orionbelt_schema_sync.py). [tool.ruff] line-length = 100 @@ -51,7 +51,7 @@ select = ["E", "F", "I", "N", "UP", "B", "A", "SIM"] [tool.mypy] python_version = "3.12" -files = ["src/osi_orionbelt"] +files = ["src/ossie_orionbelt"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/converters/orionbelt/src/osi_orionbelt/__init__.py b/converters/orionbelt/src/ossie_orionbelt/__init__.py similarity index 91% rename from converters/orionbelt/src/osi_orionbelt/__init__.py rename to converters/orionbelt/src/ossie_orionbelt/__init__.py index 9bccf42..5dc24fa 100644 --- a/converters/orionbelt/src/osi_orionbelt/__init__.py +++ b/converters/orionbelt/src/ossie_orionbelt/__init__.py @@ -1,4 +1,4 @@ -"""osi-orionbelt: bidirectional OBML <-> OSI converter. +"""ossie-orionbelt: bidirectional OBML <-> OSI converter. Converts between OrionBelt Markup Language (OBML) semantic models and Open Semantic Interchange (OSI) models, in both directions, plus an OSI ontology @@ -17,7 +17,7 @@ from __future__ import annotations -from osi_orionbelt.converter import ( +from ossie_orionbelt.converter import ( OBMLtoOSI, OBMLtoOSIOntology, OSItoOBML, diff --git a/converters/orionbelt/src/osi_orionbelt/_common.py b/converters/orionbelt/src/ossie_orionbelt/_common.py similarity index 100% rename from converters/orionbelt/src/osi_orionbelt/_common.py rename to converters/orionbelt/src/ossie_orionbelt/_common.py diff --git a/converters/orionbelt/src/osi_orionbelt/cli.py b/converters/orionbelt/src/ossie_orionbelt/cli.py similarity index 92% rename from converters/orionbelt/src/osi_orionbelt/cli.py rename to converters/orionbelt/src/ossie_orionbelt/cli.py index 351d064..196722f 100644 --- a/converters/orionbelt/src/osi_orionbelt/cli.py +++ b/converters/orionbelt/src/ossie_orionbelt/cli.py @@ -1,11 +1,11 @@ """Command-line entry point for the OBML <-> OSI converter. -A single ``osi-orionbelt`` command with two format-named subcommands, mirroring +A single ``ossie-orionbelt`` command with two format-named subcommands, mirroring the OSI converter convention (e.g. ``osi-dbt msi-to-osi``): - osi-orionbelt obml-to-osi -i model.obml.yaml -o model.osi.yaml - osi-orionbelt obml-to-osi --ontology -i model.obml.yaml -o model.ontology.yaml - osi-orionbelt osi-to-obml -i model.osi.yaml -o model.obml.yaml + ossie-orionbelt obml-to-osi -i model.obml.yaml -o model.osi.yaml + ossie-orionbelt obml-to-osi --ontology -i model.obml.yaml -o model.ontology.yaml + ossie-orionbelt osi-to-obml -i model.osi.yaml -o model.obml.yaml Both subcommands print conversion warnings and a validation summary to stderr, and exit non-zero when the produced document fails schema validation (unless @@ -21,7 +21,7 @@ import yaml -from osi_orionbelt.converter import ( +from ossie_orionbelt.converter import ( OBMLtoOSI, OBMLtoOSIOntology, OSItoOBML, @@ -108,7 +108,7 @@ def _cmd_osi_to_obml(args: argparse.Namespace) -> int: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( - prog="osi-orionbelt", + prog="ossie-orionbelt", description="Convert between OrionBelt OBML and OSI YAML.", ) subparsers = parser.add_subparsers(dest="command", required=True) diff --git a/converters/orionbelt/src/osi_orionbelt/converter.py b/converters/orionbelt/src/ossie_orionbelt/converter.py similarity index 79% rename from converters/orionbelt/src/osi_orionbelt/converter.py rename to converters/orionbelt/src/ossie_orionbelt/converter.py index 46a4aa0..3acd2f7 100644 --- a/converters/orionbelt/src/osi_orionbelt/converter.py +++ b/converters/orionbelt/src/ossie_orionbelt/converter.py @@ -14,13 +14,13 @@ This module is a thin **facade**. The converter implementation is split across sibling modules to keep each file focused: -* :mod:`osi_orionbelt._common` — shared constants and mapping tables -* :mod:`osi_orionbelt.osi_to_obml` — :class:`OSItoOBML` -* :mod:`osi_orionbelt.obml_to_osi` — :class:`OBMLtoOSI` -* :mod:`osi_orionbelt.ontology` — :class:`OBMLtoOSIOntology` -* :mod:`osi_orionbelt.validation` — :class:`ValidationResult` + ``validate_*`` +* :mod:`ossie_orionbelt._common` — shared constants and mapping tables +* :mod:`ossie_orionbelt.osi_to_obml` — :class:`OSItoOBML` +* :mod:`ossie_orionbelt.obml_to_osi` — :class:`OBMLtoOSI` +* :mod:`ossie_orionbelt.ontology` — :class:`OBMLtoOSIOntology` +* :mod:`ossie_orionbelt.validation` — :class:`ValidationResult` + ``validate_*`` -Every public name is re-exported here so ``osi_orionbelt.converter.`` +Every public name is re-exported here so ``ossie_orionbelt.converter.`` continues to work unchanged. """ @@ -32,83 +32,83 @@ import yaml -from osi_orionbelt._common import ( +from ossie_orionbelt._common import ( _COLUMN_REF_RE as _COLUMN_REF_RE, ) -from osi_orionbelt._common import ( +from ossie_orionbelt._common import ( _INTERNAL_VENDORS as _INTERNAL_VENDORS, ) -from osi_orionbelt._common import ( +from ossie_orionbelt._common import ( _OBML_VENDOR_READ as _OBML_VENDOR_READ, ) -from osi_orionbelt._common import ( +from ossie_orionbelt._common import ( _OSI_KNOWN_DIALECTS as _OSI_KNOWN_DIALECTS, ) -from osi_orionbelt._common import ( +from ossie_orionbelt._common import ( _OSI_KNOWN_VENDORS as _OSI_KNOWN_VENDORS, ) -from osi_orionbelt._common import ( +from ossie_orionbelt._common import ( _OSI_VENDOR_READ as _OSI_VENDOR_READ, ) -from osi_orionbelt._common import ( +from ossie_orionbelt._common import ( _OSI_VERSION as _OSI_VERSION, ) -from osi_orionbelt._common import ( +from ossie_orionbelt._common import ( _SQL_PARSEABLE_DIALECTS as _SQL_PARSEABLE_DIALECTS, ) -from osi_orionbelt._common import ( +from ossie_orionbelt._common import ( _VENDOR_OBML as _VENDOR_OBML, ) -from osi_orionbelt._common import ( +from ossie_orionbelt._common import ( _VENDOR_OSI as _VENDOR_OSI, ) # Shared constants / mapping tables (re-exported for backwards compatibility). # The ``X as X`` aliases mark these as intentional re-exports so historic -# ``from osi_orionbelt.converter import `` imports keep working. -from osi_orionbelt._common import ( +# ``from ossie_orionbelt.converter import `` imports keep working. +from ossie_orionbelt._common import ( OBML_TO_OSI_TYPE as OBML_TO_OSI_TYPE, ) -from osi_orionbelt._common import ( +from ossie_orionbelt._common import ( OSI_TO_OBML_TYPE as OSI_TO_OBML_TYPE, ) -from osi_orionbelt.obml_to_osi import OBMLtoOSI as OBMLtoOSI -from osi_orionbelt.ontology import OBMLtoOSIOntology as OBMLtoOSIOntology -from osi_orionbelt.osi_to_obml import OSItoOBML as OSItoOBML -from osi_orionbelt.validation import ( +from ossie_orionbelt.obml_to_osi import OBMLtoOSI as OBMLtoOSI +from ossie_orionbelt.ontology import OBMLtoOSIOntology as OBMLtoOSIOntology +from ossie_orionbelt.osi_to_obml import OSItoOBML as OSItoOBML +from ossie_orionbelt.validation import ( _OBML_SCHEMA_PATH as _OBML_SCHEMA_PATH, ) -from osi_orionbelt.validation import ( +from ossie_orionbelt.validation import ( _OSI_CORE_SPEC_RAW_URL as _OSI_CORE_SPEC_RAW_URL, ) -from osi_orionbelt.validation import ( +from ossie_orionbelt.validation import ( _OSI_ONTOLOGY_SCHEMA_PATH as _OSI_ONTOLOGY_SCHEMA_PATH, ) -from osi_orionbelt.validation import ( +from ossie_orionbelt.validation import ( _OSI_SCHEMA_PATH as _OSI_SCHEMA_PATH, ) -from osi_orionbelt.validation import ( +from ossie_orionbelt.validation import ( _SCHEMAS_DIR as _SCHEMAS_DIR, ) -from osi_orionbelt.validation import ( +from ossie_orionbelt.validation import ( _SCRIPT_DIR as _SCRIPT_DIR, ) -from osi_orionbelt.validation import ( +from ossie_orionbelt.validation import ( ValidationResult as ValidationResult, ) -from osi_orionbelt.validation import ( +from ossie_orionbelt.validation import ( _osi_core_registry as _osi_core_registry, ) -from osi_orionbelt.validation import ( +from ossie_orionbelt.validation import ( _validate_json_schema as _validate_json_schema, ) -from osi_orionbelt.validation import ( +from ossie_orionbelt.validation import ( validate_obml as validate_obml, ) -from osi_orionbelt.validation import ( +from ossie_orionbelt.validation import ( validate_osi as validate_osi, ) -from osi_orionbelt.validation import ( +from ossie_orionbelt.validation import ( validate_osi_ontology as validate_osi_ontology, ) diff --git a/converters/orionbelt/src/osi_orionbelt/obml_to_osi.py b/converters/orionbelt/src/ossie_orionbelt/obml_to_osi.py similarity index 99% rename from converters/orionbelt/src/osi_orionbelt/obml_to_osi.py rename to converters/orionbelt/src/ossie_orionbelt/obml_to_osi.py index 4b0df6c..9f3daca 100644 --- a/converters/orionbelt/src/osi_orionbelt/obml_to_osi.py +++ b/converters/orionbelt/src/ossie_orionbelt/obml_to_osi.py @@ -1,7 +1,7 @@ """OBML → OSI conversion (the :class:`OBMLtoOSI` direction). Extracted verbatim from ``converter.py``; see that module for the package-level -docstring and the shared constants in :mod:`osi_orionbelt._common`. +docstring and the shared constants in :mod:`ossie_orionbelt._common`. """ from __future__ import annotations @@ -10,7 +10,7 @@ import re from typing import Any -from osi_orionbelt._common import ( +from ossie_orionbelt._common import ( _INTERNAL_VENDORS, _OSI_VENDOR_READ, _OSI_VERSION, @@ -87,7 +87,7 @@ def convert(self) -> dict: roundtrip_data: dict[str, Any] = { "source_format": "OBML", "source_version": str(self.obml.get("version", "1.0")), - "converter": "osi-orionbelt", + "converter": "ossie-orionbelt", } # Preserve model-level static filters for roundtrip obml_filters = self.obml.get("filters", []) diff --git a/converters/orionbelt/src/osi_orionbelt/ontology.py b/converters/orionbelt/src/ossie_orionbelt/ontology.py similarity index 98% rename from converters/orionbelt/src/osi_orionbelt/ontology.py rename to converters/orionbelt/src/ossie_orionbelt/ontology.py index b846e66..86886c2 100644 --- a/converters/orionbelt/src/osi_orionbelt/ontology.py +++ b/converters/orionbelt/src/ossie_orionbelt/ontology.py @@ -8,8 +8,8 @@ from typing import Any -from osi_orionbelt._common import _OSI_VERSION -from osi_orionbelt.obml_to_osi import OBMLtoOSI +from ossie_orionbelt._common import _OSI_VERSION +from ossie_orionbelt.obml_to_osi import OBMLtoOSI class OBMLtoOSIOntology: diff --git a/converters/orionbelt/src/osi_orionbelt/osi_to_obml.py b/converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py similarity index 99% rename from converters/orionbelt/src/osi_orionbelt/osi_to_obml.py rename to converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py index c55743f..cc3e4fe 100644 --- a/converters/orionbelt/src/osi_orionbelt/osi_to_obml.py +++ b/converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py @@ -1,7 +1,7 @@ """OSI → OBML conversion (the :class:`OSItoOBML` direction). Extracted verbatim from ``converter.py``; see that module for the package-level -docstring and the shared constants in :mod:`osi_orionbelt._common`. +docstring and the shared constants in :mod:`ossie_orionbelt._common`. """ from __future__ import annotations @@ -10,7 +10,7 @@ import re from typing import Any -from osi_orionbelt._common import ( +from ossie_orionbelt._common import ( _COLUMN_REF_RE, _INTERNAL_VENDORS, _OBML_VENDOR_READ, diff --git a/converters/orionbelt/src/osi_orionbelt/schemas/obml-schema.json b/converters/orionbelt/src/ossie_orionbelt/schemas/obml-schema.json similarity index 100% rename from converters/orionbelt/src/osi_orionbelt/schemas/obml-schema.json rename to converters/orionbelt/src/ossie_orionbelt/schemas/obml-schema.json diff --git a/converters/orionbelt/src/osi_orionbelt/schemas/osi-ontology-schema.json b/converters/orionbelt/src/ossie_orionbelt/schemas/osi-ontology-schema.json similarity index 100% rename from converters/orionbelt/src/osi_orionbelt/schemas/osi-ontology-schema.json rename to converters/orionbelt/src/ossie_orionbelt/schemas/osi-ontology-schema.json diff --git a/converters/orionbelt/src/osi_orionbelt/schemas/osi-schema.json b/converters/orionbelt/src/ossie_orionbelt/schemas/osi-schema.json similarity index 100% rename from converters/orionbelt/src/osi_orionbelt/schemas/osi-schema.json rename to converters/orionbelt/src/ossie_orionbelt/schemas/osi-schema.json diff --git a/converters/orionbelt/src/osi_orionbelt/validation.py b/converters/orionbelt/src/ossie_orionbelt/validation.py similarity index 100% rename from converters/orionbelt/src/osi_orionbelt/validation.py rename to converters/orionbelt/src/ossie_orionbelt/validation.py diff --git a/converters/orionbelt/tests/fixtures/obml_as_osi.yaml b/converters/orionbelt/tests/fixtures/obml_as_osi.yaml index 6698228..7d8fe87 100644 --- a/converters/orionbelt/tests/fixtures/obml_as_osi.yaml +++ b/converters/orionbelt/tests/fixtures/obml_as_osi.yaml @@ -784,4 +784,4 @@ semantic_model: description: Average Sale custom_extensions: - vendor_name: COMMON - data: '{"source_format": "OBML", "source_version": "1.0", "converter": "osi-orionbelt"}' + data: '{"source_format": "OBML", "source_version": "1.0", "converter": "ossie-orionbelt"}' diff --git a/converters/orionbelt/tests/test_osi_converter_cumulative.py b/converters/orionbelt/tests/test_osi_converter_cumulative.py index 6196375..c19fa80 100644 --- a/converters/orionbelt/tests/test_osi_converter_cumulative.py +++ b/converters/orionbelt/tests/test_osi_converter_cumulative.py @@ -11,7 +11,7 @@ import pytest -import osi_orionbelt.converter as conv +import ossie_orionbelt.converter as conv # --------------------------------------------------------------------------- # Test OBML model with cumulative metrics diff --git a/converters/orionbelt/tests/test_osi_converter_filters.py b/converters/orionbelt/tests/test_osi_converter_filters.py index 2ed63e8..91be10a 100644 --- a/converters/orionbelt/tests/test_osi_converter_filters.py +++ b/converters/orionbelt/tests/test_osi_converter_filters.py @@ -9,7 +9,7 @@ import json from typing import Any -import osi_orionbelt.converter as conv +import ossie_orionbelt.converter as conv _OBML_WITH_FILTERS: dict[str, Any] = { "version": 1.0, diff --git a/converters/orionbelt/tests/test_osi_converter_measure_overrides.py b/converters/orionbelt/tests/test_osi_converter_measure_overrides.py index fdd4510..c610dc9 100644 --- a/converters/orionbelt/tests/test_osi_converter_measure_overrides.py +++ b/converters/orionbelt/tests/test_osi_converter_measure_overrides.py @@ -8,7 +8,7 @@ from typing import Any -import osi_orionbelt.converter as conv +import ossie_orionbelt.converter as conv _BASE_OBML: dict[str, Any] = { "version": 1.0, diff --git a/converters/orionbelt/tests/test_osi_converter_ontology.py b/converters/orionbelt/tests/test_osi_converter_ontology.py index 3a04ac7..8c2a6d7 100644 --- a/converters/orionbelt/tests/test_osi_converter_ontology.py +++ b/converters/orionbelt/tests/test_osi_converter_ontology.py @@ -10,7 +10,7 @@ from typing import Any -import osi_orionbelt.converter as conv +import ossie_orionbelt.converter as conv _OBML: dict[str, Any] = { "version": 1.0, diff --git a/converters/orionbelt/tests/test_osi_converter_pop.py b/converters/orionbelt/tests/test_osi_converter_pop.py index 79e4266..c1fdcf3 100644 --- a/converters/orionbelt/tests/test_osi_converter_pop.py +++ b/converters/orionbelt/tests/test_osi_converter_pop.py @@ -11,7 +11,7 @@ import pytest -import osi_orionbelt.converter as conv +import ossie_orionbelt.converter as conv # --------------------------------------------------------------------------- # Test OBML model with period-over-period metrics diff --git a/converters/orionbelt/tests/test_osi_converter_properties.py b/converters/orionbelt/tests/test_osi_converter_properties.py index 2d78a6b..0e33d0b 100644 --- a/converters/orionbelt/tests/test_osi_converter_properties.py +++ b/converters/orionbelt/tests/test_osi_converter_properties.py @@ -8,7 +8,7 @@ from typing import Any -import osi_orionbelt.converter as conv +import ossie_orionbelt.converter as conv _OBML_FULL: dict[str, Any] = { "version": 1.0, diff --git a/converters/orionbelt/tests/test_osi_converter_trend_v26.py b/converters/orionbelt/tests/test_osi_converter_trend_v26.py index e2320c1..0d950e0 100644 --- a/converters/orionbelt/tests/test_osi_converter_trend_v26.py +++ b/converters/orionbelt/tests/test_osi_converter_trend_v26.py @@ -13,7 +13,7 @@ import json from typing import Any -import osi_orionbelt.converter as conv +import ossie_orionbelt.converter as conv _OBML_V26: dict[str, Any] = { "version": 1.0, diff --git a/converters/orionbelt/tests/test_osi_converter_vendors.py b/converters/orionbelt/tests/test_osi_converter_vendors.py index 4d947ef..05eae9b 100644 --- a/converters/orionbelt/tests/test_osi_converter_vendors.py +++ b/converters/orionbelt/tests/test_osi_converter_vendors.py @@ -12,7 +12,7 @@ import json from typing import Any -import osi_orionbelt.converter as conv +import ossie_orionbelt.converter as conv def _osi_field(name: str, **extra: Any) -> dict[str, Any]: diff --git a/converters/orionbelt/tests/test_osi_metric_no_silent_loss.py b/converters/orionbelt/tests/test_osi_metric_no_silent_loss.py index 094d8b5..f843d75 100644 --- a/converters/orionbelt/tests/test_osi_metric_no_silent_loss.py +++ b/converters/orionbelt/tests/test_osi_metric_no_silent_loss.py @@ -20,7 +20,7 @@ import pytest -import osi_orionbelt.converter as conv +import ossie_orionbelt.converter as conv def _osi_model(metrics: list[dict[str, Any]]) -> dict[str, Any]: diff --git a/converters/orionbelt/tests/test_osi_tpcds_baseline.py b/converters/orionbelt/tests/test_osi_tpcds_baseline.py index 1a56c7e..3165b08 100644 --- a/converters/orionbelt/tests/test_osi_tpcds_baseline.py +++ b/converters/orionbelt/tests/test_osi_tpcds_baseline.py @@ -18,7 +18,7 @@ import pytest -import osi_orionbelt.converter as conv +import ossie_orionbelt.converter as conv _FIXTURE = Path(__file__).resolve().parent / "fixtures" / "tpcds_semantic_model.yaml" diff --git a/converters/orionbelt/tests/test_osi_v02_compat.py b/converters/orionbelt/tests/test_osi_v02_compat.py index bdc5c27..a6c7817 100644 --- a/converters/orionbelt/tests/test_osi_v02_compat.py +++ b/converters/orionbelt/tests/test_osi_v02_compat.py @@ -19,14 +19,14 @@ import pytest -import osi_orionbelt.converter as conv +import ossie_orionbelt.converter as conv # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- _SCHEMA_PATH = ( - Path(__file__).resolve().parents[1] / "src" / "osi_orionbelt" / "schemas" / "osi-schema.json" + Path(__file__).resolve().parents[1] / "src" / "ossie_orionbelt" / "schemas" / "osi-schema.json" ) diff --git a/converters/orionbelt/uv.lock b/converters/orionbelt/uv.lock index b81fe50..3b9a719 100644 --- a/converters/orionbelt/uv.lock +++ b/converters/orionbelt/uv.lock @@ -37,6 +37,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, ] +[[package]] +name = "apache-ossie-orionbelt" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "jsonschema" }, + { name = "pyyaml" }, + { name = "referencing" }, +] + +[package.optional-dependencies] +dev = [ + { name = "mypy" }, + { name = "pytest" }, + { name = "ruff" }, + { name = "types-jsonschema" }, + { name = "types-pyyaml" }, +] +obml-validation = [ + { name = "orionbelt-semantic-layer" }, +] + +[package.metadata] +requires-dist = [ + { name = "jsonschema", specifier = ">=4.18" }, + { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10" }, + { name = "orionbelt-semantic-layer", marker = "extra == 'obml-validation'" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "referencing", specifier = ">=0.30" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.4" }, + { name = "types-jsonschema", marker = "extra == 'dev'" }, + { name = "types-pyyaml", marker = "extra == 'dev'" }, +] +provides-extras = ["obml-validation", "dev"] + [[package]] name = "ast-serialize" version = "0.5.0" @@ -410,42 +446,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/82/d9/5c0f42a6590dc4e597d1d582805a73f0139d8d7409cfbe8aa5cf333982bc/orionbelt_semantic_layer-2.10.0-py3-none-any.whl", hash = "sha256:86b05d8ac608a5ef8ce7ade09b11e540e6eeba74df3605e2f08c0623b9e12d02", size = 566010, upload-time = "2026-06-12T16:49:14.486Z" }, ] -[[package]] -name = "osi-orionbelt" -version = "0.1.0" -source = { editable = "." } -dependencies = [ - { name = "jsonschema" }, - { name = "pyyaml" }, - { name = "referencing" }, -] - -[package.optional-dependencies] -dev = [ - { name = "mypy" }, - { name = "pytest" }, - { name = "ruff" }, - { name = "types-jsonschema" }, - { name = "types-pyyaml" }, -] -obml-validation = [ - { name = "orionbelt-semantic-layer" }, -] - -[package.metadata] -requires-dist = [ - { name = "jsonschema", specifier = ">=4.18" }, - { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10" }, - { name = "orionbelt-semantic-layer", marker = "extra == 'obml-validation'" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, - { name = "pyyaml", specifier = ">=6.0" }, - { name = "referencing", specifier = ">=0.30" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.4" }, - { name = "types-jsonschema", marker = "extra == 'dev'" }, - { name = "types-pyyaml", marker = "extra == 'dev'" }, -] -provides-extras = ["obml-validation", "dev"] - [[package]] name = "packaging" version = "26.2" From 95f824b5f0e6f51b7baa05b5f67e49041e0d381d Mon Sep 17 00:00:00 2001 From: Emil Sadek Date: Mon, 13 Jul 2026 13:59:09 -0700 Subject: [PATCH 52/89] docs: fix typos in specification and ontology documentation --- core-spec/spec.md | 2 +- ontology/ontology.md | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core-spec/spec.md b/core-spec/spec.md index b544042..e3b8410 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -71,7 +71,7 @@ The top-level container that represents a complete semantic model, including dat | `ai_context` | string/object | No | Additional context for AI tools (e.g., custom instructions) | | `datasets` | array | Yes | Collection of logical datasets (fact and dimension tables) | | `relationships` | array | No | Defines how logical datasets are connected | -| `metrics` | array | No | Quantifiable measures defined as aggregate expessions on fields from logical datsets | +| `metrics` | array | No | Quantifiable measures defined as aggregate expressions on fields from logical datasets | | `custom_extensions` | array | No | Vendor-specific attributes for extensibility | ### Example diff --git a/ontology/ontology.md b/ontology/ontology.md index 9e8aa7a..05ad614 100644 --- a/ontology/ontology.md +++ b/ontology/ontology.md @@ -48,7 +48,7 @@ e-mail address. In some modeling languages these are called either entities or o A value type is a concept that represents instances of some data type (i.e SQL types like Integer or String) with additional semantics. For instance, a social-security number is a string or positive -integer that comprises exactly nine digits. In some modeling langauges these are called data types +integer that comprises exactly nine digits. In some modeling languages these are called data types or domains. ### Built-in concepts @@ -220,7 +220,7 @@ ontology: - concept: Vehicle - concept: Date multiplicity: ManyToOne - verbalizes: [ "{Person} puchased {Vehicle} on {Date}" ] + verbalizes: [ "{Person} purchased {Vehicle} on {Date}" ] ``` the unary relationship `Person.files_married_joint` has an empty roles list, while the @@ -228,7 +228,7 @@ ternary relationship `Person.purchased_on` declares two additional roles played `Vehicle` and `Date` respectively, The role player often suffices to distinguish the role within its relationship, but when -the same concept plays more than one role, the user must declare a distinguising name for +the same concept plays more than one role, the user must declare a distinguishing name for any additional role whose player's name does not distinguish it from other roles in the same relationship. For instance, in: @@ -253,13 +253,13 @@ this relationship. Expressions that are used to define derived_by rules and requires constraints will refer to roles by name -- the name defaulting to the concept that plays the role unless an explicit role name is provided. In any expression that involving links of the `Store.ships_to_in_days` -relationship can then use the variables `Store` and `destination` to refer to objecs that +relationship can then use the variables `Store` and `destination` to refer to objects that play these two `Store`-playing roles without ambiguity. #### Multiplicities If a relationship comprises more than one role, objects that play the last role could be functionally -dermined by a tuple of objects that play the other roles. This knowledge is declared using a `ManyToOne` +determined by a tuple of objects that play the other roles. This knowledge is declared using a `ManyToOne` multiplicity constraint. In the examples above, the constraint declares that each person earns at most one salary and that for each pair of stores, the former ships to the latter in at most one number of days. For relationships of ternary and higher arity, the multiplicity applies to the n-th role, meaning @@ -422,7 +422,7 @@ An object mapping has the following schema: |---------------|---------|-----|-------| | `concept` | string | No | Names the concept being mapped to using this object map | | `expression` | string | if no `referent_mappings` | SQL expression that computes a value from fields | -| `referent_mappings` | list | if no `expression` | Referent mappings that find entity objects using identifying realtionships | +| `referent_mappings` | list | if no `expression` | Referent mappings that find entity objects using identifying relationships | When the concept is a value type or an entity type with a simple identifier, then an object mapping is just a SQL expression. For instance, given this ontology snippet: @@ -473,7 +473,7 @@ Referent mappings have the following schema: |----------------|--------|-----|-------| | `relationship` | string | Yes | Name of an identifying relationship | | `expression` | string | if no `referent_mappings` | SQL expression that computes a value from fields | -| `referent_mappings` | list | if no `expression` | Referent mappings that find entity objects using identifying realtionships | +| `referent_mappings` | list | if no `expression` | Referent mappings that find entity objects using identifying relationships | For instance, consider this ontology snippet: From ca03d3f3b8e1d4104f7d07a66a765d8d2d6f472d Mon Sep 17 00:00:00 2001 From: Emil Sadek Date: Mon, 13 Jul 2026 15:40:46 -0700 Subject: [PATCH 53/89] Add inline script metadata to validate.py --- validation/validate.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/validation/validate.py b/validation/validate.py index ad78d30..6dfcde3 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -1,4 +1,14 @@ #!/usr/bin/env python3 +# +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "jsonschema>=4.26.0", +# "pyyaml>=6.0.3", +# "sqlglot>=30.12.0", +# ] +# /// + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information From 7835af52d27e23dbf422ac02a70fd01083986e02 Mon Sep 17 00:00:00 2001 From: Emil Sadek Date: Mon, 13 Jul 2026 15:56:03 -0700 Subject: [PATCH 54/89] feat(python): add bigquery dialect --- python/src/ossie/models.py | 1 + validation/validate.py | 1 + 2 files changed, 2 insertions(+) diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index cc4aad5..4d0fe57 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -31,6 +31,7 @@ class OSIDialect(str, Enum): MAQL = "MAQL" TABLEAU = "TABLEAU" DATABRICKS = "DATABRICKS" + BIGQUERY = "BIGQUERY" class OSIVendor(str, Enum): diff --git a/validation/validate.py b/validation/validate.py index ad78d30..143126b 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -55,6 +55,7 @@ "ANSI_SQL": None, # sqlglot default "SNOWFLAKE": "snowflake", "DATABRICKS": "databricks", + "BIGQUERY": "bigquery", "MDX": None, # Not supported by sqlglot, skip validation "TABLEAU": None, # Not supported by sqlglot, skip validation "MAQL": None, # Not supported by sqlglot, skip validation From 6b876816c49f4114169bc9be792d3222c31c5bb8 Mon Sep 17 00:00:00 2001 From: Emil Sadek Date: Mon, 13 Jul 2026 16:10:18 -0700 Subject: [PATCH 55/89] docs: rename converters/index.md to converters/README.md --- converters/{index.md => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename converters/{index.md => README.md} (100%) diff --git a/converters/index.md b/converters/README.md similarity index 100% rename from converters/index.md rename to converters/README.md From 77553b35bc755a22af02736070574d8346f0b761 Mon Sep 17 00:00:00 2001 From: Onkar Dahale Date: Wed, 1 Jul 2026 20:15:35 +0900 Subject: [PATCH 56/89] python: accept free-form custom extension vendors Custom extension vendor names are free-form strings in the current OSI specification and JSON schema. The Python model still typed OSICustomExtension.vendor_name as OSIVendor, which rejects schema-valid documents using a vendor name outside the built-in well-known list. Change vendor_name to str so the Python package accepts any spec-valid custom extension vendor name. Keep OSIVendor available as well-known constants for existing callers. Update the converter guide to stop telling contributors to add new vendors to a core enum. Signed-off-by: Onkar Dahale --- converters/README.md | 2 +- python/src/ossie/models.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/converters/README.md b/converters/README.md index 1ceb3b8..9dd4f98 100644 --- a/converters/README.md +++ b/converters/README.md @@ -281,7 +281,7 @@ Given the [TPC-DS example](../examples/tpcds_semantic_model.yaml) included in th To add support for a new vendor: -1. Add the vendor to the `vendors` enum in the [core specification](../core-spec/spec.md) if not already present. +1. Use a stable `vendor_name` string in each custom extension emitted by the converter. 2. Define the custom extension schema for the vendor (what vendor-specific metadata fields are supported in the `data` JSON). 3. Implement the export converter (Ossie → Vendor). 4. Implement the import converter (Vendor → Ossie). diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index 4d0fe57..b0db1cc 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -35,7 +35,7 @@ class OSIDialect(str, Enum): class OSIVendor(str, Enum): - """Vendors with supported custom extensions.""" + """Well-known vendor names for custom extensions.""" COMMON = "COMMON" SNOWFLAKE = "SNOWFLAKE" @@ -63,7 +63,7 @@ class OSICustomExtension(BaseModel): model_config = ConfigDict(frozen=True) - vendor_name: OSIVendor + vendor_name: str data: str From ca66497486a85feaf37e2fda595edad7d7bf8f71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Tue, 14 Jul 2026 06:20:09 +0200 Subject: [PATCH 57/89] Add branch protection on main --- .asf.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.asf.yaml b/.asf.yaml index 462fcf8..dc9bbe3 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -41,6 +41,16 @@ github: review_drafts: false review_on_push: false + protected_branches: + main: + required_pull_request_reviews: + require_code_owner_reviews: false + dismiss_stale_reviews: true + require_last_push_approval: false + required_approving_review_count: 1 + + required_linear_history: true + features: wiki: false issues: true From 502f088b6157a3630a75c23995b990f850771cad Mon Sep 17 00:00:00 2001 From: Yong Zheng Date: Mon, 13 Jul 2026 23:26:20 -0500 Subject: [PATCH 58/89] Fix Polaris converter build (#188) --- converters/polaris/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/converters/polaris/pom.xml b/converters/polaris/pom.xml index 6f94523..759cdb0 100644 --- a/converters/polaris/pom.xml +++ b/converters/polaris/pom.xml @@ -41,6 +41,7 @@ 11 11 + 11 UTF-8 2.2 2.17.0 From b9ddbe8cebe2d509641fd3712e40cfa54825ba8f Mon Sep 17 00:00:00 2001 From: Yong Zheng Date: Mon, 13 Jul 2026 23:32:06 -0500 Subject: [PATCH 59/89] Fix Polaris converter architecture ASCII. (#189) --- converters/polaris/README.md | 38 ++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/converters/polaris/README.md b/converters/polaris/README.md index d24aac7..d2f252f 100644 --- a/converters/polaris/README.md +++ b/converters/polaris/README.md @@ -112,25 +112,25 @@ Since Ossie fields are expression-based and don't carry explicit types, the expo ## Architecture ``` - ┌───────────────┐ - │ Polaris REST │ - │ Catalog │ - └──────┬────────┘ - │ - ┌──────┴────────┐ - │ PolarisClient │ Iceberg REST API - └──────┬────────┘ - │ - ┌────────────┼────────────┐ - │ │ - ┌────────┴────────┐ ┌─────────┴─────────┐ - │ PolarisImporter │ │ PolarisExporter │ - │ (Polaris→Ossie) │ │ (Ossie→Polaris) │ - └────────┬────────┘ └─────────┬─────────┘ - │ │ - ┌──────┴──────┐ ┌──────┴──────┐ - │OsiYamlGen. │ │OsiModelParser│ - └─────────────┘ └─────────────┘ + ┌──────────────────┐ + │ Polaris REST │ + │ Catalog │ + └────────┬─────────┘ + │ + ┌────────┴─────────┐ + │ PolarisClient │ Iceberg REST API + └────────┬─────────┘ + │ + ┌───────────────┼───────────────┐ + │ │ + ┌────────┴─────────┐ ┌─────────┴─────────┐ + │ PolarisImporter │ │ PolarisExporter │ + │ (Polaris → Ossie)│ │ (Ossie → Polaris) │ + └────────┬─────────┘ └─────────┬─────────┘ + │ │ + ┌────────┴─────────┐ ┌─────────┴─────────┐ + │ OsiYamlGenerator │ │ OsiModelParser │ + └──────────────────┘ └───────────────────┘ ``` ## Dependencies From 52600802065fadd4b6565b525bc25c9a6d3609f5 Mon Sep 17 00:00:00 2001 From: Emil Sadek Date: Mon, 13 Jul 2026 21:33:38 -0700 Subject: [PATCH 60/89] chore: trim trailing whitespace (#181) Co-authored-by: Emil Sadek --- .asf.yaml | 2 +- CONTRIBUTING.md | 2 +- .../salesforce/src/main/resources/mappings.yaml | 2 +- core-spec/spec.md | 10 +++++----- core-spec/spec.yaml | 12 ++++++------ ontology/ontology.json | 4 ++-- ontology/ontology.md | 10 +++++----- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 462fcf8..acad1eb 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # - + # The format of this file is documented at # https://github.com/apache/infrastructure-asfyaml # also at diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9307ff3..15bf5b9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,7 +59,7 @@ it didn't happen.* [archives](https://lists.apache.org/list.html?dev@ossie.apache.org). - **commits@ossie.apache.org** — automated notifications for commits, pull requests. Subscribe via `commits-subscribe@ossie.apache.org`. -- **issues@ossie.apache.org** — automated notifications for issues. +- **issues@ossie.apache.org** — automated notifications for issues. Subscribe via `issues-subscribe@ossie.apache.org`. - **private@ossie.apache.org** — the Podling Project Management Committee (PPMC) private list, used only for confidential matters such as committer nominations. diff --git a/converters/salesforce/src/main/resources/mappings.yaml b/converters/salesforce/src/main/resources/mappings.yaml index 57552cf..15f70cb 100644 --- a/converters/salesforce/src/main/resources/mappings.yaml +++ b/converters/salesforce/src/main/resources/mappings.yaml @@ -19,7 +19,7 @@ # Ossie-Salesforce Converter - Mapping Configuration # ============================================================================= # -# This file defines ONLY straightforward property mappings between Ossie YAML +# This file defines ONLY straightforward property mappings between Ossie YAML # format and Salesforce Semantic Model JSON format. # # Complex mappings (arrays, routing, transformations) are handled programmatically diff --git a/core-spec/spec.md b/core-spec/spec.md index 918ee8d..8b9c10a 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -252,7 +252,7 @@ expression: - dialect: ANSI_SQL expression: customer_id description: Customer identifier - dimension: + dimension: is_time: false ``` @@ -463,14 +463,14 @@ semantic_model: - dialect: ANSI_SQL expression: order_id description: Order identifier - + - name: customer_id expression: dialects: - dialect: ANSI_SQL expression: customer_id description: Customer identifier - + - name: order_date expression: dialects: @@ -479,7 +479,7 @@ semantic_model: dimension: is_time: true description: Order date - + - name: amount expression: dialects: @@ -498,7 +498,7 @@ semantic_model: - dialect: ANSI_SQL expression: id description: Customer identifier - + - name: email expression: dialects: diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index c108690..a5617e8 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -18,7 +18,7 @@ # Apache Ossie - Core Metadata Spec (YAML Schema) # DRAFT version - in development, schema may change before 0.2.0 is released version: 0.2.0.dev0 -# +# # Goals: # - Standardization: Establish uniform language and structure for semantic model definitions # - Extensibility: Support domain-specific extensions while maintaining core compatibility @@ -44,7 +44,7 @@ dialects: # Examples: "COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA" vendor_name: string - + # Top-level semantic model definition semantic_model: # Required: Unique identifier for the semantic model @@ -64,7 +64,7 @@ semantic_model: # See Relationships section below for detailed structure relationships: [] - # Optional: + # Optional: # These metrics can span one or more logical datasets and use relationships # See Metrics section below for detailed structure metrics: [] @@ -86,7 +86,7 @@ datasets: # Required: Reference to the underlying physical table/view or query # Format should be either database_name.schema_name.table_name or query source: string - + # Optional: Primary key definition that uniquely identifies rows in this dataset # Can be a single column or a composite of multiple columns # This is the preferred unique identifier for this dataset and is used in relationships to determine many-to-one or one-to-one. @@ -107,8 +107,8 @@ datasets: # - [column2, column3] # # unique_keys: - # - [column1, column2] - # - [column3, column4] + # - [column1, column2] + # - [column3, column4] unique_keys: - [] # Array of column names (single or composite) diff --git a/ontology/ontology.json b/ontology/ontology.json index 72dcb34..943f6aa 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -58,7 +58,7 @@ "concept": { "$ref": "#/$defs/Concept" }, - "relationships": { + "relationships": { "type": "array", "items": { "$ref": "#/$defs/Relationship" @@ -181,7 +181,7 @@ "$ref": "#/$defs/ObjectMapping" }, "description": "Mappings from logical constructs that populate the concept in this component. Valid only when the concept is an entity type" - }, + }, "link_mappings": { "type": "array", "items": { diff --git a/ontology/ontology.md b/ontology/ontology.md index 05ad614..3e9593f 100644 --- a/ontology/ontology.md +++ b/ontology/ontology.md @@ -101,7 +101,7 @@ concept plays the first role: Concepts represent the types of things that have meaning in a business setting, e.g., person, company, or salary. Every ontology implicitly includes all of the [built-in concepts](#built-in-concepts) and -may refer to them by name without declaring them. +may refer to them by name without declaring them. Concepts have the following schema: @@ -187,7 +187,7 @@ ontology: the relationship is identified by the string `Person.earns`. This convention naturally supports expressions that navigate over the links of relationships using the “dot-join” operator in a manner that is familiar to object-oriented programming languages. This relationship links -`Person` and `Salary` objects and verbalizes each link as “Person earns Salary.” +`Person` and `Salary` objects and verbalizes each link as “Person earns Salary.” #### Roles @@ -285,7 +285,7 @@ idnetifier of a concept. ### Derivation expressions -Concepts and relationships may be derived using expressions. Think of a derived concept or +Concepts and relationships may be derived using expressions. Think of a derived concept or relationship as a view whose objects or links are derived from those of other concepts or relationships. For instance: @@ -306,7 +306,7 @@ ontology: name: "descendant" derived_by: - "Person.parent_of(descendant)" - "Person.ancestor_of.parent_of(descendant)" + "Person.ancestor_of.parent_of(descendant)" - name: taxed_at roles: - concept: TaxRate @@ -458,7 +458,7 @@ then mapping those values to `Person` objects using the declared identifier. The concept_mappings: - concept: Person object_mappings: - - expression: PERSONS.SSN + - expression: PERSONS.SSN ... ``` maps values from the `SSN` field of dataset `PERSONS` into `Person` objects. From 2d909fa26026cebd1edc15cee707dc320c7576ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 06:34:05 +0200 Subject: [PATCH 61/89] Bump com.fasterxml.jackson.core:jackson-databind (#177) Bumps [com.fasterxml.jackson.core:jackson-databind](https://github.com/FasterXML/jackson) from 2.18.6 to 2.18.9. - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-version: 2.18.9 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- converters/salesforce/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/converters/salesforce/pom.xml b/converters/salesforce/pom.xml index 8e782ca..89c5b03 100644 --- a/converters/salesforce/pom.xml +++ b/converters/salesforce/pom.xml @@ -45,7 +45,7 @@ 17 UTF-8 - 2.18.6 + 2.18.9 5.11.4 2.0.16 1.5.25 From 5c8a2a5f7e09e046e2055e5759e7df4a928b7a88 Mon Sep 17 00:00:00 2001 From: Yong Zheng Date: Tue, 14 Jul 2026 00:30:45 -0500 Subject: [PATCH 62/89] Fix validation script (#191) --- validation/validate.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/validation/validate.py b/validation/validate.py index b7cf13e..5965115 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -142,9 +142,9 @@ def validate_references(data: dict) -> list[str]: to_ds = rel.get("to") if from_ds and from_ds not in dataset_names: - errors.append(f"[Reference] Relationship '{rel_name}' references unknown dataset '{from_ds}'") + errors.append(f"[Reference] Relationship '{rel_name}' in model '{model_name}' references unknown dataset '{from_ds}'") if to_ds and to_ds not in dataset_names: - errors.append(f"[Reference] Relationship '{rel_name}' references unknown dataset '{to_ds}'") + errors.append(f"[Reference] Relationship '{rel_name}' in model '{model_name}' references unknown dataset '{to_ds}'") return errors @@ -198,7 +198,7 @@ def validate_sql(data: dict) -> list[str]: dialect = dialect_expr.get("dialect", "ANSI_SQL") expr = dialect_expr.get("expression", "") if expr: - context = f"Field '{dataset_name}.{field_name}' ({dialect})" + context = f"Field '{dataset_name}.{field_name}' in model '{model_name}' ({dialect})" error = validate_sql_expression(expr, dialect, context) if error: errors.append(error) @@ -211,7 +211,7 @@ def validate_sql(data: dict) -> list[str]: dialect = dialect_expr.get("dialect", "ANSI_SQL") expr = dialect_expr.get("expression", "") if expr: - context = f"Metric '{metric_name}' ({dialect})" + context = f"Metric '{metric_name}' in model '{model_name}' ({dialect})" error = validate_sql_expression(expr, dialect, context) if error: errors.append(error) @@ -227,7 +227,7 @@ def main(): args = sys.argv[1:] yaml_path = Path(args[0]) - schema_path = Path(__file__).parent.parent / "core-spec" / "ossie-schema.json" + schema_path = Path(__file__).parent.parent / "core-spec" / "osi-schema.json" if len(args) > 1: if len(args) == 3 and args[1] == "--schema": schema_path = Path(args[2]) From 6abd059cb6802114fab0f65aebbca4f136ae7028 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Tue, 14 Jul 2026 20:28:49 +0800 Subject: [PATCH 63/89] docs: fix converter guide links --- ROADMAP.md | 3 +-- docs/index.md | 10 +++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 5ca262c..af7c6bf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -388,7 +388,7 @@ New adopters and tool authors need clearer documentation, real-world samples, an - [Core Specification (spec.md)](core-spec/spec.md) — the current Ossie spec document - [TPC-DS Example Model](examples/tpcds_semantic_model.yaml) — reference semantic model using the TPC-DS benchmark -- [Converter Guide (converters/index.md)](converters/index.md) — hub-and-spoke converter architecture and authoring guide +- [Converter Guide (converters/README.md)](converters/README.md) — hub-and-spoke converter architecture and authoring guide --- @@ -439,4 +439,3 @@ Broad ecosystem adoption depends on practical tools that let teams validate thei - [Issue #121 — Create converter/common module (for Java binding)](https://github.com/apache/ossie/issues/121) - [Issue #111 — Follow up on Ossie ai_context and custom_extensions mapping in Snowflake YAML](https://github.com/apache/ossie/issues/111) - diff --git a/docs/index.md b/docs/index.md index 4365166..0a46482 100644 --- a/docs/index.md +++ b/docs/index.md @@ -73,7 +73,7 @@ Alation, Anomalo, Atlan, AtScale, Bigeye, BlackRock, Blue Yonder, Carto, Clouder Ossie converters follow a **hub-and-spoke** architecture: the Ossie core specification acts as the central, vendor-neutral format, and each converter handles translation to or from a specific vendor format (e.g., Snowflake, dbt, Salesforce, Databricks). This avoids the need for point-to-point converters between every pair of vendors. -For details on implementing a converter, see the [Converters Guide](../converters/index.md). +For details on implementing a converter, see the [Converters Guide](../converters/README.md). --- @@ -228,10 +228,10 @@ No. Ossie is vendor-agnostic by design. The specification is developed and gover ### Adoption **Can I use Ossie with my existing BI tool?** -Yes, as long as a converter exists (or is built) for your tool. The hub-and-spoke model means that adding Ossie support to a single tool gives it interoperability with every other Ossie-compatible tool. Check the [Converters Guide](../converters/index.md) for currently supported vendors. +Yes, as long as a converter exists (or is built) for your tool. The hub-and-spoke model means that adding Ossie support to a single tool gives it interoperability with every other Ossie-compatible tool. Check the [Converters Guide](../converters/README.md) for currently supported vendors. **What if my vendor isn't supported yet?** -You can contribute a converter. The [Converters Guide](../converters/index.md) provides a step-by-step guide for implementing import and export converters for new vendors. The community is happy to help with design reviews and testing. +You can contribute a converter. The [Converters Guide](../converters/README.md) provides a step-by-step guide for implementing import and export converters for new vendors. The community is happy to help with design reviews and testing. **Do I need to rewrite my existing semantic models?** No. Import converters translate existing vendor-specific models into the Ossie format automatically. Your existing models remain intact — Ossie provides an additional interchange layer on top of them. @@ -287,7 +287,7 @@ A practical guide for organizations looking to adopt Ossie. - **Inventory your semantic layer**: Identify which tools in your organization define semantic models — BI platforms, data modeling tools, AI/ML pipelines, metrics stores. - **Map your pain points**: Determine where semantic fragmentation causes the most friction — conflicting metric definitions, manual reconciliation, onboarding new tools. -- **Check converter availability**: Review the [Converters Guide](../converters/index.md) to see if converters exist for your tools. If not, assess the effort to build one. +- **Check converter availability**: Review the [Converters Guide](../converters/README.md) to see if converters exist for your tools. If not, assess the effort to build one. ### Phase 2: Pilot @@ -342,7 +342,7 @@ A practical guide for organizations looking to adopt Ossie. - **YAML Schema**: [core-spec/spec.yaml](../core-spec/spec.yaml) - **TPC-DS Example Model**: [examples/tpcds_semantic_model.yaml](../examples/tpcds_semantic_model.yaml) - **Validation Script**: [validation/validate.py](../validation/validate.py) -- **Converters Guide**: [converters/index.md](../converters/index.md) +- **Converters Guide**: [converters/README.md](../converters/README.md) --- From 1ae231fe592c1c6842efcb4bc6639d08046ed1ad Mon Sep 17 00:00:00 2001 From: Ralf Becher Date: Tue, 14 Jul 2026 19:11:00 +0200 Subject: [PATCH 64/89] orionbelt converter: link OSI core schema, drop vendored copies and dead constants Addresses review feedback on this PR: 1. Stop vendoring a copy of the OSI spec. validation._osi_schema_path resolves the OSI core schema from a search path: a copy vendored beside the package if present, otherwise core-spec/osi-schema.json from the enclosing checkout. The vendored osi-schema.json is deleted, so OSI-core validation now links the single canonical core-spec copy with nothing to keep in sync. 2. Omit ontology-schema validation. ossie ships no OSI ontology schema and the converter emits ontology documents but never consumes them, so the vendored osi-ontology-schema.json is removed and validate_osi_ontology runs semantic checks only (unique concepts, reference integrity). 3. Remove the unused _OSI_KNOWN_VENDORS and _OSI_KNOWN_DIALECTS constants (only re-exported; passthrough uses _INTERNAL_VENDORS, dialect parsing uses _SQL_PARSEABLE_DIALECTS). Only the OBML schema (OrionBelt's own format) stays vendored. mypy override added for the optional orionbelt engine import. 145 tests pass; ruff clean; mypy clean with the dev extra. --- .../orionbelt/osi_obml_mapping_analysis.md | 4 +- .../osi_obml_ontology_mapping_analysis.md | 12 +- converters/orionbelt/pyproject.toml | 17 +- .../orionbelt/src/ossie_orionbelt/_common.py | 12 - .../src/ossie_orionbelt/converter.py | 15 - .../schemas/osi-ontology-schema.json | 299 ---------------- .../ossie_orionbelt/schemas/osi-schema.json | 330 ------------------ .../src/ossie_orionbelt/validation.py | 96 +++-- .../tests/test_osi_converter_ontology.py | 13 +- .../orionbelt/tests/test_osi_v02_compat.py | 11 +- 10 files changed, 70 insertions(+), 739 deletions(-) delete mode 100644 converters/orionbelt/src/ossie_orionbelt/schemas/osi-ontology-schema.json delete mode 100644 converters/orionbelt/src/ossie_orionbelt/schemas/osi-schema.json diff --git a/converters/orionbelt/osi_obml_mapping_analysis.md b/converters/orionbelt/osi_obml_mapping_analysis.md index dc62018..d94fe09 100644 --- a/converters/orionbelt/osi_obml_mapping_analysis.md +++ b/converters/orionbelt/osi_obml_mapping_analysis.md @@ -167,7 +167,7 @@ The converter includes dual-layer validation for both formats, ensuring that con ### 4.2 OSI Validation -1. **JSON Schema** — validates against `osi-schema.json` (Draft 2020-12) +1. **JSON Schema** — validates against the OSI core `osi-schema.json` (Draft 2020-12), resolved from the repo's `core-spec/` (no vendored copy) 2. **Unique names** — checks uniqueness of dataset, field, metric, and relationship names 3. **References** — verifies that relationship `from`/`to` reference existing datasets @@ -258,7 +258,7 @@ Converting the OBML output back to OSI produces a valid OSI model where: |---|---| | `tests/fixtures/tpcds_osi.yaml` | Official TPC-DS OSI example (from OSI repo) | | `tests/fixtures/tpcds_as_obml.yaml` | Converted OBML output | -| `src/ossie_orionbelt/schemas/osi-schema.json` | OSI JSON Schema (Draft 2020-12, from OSI repo) | +| `core-spec/osi-schema.json` | OSI core JSON Schema (Draft 2020-12), resolved at runtime; not vendored | | `src/ossie_orionbelt/converter.py` | Bidirectional converter with validation | ## 7. Future Considerations diff --git a/converters/orionbelt/osi_obml_ontology_mapping_analysis.md b/converters/orionbelt/osi_obml_ontology_mapping_analysis.md index 6a7bc95..0764bf1 100644 --- a/converters/orionbelt/osi_obml_ontology_mapping_analysis.md +++ b/converters/orionbelt/osi_obml_ontology_mapping_analysis.md @@ -1,8 +1,8 @@ # OBML → OSI Ontology Mapping Analysis This pins the rules used by `OBMLtoOSIOntology` to derive an **OSI ontology -document** (validated against `src/ossie_orionbelt/schemas/osi-ontology-schema.json`, OSI version -`0.2.0.dev0`) from an OBML semantic model. +document** (OSI ontology format, version `0.2.0.dev0`) from an OBML semantic +model. The OSI ontology is a **separate document** from the OSI core-spec semantic model (different `$id`, different required root). It is produced alongside the @@ -68,10 +68,10 @@ falling back to the dataset name when `source` has no dotted physical table. ## Validation -`validate_osi_ontology()` runs: -1. JSON Schema (Draft 2020-12) against `osi-ontology-schema.json`. -2. Semantic checks: unique concept names; relationship `roles` reference defined - concepts; `concept_mappings` reference defined concepts. +`validate_osi_ontology()` runs semantic checks only (ossie ships no OSI ontology +schema, and the converter emits ontology documents but never consumes them): +unique concept names; relationship `roles` reference defined concepts; +`concept_mappings` reference defined concepts. ## Stability note diff --git a/converters/orionbelt/pyproject.toml b/converters/orionbelt/pyproject.toml index f580cdb..474571a 100644 --- a/converters/orionbelt/pyproject.toml +++ b/converters/orionbelt/pyproject.toml @@ -36,11 +36,11 @@ ossie-orionbelt = "ossie_orionbelt.cli:main" [tool.hatch.build.targets.wheel] packages = ["src/ossie_orionbelt"] -# All three schemas (the two OSI artefacts plus a vendored snapshot of the -# canonical OBML schema) live in src/ossie_orionbelt/schemas/ as tracked files, -# so the sdist and wheel are self-contained and build in isolation. The OBML -# snapshot is kept in sync with the repo-root schema/obml-schema.json by a -# drift-guard test (tests/unit/test_ossie_orionbelt_schema_sync.py). +# Only the OBML schema (OrionBelt's own format) is vendored under +# src/ossie_orionbelt/schemas/. OSI core documents are validated against the +# repo's core-spec/osi-schema.json, resolved at runtime by +# validation._osi_schema_path (no duplicate copy). The OSI ontology export is +# validated semantically only (ossie ships no ontology schema). [tool.ruff] line-length = 100 @@ -53,6 +53,13 @@ select = ["E", "F", "I", "N", "UP", "B", "A", "SIM"] python_version = "3.12" files = ["src/ossie_orionbelt"] +# The OrionBelt engine is an optional runtime dependency: validate_obml uses it +# for semantic validation when present and degrades gracefully otherwise, so it +# is not installed in this standalone converter and has no published stubs. +[[tool.mypy.overrides]] +module = ["orionbelt.*"] +ignore_missing_imports = true + [tool.pytest.ini_options] testpaths = ["tests"] pythonpath = ["src"] diff --git a/converters/orionbelt/src/ossie_orionbelt/_common.py b/converters/orionbelt/src/ossie_orionbelt/_common.py index 7f3afdd..3460381 100644 --- a/converters/orionbelt/src/ossie_orionbelt/_common.py +++ b/converters/orionbelt/src/ossie_orionbelt/_common.py @@ -16,8 +16,6 @@ # 0.1.x (via the legacy shim) and 0.2.x. _OSI_VERSION = "0.2.0.dev0" -# Dialect / vendor enum extras new in v0.2.0.dev0 -_OSI_KNOWN_DIALECTS = ("ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL") # SQL dialects (of the OSI enum) whose aggregation expressions our regex-based # metric parser can read, in preference order. ANSI_SQL first; SNOWFLAKE and # DATABRICKS are SQL engines OrionBelt also targets, and their simple/expression @@ -36,16 +34,6 @@ r"\s*\.\s*" r'(?P[A-Za-z_]\w*|"[^"]+"|`[^`]+`|\[[^\]]+\])' ) -_OSI_KNOWN_VENDORS = ( - "COMMON", - "ORIONBELT", - "SNOWFLAKE", - "SALESFORCE", - "DBT", - "DATABRICKS", - "GOODDATA", -) - # Vendor identities for custom_extensions. # ORIONBELT - OrionBelt/OBML-proprietary payloads we author on OBML -> OSI. # OSI - OSI-native fields OBML can't hold (unique_keys, field label, diff --git a/converters/orionbelt/src/ossie_orionbelt/converter.py b/converters/orionbelt/src/ossie_orionbelt/converter.py index 3acd2f7..9d08ff8 100644 --- a/converters/orionbelt/src/ossie_orionbelt/converter.py +++ b/converters/orionbelt/src/ossie_orionbelt/converter.py @@ -41,12 +41,6 @@ from ossie_orionbelt._common import ( _OBML_VENDOR_READ as _OBML_VENDOR_READ, ) -from ossie_orionbelt._common import ( - _OSI_KNOWN_DIALECTS as _OSI_KNOWN_DIALECTS, -) -from ossie_orionbelt._common import ( - _OSI_KNOWN_VENDORS as _OSI_KNOWN_VENDORS, -) from ossie_orionbelt._common import ( _OSI_VENDOR_READ as _OSI_VENDOR_READ, ) @@ -78,12 +72,6 @@ from ossie_orionbelt.validation import ( _OBML_SCHEMA_PATH as _OBML_SCHEMA_PATH, ) -from ossie_orionbelt.validation import ( - _OSI_CORE_SPEC_RAW_URL as _OSI_CORE_SPEC_RAW_URL, -) -from ossie_orionbelt.validation import ( - _OSI_ONTOLOGY_SCHEMA_PATH as _OSI_ONTOLOGY_SCHEMA_PATH, -) from ossie_orionbelt.validation import ( _OSI_SCHEMA_PATH as _OSI_SCHEMA_PATH, ) @@ -96,9 +84,6 @@ from ossie_orionbelt.validation import ( ValidationResult as ValidationResult, ) -from ossie_orionbelt.validation import ( - _osi_core_registry as _osi_core_registry, -) from ossie_orionbelt.validation import ( _validate_json_schema as _validate_json_schema, ) diff --git a/converters/orionbelt/src/ossie_orionbelt/schemas/osi-ontology-schema.json b/converters/orionbelt/src/ossie_orionbelt/schemas/osi-ontology-schema.json deleted file mode 100644 index 473f69b..0000000 --- a/converters/orionbelt/src/ossie_orionbelt/schemas/osi-ontology-schema.json +++ /dev/null @@ -1,299 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/open-semantic-interchange/OSI/core-spec/osi-schema.json", - "title": "OSI Ontology Metadata Specification", - "description": "JSON Schema for validating OSI (Open Semantic Interoperability) ontology definitions", - "type": "object", - "properties": { - "version": { - "type": "string", - "const": "0.2.0.dev0", - "description": "Ontology specification version" - }, - "name": { - "type": "string", - "description": "Unique identifier for the ontology" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "ai_context": { - "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/AIContext" - }, - "ontology": { - "type": "array", - "items": { - "$ref": "#/$defs/OntologyComponent" - }, - "minItems": 1, - "description": "Components that define the concepts and relationships in this ontology" - }, - "ontology_mappings": { - "type": "array", - "description": "Collection of ontology maps from logical models", - "items": { - "$ref": "#/$defs/OntologyMap" - } - } - }, - "required": ["version", "name", "ontology"], - "additionalProperties": false, - "$defs": { - "OntologyComponent": { - "type": "object", - "description": "Ontology component that defines a single concept and any relationships that are keyed primarily by that concept", - "properties": { - "description": { - "type": "string", - "description": "Human-readable description of the component" - }, - "concept": { - "$ref": "#/$defs/Concept" - }, - "relationships": { - "type": "array", - "items": { - "$ref": "#/$defs/Relationship" - }, - "description": "Defines relationships that pertain primarily to the concept defined in this component" - } - }, - "required": ["concept"], - "additionalProperties": false - }, - "Expression": { - "type": "string", - "description": "ANSI SQL expression" - }, - "Relationship": { - "type": "object", - "description": "Relationship between concepts in the ontology", - "properties": { - "name": { - "type": "string", - "description": "Name of the relationship" - }, - "description": { - "type": "string", - "description": "Human-readable description of the relationship" - }, - "roles": { - "type": "array", - "items": { - "$ref": "#/$defs/Role" - }, - "description": "Additional roles in this relationship" - }, - "multiplicity": { - "$ref": "#/$defs/Multiplicity" - }, - "derived_by": { - "type": "array", - "items": { - "$ref": "#/$defs/Expression" - }, - "description": "Expressions that define how this concept is derived" - }, - "verbalizes": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Natural language expressions that verbalize this relationship" - } - }, - "required": ["name", "verbalizes"], - "additionalProperties": false - }, - "Concept": { - "type": "object", - "description": "Defines a concept in the ontology", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the concept" - }, - "type": { - "$ref": "#/$defs/ConceptType" - }, - "description": { - "type": "string", - "description": "Human-readable description of the concept" - }, - "extends": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Indicates that this concept extends one or more other concepts" - }, - "derived_by": { - "type": "array", - "items": { - "$ref": "#/$defs/Expression" - }, - "description": "Expressions that define how this concept is derived" - }, - "identify_by": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Names of relationships to use as the preferred identifier of this concept" - }, - "requires": { - "type": "array", - "items": { - "$ref": "#/$defs/Expression" - }, - "description": "Expressions that constrain the population of this concept" - } - }, - "required": ["name", "type"], - "additionalProperties": false - }, - "ConceptMapping": { - "type": "object", - "description": "Mappings from logical model constructs to some ontology component", - "properties": { - "concept": { - "type": "string", - "description": "Name of the concept whose part of the ontology we are mapping to" - }, - "object_mappings": { - "type": "array", - "items": { - "$ref": "#/$defs/ObjectMapping" - }, - "description": "Mappings from logical constructs that populate the concept in this component. Valid only when the concept is an entity type" - }, - "link_mappings": { - "type": "array", - "items": { - "$ref": "#/$defs/LinkMapping" - }, - "description": "Mappings from logical model relationships to ontology relationships pertaining to the mapped concept" - } - }, - "required": ["concept"], - "additionalProperties": false - }, - "ConceptType": { - "type": "string", - "enum": [ "EntityType", "ValueType" ], - "description": "A concept is either an entity type or a value type" - }, - "ReferentMapping": { - "type": "object", - "description": "Mapping from logical model constructs to a relationship used to references some entity type in the ontology", - "properties": { - "relationship": { - "type": "string", - "description": "Name of referent relationship" - }, - "expression": { - "$ref": "#/$defs/Expression" - }, - "referent_mappings": { - "type": "array", - "items": { - "$ref": "#/$defs/ReferentMapping" - } - } - }, - "required": ["relationship"], - "additionalProperties": false - }, - "Role": { - "type": "object", - "description": "Role in some relationship (the container)", - "properties": { - "concept": { - "type": "string", - "description": "Name of the concept playing this role" - }, - "name": { - "type": "string", - "description": "Optional name of this role, used when the same concept plays multiple roles in the same relationship" - } - }, - "required": ["concept"], - "additionalProperties": false - }, - "ObjectMapping": { - "type": "object", - "description": "Pattern of logical-level expressions for identifying objects of some concept using the values in one or more fields", - "properties": { - "concept": { - "type": "string", - "description": "Name of the concept whose objects we are mapping to" - }, - "referent_mappings": { - "type": "array", - "items": { - "$ref": "#/$defs/ReferentMapping" - }, - "description": "Maps logical-model constructs to referent relationships of this entity type" - }, - "expression": { - "$ref": "#/$defs/Expression" - } - }, - "additionalProperties": false - }, - "LinkMapping": { - "type": "object", - "description": "Mapping from logical schema to the links of relationships in the ontology", - "properties": { - "relationship": { - "type": "string", - "description": "Name of relationship being populated by this mapping node" - }, - "object_mapping": { - "$ref": "#/$defs/ObjectMapping" - }, - "children": { - "type": "array", - "items": { - "$ref": "#/$defs/LinkMapping" - }, - "description": "Relationship maps at the next level in this hierarchy" - } - }, - "required": ["object_mapping"], - "additionalProperties": false - }, - "Multiplicity": { - "type": "string", - "enum": [ "ManyToOne", "OneToOne" ], - "description": "Relationship multiplicity" - }, - "OntologyMap": { - "type": "object", - "description": "Map from the constructs of some logical model to some ontology", - "properties": { - "name": { - "type": "string", - "description": "Name of this ontology map" - }, - "description": { - "type": "string", - "description": "Human-readable description of this ontology map" - }, - "semantic_model": { - "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/SemanticModel" - }, - "concept_mappings": { - "type": "array", - "items": { - "$ref": "#/$defs/ConceptMapping" - }, - "description": "Maps logical model constructs to some concept and its relationships in the ontology" - } - }, - "required": ["semantic_model", "concept_mappings"], - "additionalProperties": false - } - } -} diff --git a/converters/orionbelt/src/ossie_orionbelt/schemas/osi-schema.json b/converters/orionbelt/src/ossie_orionbelt/schemas/osi-schema.json deleted file mode 100644 index 72cb164..0000000 --- a/converters/orionbelt/src/ossie_orionbelt/schemas/osi-schema.json +++ /dev/null @@ -1,330 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/open-semantic-interchange/OSI/core-spec/osi-schema.json", - "title": "OSI Core Metadata Specification", - "description": "JSON Schema for validating OSI (Open Semantic Interoperability) semantic model definitions", - "type": "object", - "properties": { - "version": { - "type": "string", - "const": "0.2.0.dev0", - "description": "OSI specification version" - }, - "semantic_model": { - "type": "array", - "description": "Collection of semantic model definitions", - "items": { - "$ref": "#/$defs/SemanticModel" - } - } - }, - "required": ["version", "semantic_model"], - "additionalProperties": false, - "$defs": { - "Dialect": { - "type": "string", - "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL"], - "description": "Supported SQL and expression language dialects" - }, - "Vendor": { - "type": "string", - "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA"], - "description": "Vendor name for custom extensions. Any string value is accepted." - }, - "AIContext": { - "description": "Additional context for AI tools", - "oneOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "instructions": { - "type": "string", - "description": "Instructions for AI on how to use this entity" - }, - "synonyms": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Alternative names and terms" - }, - "examples": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Sample questions or use cases" - } - }, - "additionalProperties": true - } - ] - }, - "CustomExtension": { - "type": "object", - "description": "Vendor-specific attributes for extensibility", - "properties": { - "vendor_name": { - "$ref": "#/$defs/Vendor" - }, - "data": { - "type": "string", - "description": "JSON string containing vendor-specific data" - } - }, - "required": ["vendor_name", "data"], - "additionalProperties": false - }, - "DialectExpression": { - "type": "object", - "description": "Expression in a specific dialect", - "properties": { - "dialect": { - "$ref": "#/$defs/Dialect" - }, - "expression": { - "type": "string", - "description": "SQL or dialect-specific expression" - } - }, - "required": ["dialect", "expression"], - "additionalProperties": false - }, - "Expression": { - "type": "object", - "description": "Expression definition with multi-dialect support", - "properties": { - "dialects": { - "type": "array", - "items": { - "$ref": "#/$defs/DialectExpression" - }, - "minItems": 1 - } - }, - "required": ["dialects"], - "additionalProperties": false - }, - "Dimension": { - "type": "object", - "description": "Dimension metadata", - "properties": { - "is_time": { - "type": "boolean", - "description": "Indicates if this is a time-based dimension for temporal filtering" - } - }, - "additionalProperties": false - }, - "Field": { - "type": "object", - "description": "Row-level attribute for grouping, filtering, and metric expressions", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the field within the dataset" - }, - "expression": { - "$ref": "#/$defs/Expression" - }, - "dimension": { - "$ref": "#/$defs/Dimension" - }, - "label": { - "type": "string", - "description": "Label for categorization" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "expression"], - "additionalProperties": false - }, - "Dataset": { - "type": "object", - "description": "Logical dataset representing a business entity (fact or dimension table)", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the dataset" - }, - "source": { - "type": "string", - "description": "Reference to underlying physical table/view (database.schema.table) or query" - }, - "primary_key": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Primary key columns (single or composite)" - }, - "unique_keys": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "string" - } - }, - "description": "Array of unique key definitions (each can be single or composite)" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "fields": { - "type": "array", - "items": { - "$ref": "#/$defs/Field" - } - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "source"], - "additionalProperties": false - }, - "Relationship": { - "type": "object", - "description": "Foreign key relationship between datasets", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the relationship" - }, - "from": { - "type": "string", - "description": "Dataset on the many side of the relationship" - }, - "to": { - "type": "string", - "description": "Dataset on the one side of the relationship" - }, - "from_columns": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "description": "Foreign key columns in the 'from' dataset" - }, - "to_columns": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "description": "Primary/unique key columns in the 'to' dataset" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "from", "to", "from_columns", "to_columns"], - "additionalProperties": false - }, - "Metric": { - "type": "object", - "description": "Quantitative measure defined on business data", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the metric" - }, - "expression": { - "$ref": "#/$defs/Expression" - }, - "description": { - "type": "string", - "description": "Human-readable description of what the metric measures" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "expression"], - "additionalProperties": false - }, - "SemanticModel": { - "type": "object", - "description": "Top-level container representing a complete semantic model", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the semantic model" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "datasets": { - "type": "array", - "items": { - "$ref": "#/$defs/Dataset" - }, - "minItems": 1, - "description": "Collection of logical datasets" - }, - "relationships": { - "type": "array", - "items": { - "$ref": "#/$defs/Relationship" - }, - "description": "Defines how datasets are connected" - }, - "metrics": { - "type": "array", - "items": { - "$ref": "#/$defs/Metric" - }, - "description": "Quantifiable measures spanning datasets" - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "datasets"], - "additionalProperties": false - } - } -} diff --git a/converters/orionbelt/src/ossie_orionbelt/validation.py b/converters/orionbelt/src/ossie_orionbelt/validation.py index 738478c..64c9f51 100644 --- a/converters/orionbelt/src/ossie_orionbelt/validation.py +++ b/converters/orionbelt/src/ossie_orionbelt/validation.py @@ -13,20 +13,39 @@ _SCRIPT_DIR = Path(__file__).resolve().parent _SCHEMAS_DIR = _SCRIPT_DIR / "schemas" -# All three schemas are tracked files beside the converter, so the package is -# self-contained (no repo-root dependency, sdist/wheel build in isolation). The -# OBML schema is a vendored snapshot of the canonical repo-root -# schema/obml-schema.json, kept in sync by a drift-guard test. +# The OBML schema is OrionBelt's own format, always vendored beside the package +# and kept in sync with the repo-root schema/obml-schema.json by a drift-guard +# test. _OBML_SCHEMA_PATH = _SCHEMAS_DIR / "obml-schema.json" -_OSI_SCHEMA_PATH = _SCHEMAS_DIR / "osi-schema.json" -_OSI_ONTOLOGY_SCHEMA_PATH = _SCHEMAS_DIR / "osi-ontology-schema.json" -# The OSI ontology schema $refs the core-spec schema by its public raw URL for -# ``ai_context`` and the embedded ``semantic_model``. Resolve that URL against -# the vendored local copy so validation never touches the network. -_OSI_CORE_SPEC_RAW_URL = ( - "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json" -) + +def _osi_schema_path(filename: str) -> Path: + """Resolve an OSI core-spec schema without forcing a duplicate copy where a + canonical one already exists. + + Resolution order: + 1. A copy vendored beside this package (``schemas/``). The + standalone PyPI wheel and the OrionBelt product API rely on this, so the + package validates OSI documents self-contained and offline. + 2. Otherwise ``core-spec/`` in an enclosing Ossie monorepo + checkout, found by walking up from this file. This lets the in-tree + converter drop its vendored copy and link the single canonical schema + rather than duplicating it. + + Returns the vendored path unchanged when neither exists, so downstream + ``.exists()`` checks degrade to skip-with-warning rather than raising. + """ + vendored = _SCHEMAS_DIR / filename + if vendored.exists(): + return vendored + for parent in _SCRIPT_DIR.parents: + candidate = parent / "core-spec" / filename + if candidate.exists(): + return candidate + return vendored + + +_OSI_SCHEMA_PATH = _osi_schema_path("osi-schema.json") class ValidationResult: @@ -101,30 +120,6 @@ def _validate_json_schema( result.schema_errors.append(f"[{path}] {error.message}") -def _osi_core_registry() -> Any | None: - """Build a ``referencing.Registry`` that resolves the OSI core-spec schema - URL (referenced by the ontology schema) to the vendored local copy. Returns - ``None`` if the dependencies or the local core schema are unavailable, in - which case the caller falls back to default (network) resolution.""" - try: - from referencing import Registry, Resource - from referencing.jsonschema import DRAFT202012 - except ImportError: - return None - if not _OSI_SCHEMA_PATH.exists(): - return None - with open(_OSI_SCHEMA_PATH) as f: - core = json.load(f) - core_res = Resource.from_contents(core, default_specification=DRAFT202012) - # Register under both the raw URL used by the ontology schema's $refs and - # the core schema's own canonical $id (so its internal #/$defs refs resolve). - resources = [(_OSI_CORE_SPEC_RAW_URL, core_res)] - core_id = core_res.id() - if core_id: - resources.append((core_id, core_res)) - return Registry().with_resources(resources) - - # ── OBML Validation ────────────────────────────────────────────────────── @@ -262,30 +257,19 @@ def validate_osi(osi_dict: dict[str, Any], schema_path: Path | None = None) -> V return result -def validate_osi_ontology( - onto_dict: dict[str, Any], schema_path: Path | None = None -) -> ValidationResult: - """Validate an OSI ontology dict against JSON Schema and semantic rules. +def validate_osi_ontology(onto_dict: dict[str, Any]) -> ValidationResult: + """Validate an OSI ontology dict against semantic rules. - 1. **JSON Schema** — structural correctness against ``osi-ontology-schema.json`` - (Draft 2020-12). External ``$ref``s to the core-spec schema are resolved - against the vendored local copy via a ``referencing`` registry. - 2. **Unique concept names** across the ``ontology`` components. - 3. **Reference integrity** — relationship roles and concept_mappings + Ossie ships no OSI ontology schema (the converter emits ontology documents + but never consumes them), so JSON-Schema conformance is not checked here. + Runs the converter-owned semantic checks: + 1. **Unique concept names** across the ``ontology`` components. + 2. **Reference integrity** — relationship roles and concept_mappings reference concepts defined in the ontology. """ result = ValidationResult("OSI-ONTOLOGY") - # 1. JSON Schema validation (offline external-ref resolution). - _validate_json_schema( - onto_dict, - schema_path or _OSI_ONTOLOGY_SCHEMA_PATH, - result, - draft="draft2020", - registry=_osi_core_registry(), - ) - - # 2. Unique concept names + collect the defined set. + # 1. Unique concept names + collect the defined set. defined: set[str] = set() for comp in onto_dict.get("ontology", []): name = comp.get("concept", {}).get("name", "") @@ -293,7 +277,7 @@ def validate_osi_ontology( result.semantic_errors.append(f"[DUPLICATE_CONCEPT] Duplicate concept name '{name}'") defined.add(name) - # 3. Reference integrity — roles reference defined concepts. + # 2. Reference integrity — roles reference defined concepts. for comp in onto_dict.get("ontology", []): for rel in comp.get("relationships", []): rel_name = rel.get("name", "") diff --git a/converters/orionbelt/tests/test_osi_converter_ontology.py b/converters/orionbelt/tests/test_osi_converter_ontology.py index 8c2a6d7..7669387 100644 --- a/converters/orionbelt/tests/test_osi_converter_ontology.py +++ b/converters/orionbelt/tests/test_osi_converter_ontology.py @@ -1,9 +1,9 @@ """Tests for the OBML → OSI **ontology** converter (OBMLtoOSIOntology). -Validates that the derived ontology document conforms to the vendored -``osi-ontology-schema.json`` (with external core-spec refs resolved offline), -that OBML join cardinality maps to OSI multiplicity, and that the documented -gaps (many-to-many, composite keys, missing PK) surface as warnings. +Validates that the derived ontology document passes the converter's semantic +checks (unique concepts, reference integrity), that OBML join cardinality maps +to OSI multiplicity, and that the documented gaps (many-to-many, composite keys, +missing PK) surface as warnings. """ from __future__ import annotations @@ -105,11 +105,10 @@ def test_concept_mappings_bind_keys_and_fks(self) -> None: "expression": "ORDERS.CUSTOMER_ID", } - def test_validates_against_ontology_schema_offline(self) -> None: + def test_passes_ontology_semantic_validation(self) -> None: doc = conv.OBMLtoOSIOntology(_OBML, model_name="sales").convert() result = conv.validate_osi_ontology(doc) - assert result.valid, result.schema_errors + result.semantic_errors - assert not result.schema_errors + assert result.valid, result.semantic_errors assert not result.semantic_errors def test_one_to_one_multiplicity(self) -> None: diff --git a/converters/orionbelt/tests/test_osi_v02_compat.py b/converters/orionbelt/tests/test_osi_v02_compat.py index a6c7817..f5c110b 100644 --- a/converters/orionbelt/tests/test_osi_v02_compat.py +++ b/converters/orionbelt/tests/test_osi_v02_compat.py @@ -8,7 +8,7 @@ - Dataset ``unique_keys`` round-trips lossly via OBSL custom_extensions - Field ``label`` round-trips via OBSL custom_extensions - Legacy v0.1.1 inputs are normalized in place by the shim -- Every emitted document validates against the vendored v0.2 schema +- Every emitted document validates against the resolved v0.2 core schema """ from __future__ import annotations @@ -25,16 +25,13 @@ # Fixtures # --------------------------------------------------------------------------- -_SCHEMA_PATH = ( - Path(__file__).resolve().parents[1] / "src" / "ossie_orionbelt" / "schemas" / "osi-schema.json" -) - @pytest.fixture(scope="module") def schema_validator() -> Any: - """Draft 2020-12 validator pinned to the vendored OSI v0.2 schema.""" + """Draft 2020-12 validator pinned to the resolved OSI v0.2 core schema + (vendored copy if present, otherwise the ossie ``core-spec/`` copy).""" jsonschema = pytest.importorskip("jsonschema") - with open(_SCHEMA_PATH) as f: + with open(conv._OSI_SCHEMA_PATH) as f: schema = json.load(f) return jsonschema.Draft202012Validator(schema) From a84318afebfccc7a73ebb51e9addb77eaa262b44 Mon Sep 17 00:00:00 2001 From: Ralf Becher Date: Tue, 14 Jul 2026 19:24:36 +0200 Subject: [PATCH 65/89] orionbelt converter: bundle OSI core schema into the wheel from core-spec The vendored osi-schema.json copy was removed so the source tree carries no duplicate of the spec. But a pip-installed wheel has no core-spec/ alongside it, so validate_osi would silently skip schema validation. force-include the single canonical core-spec/osi-schema.json into the built wheel: no tracked duplicate, yet a pip install still validates OSI documents self-contained. In-tree runs still resolve it from core-spec/ via validation._osi_schema_path. --- converters/orionbelt/pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/converters/orionbelt/pyproject.toml b/converters/orionbelt/pyproject.toml index 474571a..d039657 100644 --- a/converters/orionbelt/pyproject.toml +++ b/converters/orionbelt/pyproject.toml @@ -36,6 +36,13 @@ ossie-orionbelt = "ossie_orionbelt.cli:main" [tool.hatch.build.targets.wheel] packages = ["src/ossie_orionbelt"] +# The OSI core schema is not tracked as a copy in this package (it lives once at +# the repo's core-spec/). Bundle it into the built wheel from that single source +# so a pip-installed converter still validates OSI documents self-contained. +# In-tree (dev/test) runs resolve it from core-spec/ via validation._osi_schema_path. +[tool.hatch.build.targets.wheel.force-include] +"../../core-spec/osi-schema.json" = "ossie_orionbelt/schemas/osi-schema.json" + # Only the OBML schema (OrionBelt's own format) is vendored under # src/ossie_orionbelt/schemas/. OSI core documents are validated against the # repo's core-spec/osi-schema.json, resolved at runtime by From 175491f70a687b4106e9fc63ac31c5c5fe57b1a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:41:44 +0200 Subject: [PATCH 66/89] Bump pydantic-settings from 2.14.1 to 2.14.2 in /converters/orionbelt (#202) --- converters/orionbelt/uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/converters/orionbelt/uv.lock b/converters/orionbelt/uv.lock index 3b9a719..d493c99 100644 --- a/converters/orionbelt/uv.lock +++ b/converters/orionbelt/uv.lock @@ -565,16 +565,16 @@ wheels = [ [[package]] name = "pydantic-settings" -version = "2.14.1" +version = "2.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "python-dotenv" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", size = 235551, upload-time = "2026-05-08T13:40:06.542Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/b5/8f48e906c3e0205276e8bd8cb7512217a87b2685304d64be27cad5b3019f/pydantic_settings-2.14.2.tar.gz", hash = "sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f", size = 237700, upload-time = "2026-06-19T13:44:56.324Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964, upload-time = "2026-05-08T13:40:04.958Z" }, + { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" }, ] [[package]] From 034abc002a94b2958f40c858ff04c58a6ee8e8e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:42:10 +0200 Subject: [PATCH 67/89] Bump com.fasterxml.jackson.core:jackson-databind in /converters/polaris (#201) --- converters/polaris/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/converters/polaris/pom.xml b/converters/polaris/pom.xml index 759cdb0..df9def7 100644 --- a/converters/polaris/pom.xml +++ b/converters/polaris/pom.xml @@ -44,7 +44,7 @@ 11 UTF-8 2.2 - 2.17.0 + 2.18.9 5.10.2 From 90df0cf2a8f76b5bb199c191244a9f89cb260efe Mon Sep 17 00:00:00 2001 From: Jensen Date: Wed, 15 Jul 2026 02:42:35 +0800 Subject: [PATCH 68/89] chore: ignore build artifacts (#199) --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 63e36ff..fdd3f16 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,8 @@ **/__pycache__/ **/.venv/ +**/.pytest_cache/ +**/.ruff_cache/ +**/.coverage +**/htmlcov/ +**/dist/ +**/target/ From 8e26cd7bbb7e983095fa6efea41a9c85ba67324d Mon Sep 17 00:00:00 2001 From: Emil Sadek Date: Tue, 14 Jul 2026 13:11:01 -0700 Subject: [PATCH 69/89] chore: fix comment indentation in spec.yaml (#203) Co-authored-by: Emil Sadek --- core-spec/spec.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index a5617e8..7229e08 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -198,7 +198,7 @@ fields: # Helps LLMs understand the field meaning and generate better queries ai_context: string - # Optional: Vendor-specific attributes for extensibility + # Optional: Vendor-specific attributes for extensibility custom_extensions: - vendor_name: string # Free-form string identifying the vendor data: string @@ -227,7 +227,7 @@ metrics: # Helps LLMs understand the metric meaning and suggest it appropriately ai_context: string - # Optional: Vendor-specific attributes for extensibility + # Optional: Vendor-specific attributes for extensibility custom_extensions: - vendor_name: string # Free-form string identifying the vendor data: string From 96e2a3b2e80f06f044bb36a4205aac967fdd142d Mon Sep 17 00:00:00 2001 From: Emil Sadek Date: Tue, 14 Jul 2026 13:12:09 -0700 Subject: [PATCH 70/89] chore: add .editorconfig for consistent formatting (#204) Co-authored-by: Emil Sadek --- .editorconfig | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..71498de --- /dev/null +++ b/.editorconfig @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +root = true + +[*] +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.{yml,yaml,json}] +indent_style = space +indent_size = 2 + +[*.{py,java,toml}] +indent_style = space +indent_size = 4 From 5bb40c395f4e022f4b27d846e9b091cf88f0653a Mon Sep 17 00:00:00 2001 From: Will Pugh <89424230+sfc-gh-wpugh@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:27:18 -0700 Subject: [PATCH 71/89] Expression language Doc (#150) * Test commit * Add the expression language spec * Extract embedded base64 image to img folder in core-spec Replace inline base64 PNG in expression_language.md with a relative file reference to core-spec/img/osi_layers.png, which renders correctly both locally and in the GitHub UI. * Update expression language status to Proposed Final * Remove links to old doc * Address reviewer comments * Shrink casting to supported core. * Addressed some feedback * Move to Ossie name * Update time --------- Co-authored-by: Will Pugh Co-authored-by: Cursor --- compliance/README.md | 1 + core-spec/expression_language.md | 761 +++++++++++++++++++++++++++++++ core-spec/img/ossie_layers.png | Bin 0 -> 7132 bytes impl/python/README.md | 1 + 4 files changed, 763 insertions(+) create mode 100644 compliance/README.md create mode 100644 core-spec/expression_language.md create mode 100644 core-spec/img/ossie_layers.png create mode 100644 impl/python/README.md diff --git a/compliance/README.md b/compliance/README.md new file mode 100644 index 0000000..7a59b04 --- /dev/null +++ b/compliance/README.md @@ -0,0 +1 @@ +This is for compliance tests diff --git a/core-spec/expression_language.md b/core-spec/expression_language.md new file mode 100644 index 0000000..8ec34ad --- /dev/null +++ b/core-spec/expression_language.md @@ -0,0 +1,761 @@ +# Ossie Proposal: Expression Language + +**Current Status:** Proposed Final + +**Working Group** + +| Lead(s) | Participants | +| :---- | :---- | +| Will Pugh, Snowflake Khushboo Bhatia, Snowflake | LLyod Tabb, Malloy Dianne Wood, Atscale Lior Ebel, Salesforce Quigley Malcolm, dbt Labs Kurt, Relational AI Justin Talbot, Databricks Pavel Tiunov, Cube Damian Waldron, Thoughtspot Oliver Laslett, Lightdash Martin Traverso, Starburst JB Onofré, The ASF Raul Beiroa, Denodo | + +## Overview + +![Ossie Layers](img/Ossie_layers.png) + +There are two layers in Ossie that need an expression language: + +* **Ontology layer.** This layer maps onto the ontology layer which sits above the logical layer. It maps more closely to modelling languages like OWL, [(Py)Rel](https://docs.relational.ai) from RelationalAI, and [Legend](https://legend.finos.org) from Goldman Sachs +* **Logical layer.** This layer maps directly to the databases and physical layer. It maps closely to traditional BI semantic models. + +This proposal is only targeted at the Logical Layer. It would be nice if the Ontological layer could re-use the same expression language, but that will be treated as a separate proposal. + +This document defines the SQL expression language subset that Ossie-compliant implementations MUST support. The goal is to provide a portable expression language that works across all Ossie implementations while allowing vendors to expose richer database-specific functionality through dialect extensions. In particular, it is meant for expressions at the logical layer. This means metrics, fields, filters, etc In the future, expressions such as arbitrary join expressions should also use this expression language. + +We expect there will be extensions to this language to cover concepts such as sub-queries, grain calculations, etc. However, these will each have their own proposal. + +### Design Principles + +1. **Portability**: Core functions work identically across all implementations +2. **Familiarity**: Based on widely-adopted SQL syntax and semantics +3. **Analytical Focus**: Prioritizes functions commonly used in BI and analytics +4. **Extensibility**: Vendor dialects can extend beyond the core + +### Changes to YAML + +1) Create a new dialect in the Ossie spec: Ossie\_SQL\_2026, which refers to this language specification. +2) Make Ossie\_SQL\_2026 the default dialect if one is not chosen. + +### Standards Reference + +The core language is based on **ANSI SQL:2003 Core** (ISO/IEC 9075-2:2003), selected for its: + +- Wide adoption across major databases (Snowflake, Databricks, PostgreSQL, BigQuery) +- Well-defined semantics +- Support for modern analytical features (window functions, CTEs) + +### Namespacing and Identifier Resolution + +The identifiers will match standard SQL identifiers: + +`Field: ` + +`FieldExpr: Field | Field ‘.’ Field` + +The Ossie spec currently contains three namespaces, which determine the visibility and uniqueness of each value. Where and how a field (or metric) is defined will determine the namespace for it, which in turn determines the ways it can be addressed by other fields. + +All identifiers MUST be valid names and follow ANSI SQL naming, with the size limitation of 128 characters for identifiers. Many databases support longer identifiers, however, this number is safe for a broad number of vendors. + +Regular identifiers (unquoted) should be case insensitive. For example, an identifier id is regular, so it would match with Id or iD. Comparing quoted and non-quoted identifiers is DB specific, so for best portability it is best to use simple identifiers. + +The quote character for the Ossie dialect will follow ANSI SQL and support the double quote character (“). This means that if an expression is in a field expression or as an identifier in the YAML, this will be the expected quoting. However, there are some databases that use other escape characters. Working with these have the option of either creating expressions using their dialect or having the Ossie document written in the Ossie dialect, but then having the SQL Interface queried in the local dialect. The SQL Interface will be defined in a different document. + +#### Comparison Table + +| You type this in SQL | Equivalen to | Will it match a column created as id? | +| :---- |:-------------|:--------------------------------------------------| +| id | ID | **Yes** (Standard behavior) | +| Id | ID | **Yes** (Standard behavior) | +| "ID" | ID | **Yes** (Force-matched to normalized case) | +| "id" | id | **No** (Quotes cause an exact match to lowercase) | + +Sometimes, we may refer to a **normalized identifier**. This is a form the identifiers can be put in, so they can be matched easily and matches can be made with case-sensitive, exact matching. For **normalized identifiers**: + +* Regular identifiers are upper cased +* Quoted identifiers have their quotes stripped and any escaped characters are unescaped + +#### Name Spaces + +Namespaces define how an identifier is looked up in an expression. They are covered in the semantics document. Identifiers +will be able to be multi-part and the parts will be separated by the '.' characters, E.g. `dataset.field` This matches SQL conventions. + +## SQL Language Subset + +### Supported SQL Constructs + +Ossie expressions support the following SQL constructs within any expression: + +| Construct | Notes | +| :---- |:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Column and Metric references | Varies based on whether in Ontology or Semantic models. See namespaceing in [OSI Discussion Point: Core Analytic Abstractions](https://docs.google.com/document/d/1si8DqU4arG18ZgX4HnRG5D_zS2X7V1s-vgNY35rvxhM/edit?tab=t.0#heading=h.le505t8uoyfy) And future Ontology documentation | +| Arithmetic operators | `+`, `-`, `*`, `/`, `%` (modulo) | +| Comparison operators | `=`, `<>`, `!=`, `<`, `>`, `<=`, `>=` | +| Logical operators | `AND`, `OR`, `NOT` | +| `BETWEEN` | `x BETWEEN a AND b` | +| `IN` / `NOT IN` | `x IN (a, b, c)` This only supports lists of values, not subqueries. | +| `LIKE` / `ILIKE` | Pattern matching | +| `IS NULL` / `IS NOT NULL` | Null checks | +| `CASE WHEN` | Conditional logic | +| Aggregate functions | Core functions used for aggregations | +| Window functions | Core supported window functions | +| Scalar functions | See function categories below | +| Parentheses | Expression grouping | + +### + +### Not Supported in Expressions + +| Construct | Reason | +| :---- | :---- | +| `SELECT` / `FROM` / `JOIN` | Handled by semantic layer | +| `GROUP BY` | Controlled by grain specification | +| `WHERE` | Use filter property instead | +| Subqueries | Use field references instead, or EXISTS\_IN() for filtering based on a subquery. | +| CTEs | Use field references instead | +| `UNION` / `INTERSECT` / `EXCEPT` | Not applicable to expressions | +| DDL statements | Out of scope | +| DML statements | Out of scope | + +### Operator Precedence + +Standard SQL operator precedence applies (highest to lowest): + +1. Parentheses `()` +2. Unary operators: `+`, `-`, `NOT` +3. Multiplication/Division: `*`, `/`, `%` +4. Addition/Subtraction: `+`, `-` +5. Comparison: `=`, `<>`, `<`, `>`, `<=`, `>=`, `LIKE`, `IN`, `BETWEEN`, `IS NULL` +6`AND` +7`OR` + +--- + +## Aggregation Functions + +### Core Aggregation Functions (REQUIRED) + +| Function | Syntax | Description | Decomposability | +| :---- | :---- | :---- | :---- | +| `SUM` | `SUM(expr)` | Sum of values | Distributive | +| `COUNT` | `COUNT(expr)` | Count of non-null values | Distributive | +| `COUNT(*)` | `COUNT(*)` | Count of all rows | Distributive | +| `COUNT(DISTINCT expr)` | `COUNT(DISTINCT expr)` | Count of distinct values | Holistic | +| `AVG` | `AVG(expr)` | Arithmetic mean | Algebraic | +| `MIN` | `MIN(expr)` | Minimum value | Distributive | +| `MAX` | `MAX(expr)` | Maximum value | Distributive | + +### Statistical Aggregations (REQUIRED) + +| Function | Syntax | Description | Decomposability | +| :---- | :---- | :---- | :---- | +| `STDDEV` | `STDDEV(expr)` | Sample standard deviation | Algebraic | +| `STDDEV_POP` | `STDDEV_POP(expr)` | Population standard deviation | Algebraic | +| `STDDEV_SAMP` | `STDDEV_SAMP(expr)` | Sample standard deviation (alias for STDDEV) | Algebraic | +| `VARIANCE` | `VARIANCE(expr)` | Sample variance | Algebraic | +| `VAR_POP` | `VAR_POP(expr)` | Population variance | Algebraic | +| `VAR_SAMP` | `VAR_SAMP(expr)` | Sample variance (alias for VARIANCE) | Algebraic | + +### Percentile Functions (REQUIRED) + +| Function | Syntax | Description | Decomposability | +| :---- | :---- | :---- | :---- | +| `MEDIAN` | `MEDIAN(expr)` | Median value (50th percentile) | Holistic | +| `PERCENTILE_CONT` | `PERCENTILE_CONT(p) WITHIN GROUP (ORDER BY expr)` | Continuous percentile (interpolated) | Holistic | +| `PERCENTILE_DISC` | `PERCENTILE_DISC(p) WITHIN GROUP (ORDER BY expr)` | Discrete percentile (actual value) | Holistic | + +Where `p` is a value between 0 and 1 (e.g., 0.5 for median, 0.75 for 75th percentile). + +### Approximate Aggregations (RECOMMENDED) + +Approximate functions trade exact accuracy for significantly better performance on large datasets. They use probabilistic algorithms (sketches) that are efficiently mergeable, making them well-suited for distributed computation. + +| Function | Syntax | Description | Typical Error | +| :---- | :---- | :---- | :---- | +| `APPROX_COUNT_DISTINCT` | `APPROX_COUNT_DISTINCT(expr)` | Approximate distinct count using HyperLogLog or something similar. Actual method is up to providers. | \~2% | +| `APPROX_PERCENTILE` | `APPROX_PERCENTILE(expr, p)` | Approximate percentile using t-digest or similar | \~1% | + +```sql +-- Approximate distinct count (much faster than COUNT(DISTINCT) on large data) +APPROX_COUNT_DISTINCT(customer_id) + +-- Approximate median +APPROX_PERCENTILE(amount, 0.5) + +-- Approximate 95th percentile +APPROX_PERCENTILE(response_time, 0.95) +``` + +**Database Support:** + +| Function | Snowflake | BigQuery | Databricks | PostgreSQL | +| :---- | :---- | :---- | :---- | :---- | +| `APPROX_COUNT_DISTINCT` | ✅ | ✅ | ✅ | ❌ (extension) | +| `APPROX_PERCENTILE` | ✅ | ✅ `APPROX_QUANTILES` | ✅ | ❌ | + +**Note**: BigQuery uses `APPROX_QUANTILES(expr, num_buckets)` which returns an array. To get a specific percentile: `APPROX_QUANTILES(amount, 100)[OFFSET(50)]` for median. + +--- + +### Conditional Aggregations (REQUIRED) + +SUM / COUNT aggregation functions support `DISTINCT.` +All aggregations should support filtered aggregation: + +```sql +-- DISTINCT modifier +SUM(DISTINCT amount) +COUNT(DISTINCT customer_id) + +-- Filtered aggregation via CASE +SUM(CASE WHEN status = 'completed' THEN amount ELSE 0 END) +COUNT(CASE WHEN status = 'completed' THEN 1 END) +``` + +### Decomposability Reference + +For multi-stage aggregation (see [Ossie Analytical Context Extension](https://docs.google.com/document/d/1MKNySGmEv_C6CzBZ7um9Ym3_mMvmOolpDuwPvRzQ1bo/edit?usp=sharing)): + +| Category | Functions | +| :---- | :---- | +| **Distributive** | SUM, COUNT, MIN, MAX | +| **Algebraic** | AVG, STDDEV, VARIANCE | +| **Holistic** | MEDIAN, PERCENTILE, COUNT DISTINCT | +| **Sketch-based** | APPROX\_COUNT\_DISTINCT, APPROX\_PERCENTILE | + +--- + +## Date/Time Functions + +### Current Date/Time (REQUIRED) + +| Function | Syntax | Returns | Description | +| :---- | :---- | :---- | :---- | +| `CURRENT_DATE` | `CURRENT_DATE` or `CURRENT_DATE()` | DATE | Current date | +| `CURRENT_TIMESTAMP` | `CURRENT_TIMESTAMP` or `CURRENT_TIMESTAMP()` | TIMESTAMP | Current timestamp | +| `CURRENT_TIME` | `CURRENT_TIME` or `CURRENT_TIME()` | TIME | Current time | + +### Date/Time Extraction (REQUIRED) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `YEAR` | `YEAR(date_expr)` | Extract year (integer) | +| `QUARTER` | `QUARTER(date_expr)` | Extract quarter (1-4) | +| `MONTH` | `MONTH(date_expr)` | Extract month (1-12) | +| `DAY` | `DAY(date_expr)` | Extract day of month (1-31) | +| `DAYOFYEAR` | `DAYOFYEAR(date_expr)` | Day of year (1-366) | +| `HOUR` | `HOUR(timestamp_expr)` | Extract hour (0-23) | +| `MINUTE` | `MINUTE(timestamp_expr)` | Extract minute (0-59) | +| `SECOND` | `SECOND(timestamp_expr)` | Extract second (0-59) | + +### Alternative Extraction Syntax (REQUIRED) + +```sql +-- EXTRACT function (SQL standard) +EXTRACT(YEAR FROM date_expr) +EXTRACT(MONTH FROM date_expr) +EXTRACT(DAY FROM date_expr) + +-- DATE_PART function (common alternative) +DATE_PART('year', date_expr) +DATE_PART('month', date_expr) +DATE_PART('day', date_expr) +``` + +Supported date parts for `EXTRACT` and `DATE_PART`: + +- `YEAR`, `QUARTER`, `MONTH`, `WEEK`, `DAY` +- `DAYOFWEEK`, `DAYOFYEAR` +- `HOUR`, `MINUTE`, `SECOND`, `MILLISECOND` + +### Date Truncation (REQUIRED) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `DATE_TRUNC` | `DATE_TRUNC(part, date_expr)` | Truncate to specified precision | + +Supported parts: `'year'`, `'quarter'`, `'month'`, `'week'`, `'day'`, `'hour'`, `'minute'`, `'second'` + +```sql +-- Examples +DATE_TRUNC('month', order_date) -- First day of month +DATE_TRUNC('quarter', order_date) -- First day of quarter +DATE_TRUNC('week', order_date) -- First day of week (Monday) +``` + +### Date Arithmetic (REQUIRED) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `DATEADD` | `DATEADD(part, amount, date_expr)` | Add interval to date | +| `DATEDIFF` | `DATEDIFF(part, start_date, end_date)` | Difference between dates | + +```sql +-- Add/subtract intervals +DATEADD(day, 7, order_date) -- Add 7 days +DATEADD(month, -1, order_date) -- Subtract 1 month +DATEADD(year, 1, order_date) -- Add 1 year + +-- Calculate differences +DATEDIFF(day, start_date, end_date) -- Days between dates +DATEDIFF(month, start_date, end_date) -- Months between dates +DATEDIFF(year, start_date, end_date) -- Years between dates +``` + +### Date/Time Construction (REQUIRED) + +Construct DATE, TIME, and TIMESTAMP values using ANSI typed literals or `CAST`. +ISO-8601 strings (`YYYY-MM-DD`, `YYYY-MM-DD HH:MI:SS`, `HH:MI:SS`) require no format +model and behave identically across engines, making them the portable default. + +| Form | Syntax | Description | +| :---- | :---- | :---- | +| Typed literal | `DATE '2024-01-15'` | Construct a DATE | +| Typed literal | `TIMESTAMP_NTZ '2024-01-15 10:30:00'` | Construct a wall-clock timestamp (no time zone) | +| Typed literal | `TIME '10:30:00'` | Construct a TIME | +| Cast | `CAST('2024-01-15' AS DATE)` | Parse ISO string to DATE | +| Cast | `CAST('2024-01-15 10:30:00' AS TIMESTAMP_NTZ)` | Parse ISO string to timestamp | +| Cast | `CAST('10:30:00' AS TIME)` | Parse ISO string to TIME | +| `TO_DATE` | `TO_DATE(string)` | Parse ISO string to DATE | +| `TO_TIMESTAMP` | `TO_TIMESTAMP(string)` | Parse ISO string to timestamp | + +### Date/Time Construction from Format Strings (EXPERIMENTAL) + +Parsing with an explicit format string relies on a datetime format model whose token +vocabulary differs across engines (Oracle/`TO_CHAR`-style, `strftime` `%`-codes, and +Java/LDML patterns are all in use). + +For portability, we are looking to restrict the `format` argument to +the portable core format tokens defined in Date Formatting below, and prefer the +single-argument, ISO-8601 forms above where possible. + +**Since, this differs so widely across databases, consider this experimental for now.** + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `TO_DATE` | `TO_DATE(string, format)` | Parse string to date | +| `TO_TIMESTAMP` | `TO_TIMESTAMP(string, format)` | Parse string to timestamp | + +### Date Formatting (EXPERIMENTAL) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `TO_CHAR` | `TO_CHAR(date_expr, format)` | Format date as string | + +Ossie defines a portable core of format tokens: the tokens that can be expressed in +every major engine's datetime format model. + +This feature is experimental. Implementations choosing to support these should support the following tokens for the +`format` argument of `TO_CHAR`. The `strftime` and Java/LDML columns below are informative, +provided to aid translation. + +| Token | Meaning | `strftime` (C / Python / BigQuery) | Java/LDML (Spark, .NET) | +| :---- | :---- | :---- | :---- | +| `YYYY` | 4-digit year | `%Y` | `yyyy` | +| `YY` | 2-digit year | `%y` | `yy` | +| `MM` | Month (01-12) | `%m` | `MM` | +| `MON` | Abbreviated month name | `%b` | `MMM` | +| `MONTH` | Full month name | `%B` | `MMMM` | +| `DD` | Day of month (01-31) | `%d` | `dd` | +| `DY` | Abbreviated day name | `%a` | `EEE` | +| `DAY` | Full day name | `%A` | `EEEE` | +| `HH24` | Hour (00-23) | `%H` | `HH` | +| `HH12` (`HH`) | Hour (01-12) | `%I` | `hh` | +| `MI` | Minute (00-59) | `%M` | `mm` | +| `SS` | Second (00-59) | `%S` | `ss` | +| `AM` / `PM` | Meridiem indicator | `%p` | `a` | + +**Locale-dependent output.** The name tokens (`MON`, `MONTH`, `DY`, `DAY`, `AM`/`PM`) +render text whose language is governed by engine/session locale settings; the spelling +is not guaranteed identical across engines. + +**Fractional seconds** are available everywhere but the token and precision differ +(Oracle `FF1`–`FF9`, `strftime` `%f`, Java `S`…`SSSSSS`); treat sub-second formatting as +a dialect extension. +--- + +## String Functions + +### String Manipulation (REQUIRED) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `CONCAT` | `CONCAT(str1, str2, ...)` | Concatenate strings | +| `||` | `str1 || str2` | Concatenation operator | +| `LENGTH` | `LENGTH(str)` | String length in characters | +| `LOWER` | `LOWER(str)` | Convert to lowercase | +| `UPPER` | `UPPER(str)` | Convert to uppercase | +| `TRIM` | `TRIM(str)` | Remove leading/trailing whitespace | +| `LTRIM` | `LTRIM(str)` | Remove leading whitespace | +| `RTRIM` | `RTRIM(str)` | Remove trailing whitespace | +| `LEFT` | `LEFT(str, n)` | First n characters | +| `RIGHT` | `RIGHT(str, n)` | Last n characters | +| `SUBSTRING` | `SUBSTRING(str, start, length)` | Extract substring | +| `REPLACE` | `REPLACE(str, from, to)` | Replace occurrences | +| `SPLIT_PART` | `SPLIT_PART(str, delimiter, part)` | Extract part by delimiter | + +### String Search (REQUIRED) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `POSITION` | `POSITION(substr IN str)` | Position of substring (1-based) | +| `CHARINDEX` | `CHARINDEX(substr, str)` | Alias for POSITION | +| `CONTAINS` | `CONTAINS(str, substr)` | Returns TRUE if contains | +| `STARTSWITH` | `STARTSWITH(str, prefix)` | Returns TRUE if starts with | +| `ENDSWITH` | `ENDSWITH(str, suffix)` | Returns TRUE if ends with | + +### Pattern Matching (REQUIRED) + +| Pattern | Syntax | Description | +| :---- | :---- | :---- | +| `LIKE` | `str LIKE pattern` | Case-sensitive pattern match | +| `ILIKE` | `str ILIKE pattern` | Case-insensitive pattern match | +| `REGEXP_LIKE` | `REGEXP_LIKE(str, pattern)` | Regular expression match | + +Pattern wildcards for `LIKE`: + +- `%` \- Match any sequence of characters +- `_` \- Match any single character + +### Regular Expressions (RECOMMENDED) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `REGEXP_EXTRACT` | `REGEXP_EXTRACT(str, pattern)` | Extract first match | +| `REGEXP_REPLACE` | `REGEXP_REPLACE(str, pattern, replacement)` | Replace matches | +| `REGEXP_COUNT` | `REGEXP_COUNT(str, pattern)` | Count matches | + +--- + +## Mathematical Functions + +### Basic Math (REQUIRED) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `ABS` | `ABS(x)` | Absolute value | +| `ROUND` | `ROUND(x, d)` | Round to d decimal places | +| `FLOOR` | `FLOOR(x)` | Round down to integer | +| `CEIL` / `CEILING` | `CEIL(x)` | Round up to integer | +| `TRUNC` / `TRUNCATE` | `TRUNC(x, d)` | Truncate to d decimal places | +| `MOD` | `MOD(x, y)` | Modulo (remainder) | +| `SIGN` | `SIGN(x)` | Sign (-1, 0, or 1\) | + +### Advanced Math (REQUIRED) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `POWER` | `POWER(x, y)` | x raised to power y | +| `SQRT` | `SQRT(x)` | Square root | +| `EXP` | `EXP(x)` | e raised to power x | +| `LN` | `LN(x)` | Natural logarithm | +| `LOG` | `LOG(base, x)` | Logarithm with specified base | +| `LOG10` | `LOG10(x)` | Base-10 logarithm | + +### Trigonometric (RECOMMENDED) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `SIN` | `SIN(x)` | Sine (x in radians) | +| `COS` | `COS(x)` | Cosine | +| `TAN` | `TAN(x)` | Tangent | +| `ASIN` | `ASIN(x)` | Arc sine | +| `ACOS` | `ACOS(x)` | Arc cosine | +| `ATAN` | `ATAN(x)` | Arc tangent | +| `ATAN2` | `ATAN2(y, x)` | Arc tangent of y/x | +| `RADIANS` | `RADIANS(degrees)` | Convert degrees to radians | +| `DEGREES` | `DEGREES(radians)` | Convert radians to degrees | +| `PI` | `PI()` | Value of π | + +### Comparison Functions (REQUIRED) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `GREATEST` | `GREATEST(x, y, ...)` | Maximum of arguments | +| `LEAST` | `LEAST(x, y, ...)` | Minimum of arguments | + +--- + +## Conditional Functions + +### CASE Expression (REQUIRED) + +```sql +-- Searched CASE +CASE + WHEN condition1 THEN result1 + WHEN condition2 THEN result2 + ELSE default_result +END + +-- Simple CASE +CASE expression + WHEN value1 THEN result1 + WHEN value2 THEN result2 + ELSE default_result +END +``` + +### Conditional Functions (REQUIRED) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `IF` | `IF(condition, true_result, false_result)` | Ternary conditional | +| `IFF` | `IFF(condition, true_result, false_result)` | Alias for IF | +| `NULLIF` | `NULLIF(expr1, expr2)` | Returns NULL if equal | +| `COALESCE` | `COALESCE(expr1, expr2, ...)` | First non-null value | +| `IFNULL` | `IFNULL(expr, default)` | Alias for COALESCE with 2 args | +| `NVL` | `NVL(expr, default)` | Alias for COALESCE with 2 args | +| `NVL2` | `NVL2(expr, not_null_result, null_result)` | Different results for null/not-null | +| `ZeroIfNull` | `ZEROIFNULL(expr)` | Returns 0 if null | +| `NullIfZero` | `NULLIFZERO(expr)` | Returns NULL if zero | + +### Boolean Functions (REQUIRED) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `BOOLEAN` | `TRUE`, `FALSE` | Boolean literals | +| `NOT` | `NOT expr` | Logical negation | +| `AND` | `expr1 AND expr2` | Logical AND | +| `OR` | `expr1 OR expr2` | Logical OR | + +--- + +## Window Functions + +Window functions operate over a window frame defined by `OVER()`. This should act consistently with window functions in ANSI SQL. +When, adding the query interface, window functions will be subject to where they are allowed. + +### Syntax + +```sql +function_name(args) OVER ( + [PARTITION BY partition_expr, ...] + [ORDER BY order_expr [ASC|DESC], ...] + [frame_clause] +) +``` + +Frame clause options: + +- `ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW` +- `ROWS BETWEEN n PRECEDING AND n FOLLOWING` +- `RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW` + +### Ranking Functions (REQUIRED) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `ROW_NUMBER` | `ROW_NUMBER() OVER (...)` | Sequential row number | +| `RANK` | `RANK() OVER (...)` | Rank with gaps for ties | +| `DENSE_RANK` | `DENSE_RANK() OVER (...)` | Rank without gaps | +| `NTILE` | `NTILE(n) OVER (...)` | Divide into n buckets | +| `PERCENT_RANK` | `PERCENT_RANK() OVER (...)` | Relative rank (0-1) | +| `CUME_DIST` | `CUME_DIST() OVER (...)` | Cumulative distribution | + +### Offset Functions (REQUIRED) + +| Function | Syntax | Description | +| :---- | :---- | :---- | +| `LAG` | `LAG(expr, offset, default) OVER (...)` | Value from previous row | +| `LEAD` | `LEAD(expr, offset, default) OVER (...)` | Value from next row | +| `FIRST_VALUE` | `FIRST_VALUE(expr) OVER (...)` | First value in window | +| `LAST_VALUE` | `LAST_VALUE(expr) OVER (...)` | Last value in window | +| `NTH_VALUE` | `NTH_VALUE(expr, n) OVER (...)` | Nth value in window | + +### Window Aggregations (REQUIRED) + +All standard aggregation functions can be used as window functions: + +```sql +-- Running total +SUM(amount) OVER (ORDER BY order_date) + +-- Running average +AVG(amount) OVER (ORDER BY order_date ROWS BETWEEN 6 PRECEDING AND CURRENT ROW) + +-- Partition totals +SUM(amount) OVER (PARTITION BY region) + +-- Percent of total +amount / SUM(amount) OVER () * 100 +``` + +--- + +## Type Conversion Functions + +### CAST (REQUIRED) + +```sql +CAST(expression AS target_type) +``` + +Supported target types: + +- `VARCHAR` / `STRING` \- Character string +- `INTEGER` / `INT` / `BIGINT` \- Integer +- `DECIMAL` / `NUMERIC` \- Fixed-point decimal +- `FLOAT` / `DOUBLE` \- Floating-point +- `BOOLEAN` \- Boolean +- `DATE` \- Date +- `TIMESTAMP` \- Timestamp +- `TIME` \- Time + +### TRY\_CAST (RECOMMENDED) + +```sql +TRY_CAST(expression AS target_type) -- Returns NULL on failure +``` +--- + +### Null-Safe Comparison + +```sql +-- Standard comparison (returns NULL if either side is NULL) +a = b + +-- Null-safe comparison (treats NULLs as equal) +a IS NOT DISTINCT FROM b -- TRUE if both are NULL +a IS DISTINCT FROM b -- TRUE if one is NULL and other isn't +``` + +--- + +## Dialect Extensions + +Ossie implementations MAY support additional functions through dialect-specific extensions. When using dialect extensions, the expression must specify the dialect. + +The Ossie dialect should always be supported. Other dialects MAY be ignored. There is no guarantee that all different dialects for an expression will act the same, so implementations should be consistent with their dialect handling. This means that if an Ossie model has an expression written in two dialects, the implementation should deterministically choose which dialect to use. + +### Declaring Dialect-Specific Expressions + +``` +expression: + dialects: + - dialect: ANSI_SQL + expression: DATE_TRUNC('month', order_date) + - dialect: SNOWFLAKE + expression: DATE_TRUNC('month', order_date) + - dialect: BIGQUERY + expression: DATE_TRUNC(order_date, MONTH) +``` + +### Common Dialect Variations + +| Function | ANSI\_SQL | Snowflake | BigQuery | Databricks | PostgreSQL | +| :---- | :---- | :---- | :---- | :---- | :---- | +| Date truncation | `DATE_TRUNC('month', d)` | `DATE_TRUNC('month', d)` | `DATE_TRUNC(d, MONTH)` | `DATE_TRUNC('month', d)` | `DATE_TRUNC('month', d)` | +| Date add | `DATEADD(day, 7, d)` | `DATEADD(day, 7, d)` | `DATE_ADD(d, INTERVAL 7 DAY)` | `DATE_ADD(d, 7)` | `d + INTERVAL '7 days'` | +| String concat | `CONCAT(a, b)` | `CONCAT(a, b)` | `CONCAT(a, b)` | `CONCAT(a, b)` | `a || b` | +| Null coalesce | `COALESCE(a, b)` | `COALESCE(a, b)` or `NVL(a, b)` | `COALESCE(a, b)` or `IFNULL(a, b)` | `COALESCE(a, b)` | `COALESCE(a, b)` | +| Current timestamp | `CURRENT_TIMESTAMP` | `CURRENT_TIMESTAMP()` | `CURRENT_TIMESTAMP()` | `CURRENT_TIMESTAMP()` | `CURRENT_TIMESTAMP` | +| Substring | `SUBSTRING(s, start, len)` | `SUBSTR(s, start, len)` | `SUBSTR(s, start, len)` | `SUBSTRING(s, start, len)` | `SUBSTRING(s, start, len)` | + +### + +### Dialect-Specific Extensions + +Vendors may expose their own feature through extensions, however the default for Ossie should be to pass unknown values through.: +--- + +## Cross-Reference: Tool Mappings + +This section maps Ossie standard functions to their equivalents in popular BI tools. + +### Aggregation Function Mapping + +| Ossie Standard | Tableau | Looker Studio | DAX | +| :---- | :---- | :---- | :---- | +| `SUM(x)` | `SUM(x)` | `SUM(X)` | `SUM(x)` | +| `COUNT(x)` | `COUNT(x)` | `COUNT(X)` | `COUNT(x)` | +| `COUNT(DISTINCT x)` | `COUNTD(x)` | `COUNT_DISTINCT(X)` | `DISTINCTCOUNT(x)` | +| `AVG(x)` | `AVG(x)` | `AVG(X)` | `AVERAGE(x)` | +| `MIN(x)` | `MIN(x)` | `MIN(X)` | `MIN(x)` | +| `MAX(x)` | `MAX(x)` | `MAX(X)` | `MAX(x)` | +| `STDDEV(x)` | `STDEV(x)` | `STDDEV(X)` | `STDEV.S(x)` | +| `STDDEV_POP(x)` | `STDEVP(x)` | `STDDEV(X)` | `STDEV.P(x)` | +| `VARIANCE(x)` | `VAR(x)` | `VARIANCE(X)` | `VAR.S(x)` | +| `MEDIAN(x)` | `MEDIAN(x)` | `MEDIAN(X)` | `MEDIAN(x)` | +| `PERCENTILE_CONT(x, 0.75)` | `PERCENTILE(x, 0.75)` | `PERCENTILE(X, 75)` | `PERCENTILE.INC(x, 0.75)` | + +### Date Function Mapping + +| Ossie Standard | Tableau | Looker Studio | DAX | +| :---- | :---- | :---- | :---- | +| `YEAR(d)` | `YEAR(d)` | `YEAR(Date)` | `YEAR(d)` | +| `MONTH(d)` | `MONTH(d)` | `MONTH(Date)` | `MONTH(d)` | +| `DAY(d)` | `DAY(d)` | `DAY(Date)` | `DAY(d)` | +| `DATE_TRUNC('month', d)` | `DATETRUNC('month', d)` | `TODATE(d, "YYYYMM01", "YYYYMMDD")` | `DATE(YEAR(d), MONTH(d), 1)` | +| `DATEADD(day, n, d)` | `DATEADD('day', n, d)` | `DATE_ADD(d, n)` (days only) | `DATE(d) + n` or `DATEADD(d, n, DAY)` | +| `DATEDIFF(day, d1, d2)` | `DATEDIFF('day', d1, d2)` | `DATE_DIFF(d1, d2)` | `DATEDIFF(d1, d2, DAY)` | +| `CURRENT_DATE` | `TODAY()` | `TODAY()` | `TODAY()` | + +### String Function Mapping + +| Ossie Standard | Tableau | Looker Studio | DAX | +| :---- | :---- | :---- | :---- | +| `CONCAT(a, b)` | `a + b` | `CONCAT(X, Y)` | `CONCATENATE(a, b)` or `a & b` | +| `LENGTH(s)` | `LEN(s)` | `LENGTH(X)` | `LEN(s)` | +| `LOWER(s)` | `LOWER(s)` | `LOWER(X)` | `LOWER(s)` | +| `UPPER(s)` | `UPPER(s)` | `UPPER(X)` | `UPPER(s)` | +| `TRIM(s)` | `TRIM(s)` | `TRIM(X)` | `TRIM(s)` | +| `LEFT(s, n)` | `LEFT(s, n)` | `LEFT_TEXT(X, n)` | `LEFT(s, n)` | +| `RIGHT(s, n)` | `RIGHT(s, n)` | `RIGHT_TEXT(X, n)` | `RIGHT(s, n)` | +| `SUBSTRING(s, start, len)` | `MID(s, start, len)` | `SUBSTR(X, start, len)` | `MID(s, start, len)` | +| `REPLACE(s, from, to)` | `REPLACE(s, from, to)` | `REPLACE(X, Y, Z)` | `SUBSTITUTE(s, from, to)` | +| `CONTAINS(s, sub)` | `CONTAINS(s, sub)` | `CONTAINS_TEXT(X, text)` | `CONTAINSSTRING(s, sub)` | + +### Conditional Function Mapping + +| Ossie Standard | Tableau | Looker Studio | DAX | +| :---- | :---- | :---- | :---- | +| `CASE WHEN...` | `CASE WHEN...` or `IF...` | `CASE WHEN...` | `SWITCH(TRUE(), ...)` | +| `IF(cond, t, f)` | `IF cond THEN t ELSE f END` | N/A (use CASE) | `IF(cond, t, f)` | +| `COALESCE(a, b)` | `IFNULL(a, b)` or `ZN(a)` | `COALESCE(...)` | `COALESCE(a, b)` | +| `NULLIF(a, b)` | `IF a = b THEN NULL ELSE a END` | N/A | `IF(a = b, BLANK(), a)` | + +### Window Function Mapping + +| Ossie Standard | Tableau | Looker Studio | DAX | +| :---- | :---- | :---- | :---- | +| `ROW_NUMBER() OVER(...)` | `INDEX()` | N/A | `RANKX(...)` with DENSE | +| `RANK() OVER(...)` | `RANK(expr)` | N/A | `RANKX(...)` | +| `SUM(...) OVER(PARTITION BY...)` | `{FIXED [...]: SUM(...)}` | N/A (blending only) | Context-dependent | +| `LAG(x, 1) OVER(ORDER BY...)` | `LOOKUP(x, -1)` | N/A | `CALCULATE(x, PREVIOUSDAY(...))` | +| `RUNNING_SUM(...)` | `RUNNING_SUM(SUM(...))` | N/A | `CALCULATE(SUM(...), FILTER(...))` | + +--- + +## Compliance Levels + +### MUST Support (Core) + +Implementations MUST support all functions marked as **REQUIRED** in this specification. These represent the minimum portable expression language. + +### SHOULD Support (Recommended) + +Implementations SHOULD support functions marked as **RECOMMENDED**. These are common analytical functions that may not be available in all databases. + +### MAY Support (Extensions) + +Implementations MAY support additional functions through dialect extensions. These should be documented as dialect-specific. + +--- + +## Version History + +| Version | Date | Changes | +|:----------|:-----------| :---- | +| 0.2.0.dev | 2026-07-15 | Initial draft | + +--- + +## References + +- [SQL:2003 Standard](https://www.iso.org/standard/34132.html) (ISO/IEC 9075-2:2003) +- [Tableau Functions Reference](https://help.tableau.com/current/pro/desktop/en-us/functions.htm) +- [Looker Studio Function List](https://support.google.com/looker-studio/table/6379764) +- [DAX Function Reference](https://learn.microsoft.com/en-us/dax/dax-function-reference) +- [Snowflake SQL Reference](https://docs.snowflake.com/en/sql-reference-functions) +- [BigQuery Standard SQL Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-and-operators) +- [Databricks SQL Functions](https://docs.databricks.com/sql/language-manual/sql-ref-functions.html) +- [PostgreSQL Functions](https://www.postgresql.org/docs/current/functions.html) diff --git a/core-spec/img/ossie_layers.png b/core-spec/img/ossie_layers.png new file mode 100644 index 0000000000000000000000000000000000000000..7887c9f93d75f2435cf062e5447abe462fa517f7 GIT binary patch literal 7132 zcmb_>cTf{fw|0IuR0KqjB0^M@BHhqiL{OxLj+D@(NH3v<5<~@55D*ZMl1K@WE>-D8 ziqz1NUL_EuCLs_K^6}37=FZ$d?%es_`|Zrm?mlOpvvYQ4=R9-HI|F@9rpq@j0{{Rf zt*07B0Kgd(0PxSbi)T*}i->Z;)8*Flr$%}JK!^YU@bV1+Kskk8t^)u8k^lhS4ggS0 z2LL#Ivg!?$PYp8$x+ah5bUGf7Pfkv@u&@{z896>a*3i&6HHwRii;3O+_|ZQv@0Gf` zOj}zfmD+oF*cB1s+|kiNBoZqt6R_9SX8qXKNJZ&^Ku2K&eXihmDlgL8xF#z=Ls4|3_N|w# zCOX00LQetTo1PqD%f-&J(DsoyisWQx0p4Kaxyfg4WoPf~A#`8VA0C;8{F;)9)`gh3 z`vir(datJOR9a3kG40Fy#0+L;mN)N{2C}fLr;+^Qprfe)czd4r>FME;&r=H+0Kk>` zulbKw!0VJ#kilO|?=b_OfrW+jvM|@E6ac{8simQ25`GO^KDc@UG$E&wB^-h~bSge4+*`R9y?uLWWNyB{JIY8$rjl6d4&$J^ycpGcRWC zl~mz)QxVHb*0KzH;uka0$1I(jwuPN&0j^jy$2l5THF8VB&O@%yJ}8G5aLr2XUM8SpP=R{@#!=Pb1Ogl+>NQKCpWTiR2OApwxFmGMbIT z36mg1hIx3Jcfxy9b;$Tiw2{{Me$!bC@Qk)t<{RIU1S72m0M%o@pJFrZfCZV?r+%{YX@P6MLsx6b;oJwbSrRmv$ zL2%RZFW-mOWOOZOpcOSEPhh1Dv-4bA7>h{Nyt$g=>Rt(=pco;VSgWK{+WGF-0uuJe z?*vz5qoPeyd%$(*I_NcRvre5b!(Qn!zwPv+uvaVh32)xE~*YUEUWs}z`<1I;lOq=!wn5K*HwcqE- za~;&NkS8iD&ot=NUZ!?=bb&cza+iKC8DEF8QJ29Z*@V#qEl5L#e1%jhu7UI|RkiPy z%^`PJ*fGrB+&L$blT@YfVt#8=?7f}E#TBYCL^MF?ZpO9u`EDXe^^LX*u`?EZwArC4 zkC=W>S8!UP_ZQEsh&ilG$lL=~bMGc~rRt~8h0;+EV@6gAlEj%8R$P8#d^0Fx_4STf z@sY}EXA|YssaNi>;gv%q)&i7KR7dOw5?Y{n)#YbEfWqn|YD-(4R6OBbi3ntTtx}R6 z4U6rpx*bOw0|^CKf8Q%{J(;Z3FAhT}k*hoj(fT=6h^YC{JXOoQFo)zP>6X%E;xo2a zX`Yoo*bYXG_#3Qi`*I3odF=avNfFod-BjCZcyxZv_W!I_rWp4?G7UOx;O4X3>kgFN zLyPn_glyGayX0|?Y02(hFQ&C1KX7J|b%%#cgP7NeExBscW?S5-x!B1%&(k~IiK8Jp zb*f@{TYm(C0UBPgFzEQr?yP-Jol3;Nb7wPHC}T@kD1Zx&?8l^0rD-~9`N%&g%Z&}- zj)pva_GY8a3YCCH&6VH^m3BeJcFkP4iOO4z3cIknyCat$zn*vjuDh{v@U~*Yd zZq0fr*(CPQVcloQm3Jsb?yWM9faW-&v9}{UKn3JuD7PCs_G=T^or=B>u2HD5S%VHk z*w$a<8WO9#s&wUc|1OtpJu9~W$8ap2MkRFqSI|9iM`q(#4|-T39rpYOihm!{Yf1c~a``U?a)62**r-aBjUwY&d@rRCy8=E;af@@BAIp!}lg+J^heBf@lh`^E6-^N{-mV2#!1jo!) zrvn`$?5JCwuLmHGHr#sr@=AvC!+EWDH{TX3AzjrqyUIip3A)Ta>0$e)y!?fQ-@f)ieX% z&Ld3I{lPAq(HPVdWtW_8OzH?LE}?QWt#P?b19Vjdyhy=*ShFmisU{uPtB^ghx!xuI z4Fz+a?@x9V5i-lZF`!x3agO8hHIGYPn*m+Ti23`g zEfv{`JZp~2oR3XgiARMqe(&G8`|&3PacJ5`Np$=`bcGFEBpWyDXG5+lR1Gz$%8ueN zb*hdqxTmpjj)>fZ9A^wvObq)2p(sqAY4Kf>{A5_r`Z=<1rOWv;jasR}IaHI63>~db z8x5C#{(Xlzo9ut=Z6a>-C<@_8DUattxe|?I8UwAw7XxYq5<_p?-Cnp7x^us{Y2GCf zW^YN$(({R^P~J5@6$32Pl(UH5)8zdUwciR+Sj7BK@N?LOE~RvtJ9WjsRh2fe(Yt^% z#Hj9}!wzY1=Xq~70h6h%bvxw=ITwf4Negv`p_+p?;tw;Q%bLlKyZk_g?+WrZwS8|oveJ(RpM+Rg!4n4e$e z-oQNPvs~58LxpmG?Q$Q9Qp-2Va5P^JU5o4}bY}?b{^W>#DXe!^Nz``LN)n z;;n}A*;dA5)p9RB3bIQ-NoDmaBEBt@c5^`YcmrLeCKk#?o4XYhsogWHSk70Nvh8%> z!2D|$b_IEKv_(8TBwEP^AcB9u+K%$|j3wgtj~bLhySp{n&27nq<@McQZ``Ze*3GT` zSiS{Rq51Sd4uaNZO9}wbg?W5@X|nI3=yq;vuP-|7XydW$_kOt{iKsbwV2|N< zqROE#4uh0eQdE3yK1k}=`2IXx=mNrqD+mk0rVQf^Btn~Ey0uMfmOMA3eg53WKz$Ao7B^a1voS&cnfoGIZa0#8mxCbPQ>Z&d+V&bXO8 zbqEK&08!cZVdS_wxNW3jgCs*reM;bE1YhwMqbzxKbNeK+a6RagB5(;eUom&VGaSWB zKL`lQWTzK_-_gV|OVq{soP zW2T?}0ZW1cH`fyGXt+Yl1Pc;o853r3YCktxj+Iri2XoqVK=9s0q!N3@@zt(OYMd$l z8n&C?*Nw@G(6=b0Bb-022g;YAd&%p>x~-!toRmU%%&~MG zf?Nku{U$ic$x-PQ+-s$7`K+$(6(*a{KgUdLME4lpku}Hw($QS0Rs_RLBu-mqsm{QaM5t0swk>%{Fb#E|umB*6kg>rpMf%yz+z$6@e8O zo3DTQ_WRWFPg~?4Tp!vP7QrPPT8|#I>3~6ldYcej-h;E)3-_l%rtEsf2QVM~Ux6Y~ zVJE-j+P)!CVwE_#re>=~ zg^utG39BW4fMlK;lvL~GWkgcT>HYFjxOEmRGr47Cvs?hkOYA`H@Otm;T{`seazePp zP_)>|l^iq0le5iR^H*Nqz5J^|a*cqu&$_Z^*>B~QmEf~;PDZY5CJ-F=yaUU!_)2D$ zWA;*yCr$nSj}~>zm+eEV&kpxVFnT}n+-G*9zZ3DsioTXiKj=@9+BE6T{=JPrv*9>w zo7$%Y{z%lm$xQQ(fp1m#efE9vev-j>IM1OlWSs21jhNx|NPq|!pj%FJ4aN9gyetX!{IY zv3(-weW|3#o0iy3l%0o{7eQ?)lB*s;Hznm{vv`$hpG5#z|8E^(hkBd$;rpkwLduUp zrggidZc}iPY6tI|8p-3@l#n;0ro(*-_Zosk2KrF zEzZvZl}QITSP~LiH2V=n`+=XnQ^)evpIn7OaewxKA!BFL?%+`f(eM~dz%N2aSeEC7>i7Uh!nVIX0k;Z z#Qmgk2Gx#5wTuO2P>zldwnxtWS_lfVW>DFS#q>4IxN9qWzJk+Nwzo%WiE@+QgMz?9 zW%_lfwZjn+iNH^&bN+xUYkHW}CYMx};}6n0=A;qV8zhCZ{%%ayN$_|KtWBuc53iLW zpth&JBDpu(adM0D@x-I*ai&LlLi_-&p-~uNe^5LukqR#49UF_?s<+?B3}XhP)}xFj zi4tT@k+cst_B6SHEu+T?@N2urALoy}W}E7kBX&0}c0rO3>D{$#T5XVSfxv&n78dVa zGwKxAn_1s;uOFW;OPcmbddmp3T)D3GEo0zLQzdE|Y{UN`lZB^q)-O^nGvqP!g0Opb za}RD86pO;+DU|m~h6f`dA*FofK>yT`v?F*qXH$56Sbk=n_xMTzW>X36<=hH!8>m6L zE-GAN;P(U!0NkeCxI-T!ETdM&WcxO5ftr~-}IDtC} zG-tBvAz(fE1C13sR4NT;zJ|-Dc&!=zZaNAebqLUrOyJD|VS!kq%c&FCTMszi9DIdc zTv@9|8r&Y0%nbUyaVzjenMJ40K)cMyQ0!|-y1})2eRp}8>F*t~y>%#No;f-DN|#O# zc66*>V&H4)q$x#!u#G8Oh*-dPQ6dYz>b2y{A3fE#DYT-{QJG=aDGT@X*ximYch)j> zTToFKSAC>K#xVKe@h|ZTXG~TZgjj zx_cbqq4ZJy5_gy+`hu(SA&I@Er}5aeZdJH)_Z_uSTJJ=K;aPI($x(o}`LLz4@1!4n z6FZle!36fKE}5Jawd#+rMQVefH_n~uIAELEf7>Z%m2yZjT?7!ZEk&dbsN}w zLtH+564zMu*JYuE*NFW?inrXMM?NxSnIM6_2?nbjBb@Ya`EF&CBq~tkf=FDW38VJ; z5l=JVC$UD%=Y#~%lFQQ=WNcQnEfMsEdsF>6A}2Nf z92O=I_hCY5E*#X7kGIOo6xwwY)xz07D|XcECG+^YZLvXdZ=ycP(^7IhR;CCZjY!vb z**KuK)fN9gD~Tpp+E`}!l-|AjA*Spx`!R1tnXD(PdRlQLf+chg_OWBu$KIO&ncrj( zpK{Il{0q$M_W_Va+aa zUsO9^60l=@FHmXKhzz%R*uX0@B;OxY?Vs|gq_>?iHL;%tu;iRo48&%`2t|_){aBid zn`-4Q63tx`T-&;Io~>g5n0-q_WptV#chrG6X*3;kB92Ov%v*qE<@+aChU5>b)Z7zn zrY3EFDX@2w9~@uUr4<@Bly}R=Gdy1ec~=sz5dcFcl8ia~rQ}uwc-9$MCpD+9G`Y1q z*B5wK?ijt@-27#+(-1XJOhVPCCv0l<78b>auiZ6V!yHL(3w-@7Zd^rDjvc=}3!xOK z=8_+w@KsMzpAO0Kt2+N36+R$*7Y_;UdaXB&y1)n}9Py7TPzVeyjTvs>s#wHtWOL

JMPIknAbn1m)!27LCS%q@j=K;lwW11-qM-EBorUYZTTQOwUmWAW)J#f z+Bm(~IPvl?<%)W(k_DJ0c7p_~E}JXz<3SW?^xE zI63%u2AS7FMUYoc_P0aa%ISX%-(1>P=el5cEm%!=Cs^%Kc5L`}GaNa@_uJxFt_NJ= zZSE~;?~x)cY55$s4zDCVd74mRnyU1-zM(+8)>u$`deCs;c*xY1tuSk>`2!%y1II7- zo)8{Hd55f?@{x$g$s^+55C*s%*2+-87|HBk-?MGN-eo3H6}#StCJ6nvt|WpDbdRJB zI!8@$v7|Mc;g3sH(k>x-L8XY3u#D2#Q2_YQXG^fl)qh)^1e?(ISS|{&?un_|h4Dql z=`nQWdow%+JSXJ&E8V+#yZSX>q%S-;Y{6%05qPRjP@CKhSA#Ft(Miv^L@xbOBjccR zk-n1Zi3O+H*KRxIkmS(2SN}KN6M$XUQ3P{R1)N&9$Z_iEp8w-# z{*LzkPKwX{oK6?O1IY)nVv=%VQZgn|GKx}iic*pyl9Gy&k_G;9;{S!<>Fwz14FB&5 qlfnr}rv#z@yushq(+TEp@9Fd3b6zT!own%!0IkRR8fA~{Ui}|Hhij(* literal 0 HcmV?d00001 diff --git a/impl/python/README.md b/impl/python/README.md new file mode 100644 index 0000000..755b306 --- /dev/null +++ b/impl/python/README.md @@ -0,0 +1 @@ +Python implemention From 309acbf2c0a29716514ad11168e9131bc12c470b Mon Sep 17 00:00:00 2001 From: Jamie Davidson Date: Wed, 15 Jul 2026 16:14:06 -0700 Subject: [PATCH 72/89] [OSI][Omni] Add Omni semantic model converter (#175) * Add Omni bidirectional converter Adds converters/omni: an offline OSI <-> Omni semantic model converter, filling the Omni spoke in the hub-and-spoke architecture. - Export (osi-omni export): OSI -> Omni model files (views/*.view.yaml, relationships.yaml, a generated topic, model.yaml when stashed). - Import (osi-omni import): Omni -> OSI. Omni-only features are preserved in custom_extensions[OMNI], so Omni -> OSI -> Omni is lossless. - Unified osi-omni CLI plus a string-in / files-dict-out Python API. - Example-based unit tests, fixture round-trips (incl. TPC-DS), and Hypothesis property-based round-trip tests with a seeded-random fallback. - Adds the OMNI vendor row to converters/index.md. Co-Authored-By: Claude Fable 5 * Handle real-world Omni model layouts in the OSI converter Testing against production Omni instances surfaced model shapes the fixtures did not cover; importing all of them now round-trips losslessly (verified on a 3,498-file and a 427-file production model plus the Ecommerce demo model, all validating against the OSI spec): - Resolve schema-qualified view names (schema__table) from the "# Reference this view as" header, accept per-schema folder layouts, and preserve original file paths through a round trip. - Stash-and-restore instead of erroring: extends-only views, non-equi/ range joins, joins touching query/extends views, and fields or measures using Omni template ({{...}}) syntax. - Preserve explicit Omni defaults verbatim (join_type, relationship_type, redundant table_name, self-named sql), non-canonical on_sql (aliases, casing, spacing), ${view.field} compound keys, empty-string metadata, and timeframes: []. - Accept real Omni identifiers (_fivetran_id, trailing underscores, camelCase); collision checks stay case-insensitive. - Quote source parts that need it ("Omni Views".upload), tag date values in the JSON stash, and suffix duplicate generated relationship names. Co-Authored-By: Claude Fable 5 * Catch sqlglot TokenError in the SQL validator An expression that cannot be tokenized (e.g. templated SQL) crashed the validator instead of being reported as a finding. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- converters/README.md | 1 + converters/omni/README.md | 168 +++++ converters/omni/pyproject.toml | 31 + converters/omni/src/osi_omni/__init__.py | 16 + converters/omni/src/osi_omni/_common.py | 405 ++++++++++ converters/omni/src/osi_omni/cli.py | 101 +++ converters/omni/src/osi_omni/omni_to_osi.py | 701 ++++++++++++++++++ converters/omni/src/osi_omni/osi_to_omni.py | 609 +++++++++++++++ converters/omni/tests/_roundtrip_helpers.py | 361 +++++++++ converters/omni/tests/_util.py | 116 +++ converters/omni/tests/conftest.py | 6 + .../fixtures/fixtureA_omni/relationships.yaml | 4 + .../fixtureA_omni/topics/sales.topic.yaml | 5 + .../fixtureA_omni/views/customer.view.yaml | 7 + .../fixtureA_omni/views/orders.view.yaml | 32 + .../omni/tests/fixtures/fixtureA_osi.yaml | 83 +++ .../tests/fixtures/fixtureB_omni/model.yaml | 10 + .../fixtures/fixtureB_omni/relationships.yaml | 12 + .../topics/order_analysis.topic.yaml | 15 + .../fixtureB_omni/views/order_items.view.yaml | 46 ++ .../fixtureB_omni/views/orders.view.yaml | 6 + .../fixtureB_omni/views/users.view.yaml | 9 + .../fixtures/tpcds_omni/relationships.yaml | 16 + .../topics/tpcds_retail_model.topic.yaml | 11 + .../tpcds_omni/views/customer.view.yaml | 27 + .../tpcds_omni/views/date_dim.view.yaml | 53 ++ .../fixtures/tpcds_omni/views/item.view.yaml | 33 + .../fixtures/tpcds_omni/views/store.view.yaml | 32 + .../tpcds_omni/views/store_sales.view.yaml | 90 +++ converters/omni/tests/test_omni_to_osi.py | 376 ++++++++++ converters/omni/tests/test_osi_to_omni.py | 334 +++++++++ .../omni/tests/test_real_world_layout.py | 170 +++++ converters/omni/tests/test_roundtrip.py | 65 ++ .../omni/tests/test_roundtrip_properties.py | 83 +++ validation/validate.py | 6 +- 35 files changed, 4037 insertions(+), 3 deletions(-) create mode 100644 converters/omni/README.md create mode 100644 converters/omni/pyproject.toml create mode 100644 converters/omni/src/osi_omni/__init__.py create mode 100644 converters/omni/src/osi_omni/_common.py create mode 100644 converters/omni/src/osi_omni/cli.py create mode 100644 converters/omni/src/osi_omni/omni_to_osi.py create mode 100644 converters/omni/src/osi_omni/osi_to_omni.py create mode 100644 converters/omni/tests/_roundtrip_helpers.py create mode 100644 converters/omni/tests/_util.py create mode 100644 converters/omni/tests/conftest.py create mode 100644 converters/omni/tests/fixtures/fixtureA_omni/relationships.yaml create mode 100644 converters/omni/tests/fixtures/fixtureA_omni/topics/sales.topic.yaml create mode 100644 converters/omni/tests/fixtures/fixtureA_omni/views/customer.view.yaml create mode 100644 converters/omni/tests/fixtures/fixtureA_omni/views/orders.view.yaml create mode 100644 converters/omni/tests/fixtures/fixtureA_osi.yaml create mode 100644 converters/omni/tests/fixtures/fixtureB_omni/model.yaml create mode 100644 converters/omni/tests/fixtures/fixtureB_omni/relationships.yaml create mode 100644 converters/omni/tests/fixtures/fixtureB_omni/topics/order_analysis.topic.yaml create mode 100644 converters/omni/tests/fixtures/fixtureB_omni/views/order_items.view.yaml create mode 100644 converters/omni/tests/fixtures/fixtureB_omni/views/orders.view.yaml create mode 100644 converters/omni/tests/fixtures/fixtureB_omni/views/users.view.yaml create mode 100644 converters/omni/tests/fixtures/tpcds_omni/relationships.yaml create mode 100644 converters/omni/tests/fixtures/tpcds_omni/topics/tpcds_retail_model.topic.yaml create mode 100644 converters/omni/tests/fixtures/tpcds_omni/views/customer.view.yaml create mode 100644 converters/omni/tests/fixtures/tpcds_omni/views/date_dim.view.yaml create mode 100644 converters/omni/tests/fixtures/tpcds_omni/views/item.view.yaml create mode 100644 converters/omni/tests/fixtures/tpcds_omni/views/store.view.yaml create mode 100644 converters/omni/tests/fixtures/tpcds_omni/views/store_sales.view.yaml create mode 100644 converters/omni/tests/test_omni_to_osi.py create mode 100644 converters/omni/tests/test_osi_to_omni.py create mode 100644 converters/omni/tests/test_real_world_layout.py create mode 100644 converters/omni/tests/test_roundtrip.py create mode 100644 converters/omni/tests/test_roundtrip_properties.py diff --git a/converters/README.md b/converters/README.md index 9dd4f98..8daf5d0 100644 --- a/converters/README.md +++ b/converters/README.md @@ -73,6 +73,7 @@ The Ossie specification currently defines extensions for the following vendors: | `SALESFORCE` | Salesforce / Tableau semantic layer | | `DBT` | dbt semantic models | | `DATABRICKS` | Databricks semantic layer | +| `OMNI` | Omni semantic model | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/converters/omni/README.md b/converters/omni/README.md new file mode 100644 index 0000000..c1029cc --- /dev/null +++ b/converters/omni/README.md @@ -0,0 +1,168 @@ +# OSI <-> Omni converter + +Bidirectional, offline conversion between an OSI semantic model and +[Omni](https://docs.omni.co/modeling) semantic model files. No Omni connection +required. + +An Omni model is a *directory* of YAML files rather than a single document, so +this converter maps one OSI YAML document to/from the Omni model layout: + +``` +model.yaml # model-wide config (restored verbatim from a prior import) +relationships.yaml # top-level list of join definitions +views/.view.yaml # one per OSI dataset +topics/.topic.yaml # one generated topic per OSI model (or restored originals) +``` + +Import also accepts the layout Omni's API/IDE emits (`omni models yaml-get`, +git sync): view/topic files in per-schema or arbitrary folders +(`DELIGHTED/response.view`), bare `.view`/`.topic` suffixes, and +schema-qualified view names (`delighted__response`) taken from each file's +`# Reference this view as ...` header (falling back to the file's basename). +Original file paths are preserved through a round trip. + +- **Export** (`osi-omni export`): OSI -> Omni files. Datasets become views, + relationships become `relationships.yaml` joins, metrics become measures on + the view they reference, and the model becomes a topic (rooted at the + fact/FK-sink dataset, or `--base-view`). +- **Import** (`osi-omni import`): Omni files -> OSI. Omni features OSI has no + native field for are preserved in `custom_extensions[OMNI]`, so + **Omni -> OSI -> Omni is lossless**. + +On **export** (OSI -> Omni), OSI features with no Omni slot -- `unique_keys`, +relationship `ai_context`, dataset-level `ai_context` synonyms, model-level +`ai_context` synonyms/examples, foreign-vendor `custom_extensions`, fields and +metrics without a usable dialect -- are **dropped with a warning**. On +**import** (Omni -> OSI), Omni-only features (formats, timeframes, hidden/tags, +topic curation, the model file, ...) are instead **preserved** in +`custom_extensions[OMNI]`. Any input that breaks a +[requirement](#requirements) **raises a `ConversionError`** -- the converter +never silently drops a field or produces an invalid result. + +## Installation + +```bash +pip install osi-omni # once published to PyPI +# or, from a checkout of this directory: +pip install -e . +``` + +The only runtime dependency is `PyYAML`. Python 3.9+. + +## Usage + +### Command line + +```bash +osi-omni export -i model.yaml -o omni_model/ [--base-view orders] [--dialect SNOWFLAKE] +osi-omni import -i omni_model/ [-o model.yaml] [--name my_model] [--topic orders] +``` + +`export` writes the Omni files into the `-o` directory. `import` reads a model +directory (the [local-editor / git layout](https://docs.omni.co/guides/modeling/local-development), +`.yaml`-suffixed or bare `.view`/`.topic` names both work); with no `-o` the +OSI YAML goes to stdout. `--base-view` picks the dataset the generated topic is +rooted at (default: the FK-sink dataset). `--topic` picks which topic's +description/AI context map onto the OSI model when there are several. + +### Python API + +```python +from osi_omni import convert_osi_to_omni, convert_omni_to_osi + +files = convert_osi_to_omni(osi_yaml_str) # -> {relative filename: YAML str} +osi_yaml = convert_omni_to_osi(files) # {relative filename: YAML str} -> str +``` + +## Mapping + +Each row maps in both directions; the **Notes** flag where a behavior is +specific to **export** (OSI -> Omni) or **import** (Omni -> OSI). + +| OSI | Omni | Notes | +|---|---|---| +| `semantic_model.name` | topic file name | Import: the mapped topic's name (override with `--name`). | +| `model.description` / `ai_context.instructions` | topic `description` / `ai_context` | Import: taken from the sole topic, or `--topic`. | +| dataset | `views/.view.yaml` | Import: a stashed original path (`DELIGHTED/response.view`) is restored on export. | +| `dataset.source` `catalog.schema.table` / `schema.table` | view `catalog` + `schema` + `table_name` | `table_name` left implicit when it matches the file name; a part that is not a plain identifier is double-quoted (`"Omni Views".upload`). | +| `dataset.source` `SELECT ...` | view `sql:` | A SQL-defined view. | +| `dataset.description` / `ai_context.instructions` | view `description` / `ai_context` | | +| `dataset.primary_key` (single) | `primary_key: true` on the matching dimension | Export: a key column no field covers becomes a hidden dimension. | +| `dataset.primary_key` (composite) | view `custom_compound_primary_key_sql` | Import: `${view.field}` entries resolve to plain field names (original list stashed); multiple `primary_key: true` dimensions also form a composite key. | +| field | dimension | Export: an already-valid Omni identifier (incl. `_fivetran_id`, `..._day_`, camelCase) passes through; anything else sanitizes to `[a-z][a-z0-9_]*`. A case-insensitive collision is an error. | +| `field.expression` | dimension `sql` | Export: a bare column named like the field emits `{}` (the schema-layer default); import translates `${field}`/`${view.field}`/`${TABLE}.col` references and stashes the original `sql`. | +| `field.dimension.is_time` | dimension `timeframes` | Export: the Omni default list; import stashes the exact list. | +| `field.label` / `description` | `label` / `description` | | +| `field.ai_context` synonyms / instructions | `synonyms` / `ai_context` | | +| relationship | `relationships.yaml` entry | `from`(many) -> `join_from_view`, `to`(one) -> `join_to_view`, columns -> `on_sql` equi-join. Declared `join_type`/`relationship_type` (even Omni defaults) and any `on_sql` the rebuild would not reproduce (aliases, `and` casing, spacing) are stashed verbatim. | +| `relationship.name` | -- | Regenerated as `_to_` on import (suffixed `_2`, `_3`, ... when several joins share a view pair). | +| metric | measure on the view its expression references (else the base view) | `AGG(view.field)` <-> `aggregate_type` + `sql: ${field}`; `COUNT(*)` <-> `aggregate_type: count`; anything else <-> a raw-`sql` measure. | +| `metric.description` / `ai_context` | measure `description` / `synonyms` / `ai_context` | | +| `custom_extensions[OMNI]` | everything Omni-only | Import stashes; export restores -- keeping `Omni -> OSI -> Omni` lossless. | + +**Stashed on import** (and restored on export): the model file (verbatim), +topics (verbatim, minus the natively-mapped description/AI context), original +file paths, view extras (`label`, `hidden`, `tags`, view-level `filters:`, +...), dimension extras (`format`, `group_label`, exact `timeframes`, original +`sql`, ...), present-but-empty metadata (`description: ''`), join extras +(declared `join_type`/`relationship_type`, `reversible`, `where_sql`, aliases, +non-canonical `on_sql`), joins OSI cannot represent (non-equi/range joins, +joins touching a query or extends-only view -- restored at their original +positions), fields/measures whose sql uses Omni template (`{{...}}`) syntax, +non-reconstructible measures (filtered, `percentile`/`list`/`*_distinct_on`, +raw-SQL), and files with no OSI form (query views, extends-only views, +unrecognized files). + +**Expression dialects**: Omni SQL is the SQL of the model's database +connection, and the OSI dialect enum has no `OMNI` entry -- so import emits +`ANSI_SQL` expressions, and export prefers `ANSI_SQL` with `--dialect` +prepending a warehouse dialect (e.g. `SNOWFLAKE` for a Snowflake-backed Omni +model). A field/metric with neither is dropped with a warning. + +## Requirements + +Conversion raises a `ConversionError` (rather than guessing or emitting +something invalid) when an input breaks one of these: + +- a dataset `source` has no schema part (Omni views require `schema`); +- the relationship graph gives no unambiguous base view for the generated topic + (multiple FK sinks or a cycle) and `--base-view` is not given; +- two names sanitize to the same Omni identifier, case-insensitively (never + silently merged); two view files resolve to the same canonical view name; +- a measure has an unknown `aggregate_type`; an import directory has no + convertible view files; the input YAML is malformed. + +## Notes and limitations + +- Exported `on_sql` references columns as `${view.column}`. Omni resolves these + against the schema layer, which auto-generates a dimension per physical + column, so the reference is valid even when the OSI model declares no field + for the column. +- `dimension: {is_time: false}` is equivalent to omitting `dimension` and is + normalized away on a round trip. +- One generated topic per OSI model; multi-path (aliased) join fan-out and + Omni query views, extends-only views, non-equi joins, composite topics, + access grants/filters, and templated filters are stash-and-restore only (no + OSI semantics). +- A model containing *only* query/extends views (e.g. a pure GA4-export model) + has nothing to convert and is rejected. +- OSI metric order is regrouped by view on import (order is not semantic). + +## Development + +```bash +pip install -e ".[dev]" +python3 -m pytest tests/ +``` + +Example-based unit tests plus Hypothesis property-based round-trip tests +(`test_roundtrip_properties.py`, which fall back to a seeded-random sweep if +`hypothesis` is not installed). + +## Future effort + +Both the OSI specification and Omni's model YAML are still evolving. As either +side adds or changes fields, this converter will be updated to track them -- +extending the mapping and coverage in both directions (query views, composite +topics, measure filters as first-class OSI once the spec grows a slot for +them) to keep the conversion current. diff --git a/converters/omni/pyproject.toml b/converters/omni/pyproject.toml new file mode 100644 index 0000000..b4be76f --- /dev/null +++ b/converters/omni/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "osi-omni" +version = "0.2.0.dev0" +description = "Bidirectional converter between OSI semantic models and Omni semantic model files" +requires-python = ">=3.9" +dependencies = [ + "PyYAML>=6.0", +] + +[project.license] +text = "Apache-2.0" + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "hypothesis>=6.0", +] + +[project.scripts] +osi-omni = "osi_omni.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/osi_omni"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/converters/omni/src/osi_omni/__init__.py b/converters/omni/src/osi_omni/__init__.py new file mode 100644 index 0000000..8eab424 --- /dev/null +++ b/converters/omni/src/osi_omni/__init__.py @@ -0,0 +1,16 @@ +"""Bidirectional converter between OSI semantic models and Omni semantic model +files (model.yaml / relationships.yaml / views/*.view.yaml / topics/*.topic.yaml). +Pure offline transforms: OSI YAML string <-> {relative filename: YAML string}. + + from osi_omni import convert_osi_to_omni, convert_omni_to_osi +""" + +from ._common import ConversionError +from .omni_to_osi import convert_omni_to_osi +from .osi_to_omni import convert_osi_to_omni + +__all__ = [ + "ConversionError", + "convert_omni_to_osi", + "convert_osi_to_omni", +] diff --git a/converters/omni/src/osi_omni/_common.py b/converters/omni/src/osi_omni/_common.py new file mode 100644 index 0000000..571b525 --- /dev/null +++ b/converters/omni/src/osi_omni/_common.py @@ -0,0 +1,405 @@ +"""Shared helpers for the OSI <-> Omni converters. + +Both directions are pure offline YAML transforms. The cross-cutting concerns live +here: version constants, the dialect preference order, the `custom_extensions` +stash protocol, Omni file-name conventions, identifier sanitization, and the +`${...}` reference translation between Omni SQL and the plain column references +OSI expressions use. +""" + +import datetime +import json +import re + +import yaml + +# OSI semantic model spec version this converter targets (see core-spec). +OSI_VERSION = "0.2.0.dev0" + +# Vendor id used for the `custom_extensions` stash. +VENDOR = "OMNI" + +# Omni SQL is the SQL of the model's database connection, so there is no OMNI +# entry in the OSI dialect enum. Import emits ANSI_SQL; export prefers ANSI_SQL +# and lets the caller prepend a warehouse dialect (e.g. SNOWFLAKE) that the +# actual connection would accept. +DIALECT_ANSI = "ANSI_SQL" + +# Bump when the shape of a stashed `data` blob changes. +STASH_VERSION = 1 + +# Omni model file names (the local-editor/git layout, with `.yaml` appended). +MODEL_FILE = "model.yaml" +RELATIONSHIPS_FILE = "relationships.yaml" +VIEW_DIR = "views" +TOPIC_DIR = "topics" + +# Omni relationship defaults (left implicit on export when they hold). +DEFAULT_JOIN_TYPE = "always_left" +REL_MANY_TO_ONE = "many_to_one" +REL_ONE_TO_MANY = "one_to_many" + +# The timeframes Omni applies to a time dimension by default; used to represent +# OSI `dimension.is_time` when no exact list is stashed. +DEFAULT_TIMEFRAMES = ["raw", "date", "week", "month", "quarter", "year"] + +# A valid Omni identifier (view, dimension, measure, topic name). Broader than +# the documented lowercase convention because real Omni-generated models use +# more: leading underscores (Fivetran's `_fivetran_id`), trailing underscores +# (truncated column names), and camelCase (JSON-flattened `..._dimensionIndex`). +_OMNI_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + +# A bare SQL identifier (single column reference), e.g. `c_name`. +_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + +# An Omni `${...}` reference: ${field}, ${view.field}, ${view.field[timeframe]}, +# ${TABLE}.column. Group 1 is the reference body (without the braces). +_OMNI_REF_RE = re.compile(r"\$\{\s*([^}]*?)\s*\}") + + +class ConversionError(Exception): + """Raised when an input cannot be converted.""" + + +def require(obj, key, what): + """Return `obj[key]`, or raise a clean ConversionError if it's missing/empty -- + so malformed input surfaces as an error message rather than a raw KeyError. + + Presence is tested by key (not truthiness), so a legitimately falsy value such + as `0` or `False` is returned; a missing key, a null, or an empty/whitespace + string is rejected. + """ + if not isinstance(obj, dict) or key not in obj or obj[key] is None: + raise ConversionError(f"{what} is missing required '{key}'") + value = obj[key] + if isinstance(value, str) and not value.strip(): + raise ConversionError(f"{what} has an empty '{key}'") + return value + + +def require_str(obj, key, what): + """Like require(), but also enforce the value is a string -- so a non-string + scalar (e.g. a YAML number for a name or expression) raises a clean + ConversionError instead of crashing later in a string operation.""" + value = require(obj, key, what) + if not isinstance(value, str): + raise ConversionError( + f"{what}: '{key}' must be a string, got {type(value).__name__}") + return value + + +# PyYAML's default YAML 1.1 semantics turn bare on/off/yes/no into booleans, which +# would corrupt Omni string values (a label "On", a week_start_day, a synonym). +# The Loader below uses YAML 1.2 booleans (only true/false); the Dumper +# force-quotes bool-like string tokens so the output round-trips through a 1.1 +# reader too. Same approach as the osi-databricks converter. +class _Yaml12Loader(yaml.SafeLoader): + """SafeLoader with YAML 1.2 boolean semantics.""" + + +class _Yaml12Dumper(yaml.SafeDumper): + """SafeDumper with YAML 1.2 boolean semantics.""" + + +_YAML12_BOOL = re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$") +for _cls in (_Yaml12Loader, _Yaml12Dumper): + # Drop the YAML 1.1 bool resolver (yes/no/on/off/y/n) and re-add a 1.2 one. + _cls.yaml_implicit_resolvers = { + ch: [(tag, rx) for (tag, rx) in resolvers if tag != "tag:yaml.org,2002:bool"] + for ch, resolvers in _cls.yaml_implicit_resolvers.items() + } + _cls.add_implicit_resolver("tag:yaml.org,2002:bool", _YAML12_BOOL, list("tTfF")) + + +_YAML11_BOOL_STRS = frozenset( + variant + for word in ("y", "n", "yes", "no", "on", "off", "true", "false") + for variant in (word, word.capitalize(), word.upper()) +) + + +def _represent_str(dumper, data): + style = "'" if data in _YAML11_BOOL_STRS else None + if "\n" in data: + style = "|" + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style) + + +_Yaml12Dumper.add_representer(str, _represent_str) + + +def load_yaml(text, what="input"): + """Parse YAML with 1.2 boolean semantics. A syntax error is surfaced as a + ConversionError so callers (and the CLI) get a clean message.""" + try: + return yaml.load(text, Loader=_Yaml12Loader) + except yaml.YAMLError as e: + raise ConversionError(f"Invalid YAML in {what}: {e}") from e + + +def dump_yaml(obj): + """Serialize to YAML with 1.2 boolean semantics; bool-like string tokens are + force-quoted so a YAML 1.1 reader of this output sees strings, not booleans.""" + return yaml.dump(obj, Dumper=_Yaml12Dumper, sort_keys=False, + default_flow_style=False, allow_unicode=True) + + +def is_simple_identifier(expr): + """True if `expr` is a single bare column reference (no operators/functions).""" + return isinstance(expr, str) and bool(_IDENTIFIER_RE.match(expr.strip())) + + +def is_omni_name(name): + """True if `name` is already a valid Omni identifier.""" + return isinstance(name, str) and bool(_OMNI_NAME_RE.match(name)) + + +def sanitize_name(name, what, taken): + """Coerce an OSI name into a valid Omni identifier. + + A name that is already a valid Omni identifier passes through untouched + (leading/trailing underscores and camelCase are legal and occur in real + Omni-generated models); anything else is lowercased with every invalid + character run replaced by `_`. A result colliding case-insensitively with + one already in `taken` (a set of casefolded names) is an error rather than + a silent merge; the caller adds `result.lower()` to `taken`. + """ + raw = str(name) + if _OMNI_NAME_RE.match(raw): + out = raw + else: + out = re.sub(r"[^a-z0-9_]+", "_", raw.lower()).strip("_") + if not out or not out[0].isalpha(): + out = f"v_{out}" if out else "v" + if out.lower() in taken: + raise ConversionError( + f"{what} '{name}' sanitizes to '{out}', which collides with another " + f"name; rename it in the OSI model." + ) + return out + + +def view_file(view_name): + return f"{VIEW_DIR}/{view_name}.view.yaml" + + +def topic_file(topic_name): + return f"{TOPIC_DIR}/{topic_name}.topic.yaml" + + +# YAML parses a bare `2024-01-01` (e.g. in a topic's default_filters) into a +# datetime.date, which JSON cannot hold; the stash tags such values so they come +# back as dates and re-dump unquoted, keeping the round trip lossless. +_JSON_TEMPORAL = {"date": datetime.date, "datetime": datetime.datetime} + + +def _json_default(o): + for tag, cls in _JSON_TEMPORAL.items(): + if type(o) is cls: + return {"__osi_omni__": tag, "v": o.isoformat()} + raise TypeError(f"Object of type {type(o).__name__} is not JSON serializable") + + +def _json_object_hook(d): + cls = _JSON_TEMPORAL.get(d.get("__osi_omni__", "")) + if cls is not None and set(d) == {"__osi_omni__", "v"}: + return cls.fromisoformat(d["v"]) + return d + + +def read_stash(obj): + """Return the OMNI stash dict on an OSI object, or {} if absent. + + The `_v` version marker is stripped from the returned dict. + """ + for ext in (obj or {}).get("custom_extensions") or []: + if ext.get("vendor_name") == VENDOR: + data = json.loads(ext.get("data") or "{}", + object_hook=_json_object_hook) + data.pop("_v", None) + return data + return {} + + +def write_stash(obj, data): + """Attach an OMNI `custom_extensions` entry holding `data` (a dict). + + No-op when `data` is empty, so hand-authored OSI stays clean. Merges into an + existing OMNI entry if one is already present. + """ + if not data: + return + payload = {"_v": STASH_VERSION} + payload.update(data) + blob = json.dumps(payload, default=_json_default) + exts = obj.setdefault("custom_extensions", []) + for ext in exts: + if ext.get("vendor_name") == VENDOR: + ext["data"] = blob + return + exts.append({"vendor_name": VENDOR, "data": blob}) + + +def foreign_vendor_extensions(obj): + """Return non-OMNI custom_extensions (dropped on export, with a warning).""" + return [ + ext + for ext in (obj or {}).get("custom_extensions") or [] + if ext.get("vendor_name") != VENDOR + ] + + +def pick_expression(osi_expression, preferred=None): + """Choose the SQL string for an OSI expression. + + Preference order: the caller-chosen warehouse dialect (Omni passes SQL through + to the connection's database, so e.g. SNOWFLAKE SQL is valid on a Snowflake- + backed Omni model), then ANSI_SQL. Returns None if neither is present (the + caller warns and skips). + """ + dialects = { + d.get("dialect"): d.get("expression") + for d in (osi_expression or {}).get("dialects") or [] + } + expr = None + if preferred: + expr = dialects.get(preferred) + if expr is None: + expr = dialects.get(DIALECT_ANSI) + if expr is not None and not isinstance(expr, str): + raise ConversionError( + f"expression must be a string, got {type(expr).__name__}") + return expr + + +def synonyms_of(ai_context): + """Extract the synonyms list from an OSI ai_context (object form only).""" + if isinstance(ai_context, dict): + return list(ai_context.get("synonyms") or []) + return [] + + +def instructions_of(ai_context): + """The free-text part of an OSI ai_context: the string itself, or the + object form's `instructions`.""" + if isinstance(ai_context, str) and ai_context.strip(): + return ai_context + if isinstance(ai_context, dict): + text = ai_context.get("instructions") + if isinstance(text, str) and text.strip(): + return text + return None + + +# One part of a dotted source reference: double-quoted (may hold spaces/dots -- +# Omni's uploaded-CSV schema is literally `Omni Views`) or a bare name. +_SOURCE_PARTS_RE = re.compile(r'^(?:"[^"]+"|[^".]+)(?:\.(?:"[^"]+"|[^".]+))*$') +_SOURCE_PART_RE = re.compile(r'"([^"]+)"|([^".]+)') + + +def quote_source_part(part): + """Quote one part of an OSI dotted source when it needs it.""" + p = str(part) + return p if re.fullmatch(r"[A-Za-z0-9_$]+", p) else f'"{p}"' + + +def parse_source(source, dataset_name): + """Split an OSI dataset `source` into Omni view placement. + + Returns ("sql", sql_text) for a SELECT/WITH subquery source, or + ("table", catalog_or_None, schema, table) for a dotted table reference + (parts may be double-quoted: `"Omni Views".channel_info`). + Omni views require a `schema`, so a bare 1-part table name is rejected. + """ + if not source or not str(source).strip(): + raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") + s = str(source).strip() + if re.match(r"(?i)(select|with)\b", s): + return ("sql", s) + if not _SOURCE_PARTS_RE.match(s): + raise ConversionError( + f"Dataset '{dataset_name}': source '{source}' is not a valid dotted " + f"table reference or SELECT/WITH subquery" + ) + parts = [] + for m in _SOURCE_PART_RE.finditer(s): + quoted, bare = m.group(1), m.group(2) + if bare is not None and any(ch.isspace() for ch in bare): + raise ConversionError( + f"Dataset '{dataset_name}': source '{source}' is not a valid " + f"dotted table reference or SELECT/WITH subquery" + ) + parts.append(quoted if quoted is not None else bare) + if len(parts) == 3: + return ("table", parts[0], parts[1], parts[2]) + if len(parts) == 2: + return ("table", None, parts[0], parts[1]) + raise ConversionError( + f"Dataset '{dataset_name}': source '{source}' has no schema part; Omni " + f"views require a `schema`, so use `schema.table` or `catalog.schema.table`" + ) + + +def join_source(view): + """Rebuild an OSI dataset `source` string from an Omni view dict.""" + if view.get("sql") is not None: + return str(view["sql"]).strip() + schema = view.get("schema") + table = view.get("table_name") + if not schema or not table: + return None + parts = [view["catalog"], schema, table] if view.get("catalog") else [schema, table] + return ".".join(quote_source_part(p) for p in parts) + + +def omni_sql_to_osi(sql, own_view): + """Translate Omni `${...}` references in a SQL string to the plain references + OSI expressions use. Returns (translated, changed). + + - `${TABLE}.col` -> `col` (a raw column of the owning view) + - `${field}` -> `field` (same-view field) + - `${own_view.field}` -> `field` (qualified same-view field) + - `${other.field}` -> `other.field` + - `${view.field[tf]}` -> `view.field` (timeframe access has no OSI form; + the caller warns and stashes the original) + OSI has no field-vs-column distinction, so both flavors flatten to names. + """ + changed = False + + def repl(m): + nonlocal changed + changed = True + body = m.group(1) + if body == "TABLE": + return "__OSI_TABLE__" # handled below with its trailing dot + body = re.sub(r"\[[^\]]*\]$", "", body).strip() # drop [timeframe] + if "." in body: + head, rest = body.split(".", 1) + if head == own_view: + return rest + return body + + out = _OMNI_REF_RE.sub(repl, sql) + out = out.replace("__OSI_TABLE__.", "").replace("__OSI_TABLE__", "") + return out, changed + + +def has_timeframe_ref(sql): + """True if the Omni SQL contains a `${view.field[timeframe]}` reference.""" + return bool(re.search(r"\$\{[^}]*\[[^\]]*\][^}]*\}", sql or "")) + + +def osi_expr_refs_to_omni(expr, view_names): + """Rewrite `view.column` references in an OSI expression to Omni `${view.column}` + form, for the known `view_names` only -- so a genuine schema-qualified table or + an unrelated dotted token is left alone. Bare columns stay bare (raw columns + are legal in Omni SQL).""" + if not view_names: + return expr + + pattern = re.compile( + r"(? Omni converter. + + osi-omni export -i model.yaml -o omni_model/ [--base-view orders] [--dialect SNOWFLAKE] + osi-omni import -i omni_model/ [-o model.yaml] [--name my_model] [--topic orders] + +`export` converts an OSI semantic model into an Omni model directory +(model.yaml / relationships.yaml / views/*.view.yaml / topics/*.topic.yaml); +`import` does the reverse. Import with no `-o` writes the OSI YAML to stdout; +export always needs `-o` (a directory). Conversions that drop information emit +warnings to stderr. +""" + +import argparse +import os +import sys + +from ._common import ConversionError +from .omni_to_osi import convert_omni_to_osi +from .osi_to_omni import convert_osi_to_omni + + +def _build_parser(): + parser = argparse.ArgumentParser(prog="osi-omni", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command") + sub.required = True + + exp = sub.add_parser("export", help="OSI semantic model -> Omni model directory") + exp.add_argument("-i", "--input", required=True, help="OSI YAML file") + exp.add_argument("-o", "--output", required=True, + help="output directory for the Omni model files") + exp.add_argument("-b", "--base-view", + help="dataset the generated topic is rooted at " + "(default: the FK-sink dataset)") + exp.add_argument("-d", "--dialect", + help="preferred OSI expression dialect (e.g. SNOWFLAKE); " + "ANSI_SQL is always the fallback") + + imp = sub.add_parser("import", help="Omni model directory -> OSI semantic model YAML") + imp.add_argument("-i", "--input", required=True, help="Omni model directory") + imp.add_argument("-o", "--output", help="output OSI YAML file (default: stdout)") + imp.add_argument("--name", help="OSI model name (default: the mapped topic's name)") + imp.add_argument("--topic", + help="topic whose description/AI context map onto the OSI model " + "(default: the sole topic, if there is exactly one)") + return parser + + +def _read_model_dir(path): + """Collect every YAML file under an Omni model directory as {relative path: + text}. Hidden files and non-YAML extensions (except the canonical + extensionless `model`/`relationships`/`*.view`/`*.topic` names) are skipped.""" + if not os.path.isdir(path): + raise ConversionError(f"'{path}' is not a directory") + files = {} + for dirpath, dirnames, filenames in os.walk(path): + dirnames[:] = [d for d in sorted(dirnames) if not d.startswith(".")] + for fname in sorted(filenames): + if fname.startswith("."): + continue + rel = os.path.relpath(os.path.join(dirpath, fname), path) + rel = rel.replace(os.sep, "/") + base = fname.lower() + if not (base.endswith((".yaml", ".yml", ".view", ".topic")) + or base in ("model", "relationships")): + continue + with open(os.path.join(dirpath, fname)) as fh: + files[rel] = fh.read() + return files + + +def main(argv=None): + args = _build_parser().parse_args(argv) + try: + if args.command == "export": + with open(args.input) as fh: + osi_yaml = fh.read() + files = convert_osi_to_omni(osi_yaml, base_view=args.base_view, + dialect=args.dialect) + for rel, text in files.items(): + dest = os.path.join(args.output, *rel.split("/")) + os.makedirs(os.path.dirname(dest) or ".", exist_ok=True) + with open(dest, "w") as fh: + fh.write(text) + print(f"Wrote {len(files)} file(s) to {args.output}", file=sys.stderr) + else: + files = _read_model_dir(args.input) + out = convert_omni_to_osi(files, model_name=args.name, topic=args.topic) + if args.output: + with open(args.output, "w") as fh: + fh.write(out) + else: + sys.stdout.write(out) + except (ConversionError, OSError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/omni/src/osi_omni/omni_to_osi.py b/converters/omni/src/osi_omni/omni_to_osi.py new file mode 100644 index 0000000..b627509 --- /dev/null +++ b/converters/omni/src/osi_omni/omni_to_osi.py @@ -0,0 +1,701 @@ +"""Convert Omni semantic model files to an OSI semantic model. + +Pure offline conversion. Accepts the Omni model-directory layout as a mapping of +{relative filename: YAML string} -- `model.yaml`, `relationships.yaml`, +`views/*.view.yaml`, `topics/*.topic.yaml`. Omni features OSI has no native field +for (formats, timeframes, hidden/tags, topic curation, the model file itself, +query views, extends-only views, non-equi joins, ...) are preserved in +`custom_extensions[OMNI]` so that converting back reproduces the original files. +See README.md. + +Usage (CLI): + osi-omni import -i omni_model/ [-o model.yaml] [--name NAME] [--topic TOPIC] +""" + +import re +import warnings + +from ._common import ( + ConversionError, + DIALECT_ANSI, + MODEL_FILE, + OSI_VERSION, + REL_MANY_TO_ONE, + REL_ONE_TO_MANY, + RELATIONSHIPS_FILE, + TOPIC_DIR, + dump_yaml, + has_timeframe_ref, + is_simple_identifier, + join_source, + load_yaml, + omni_sql_to_osi, + require_str, + view_file, + write_stash, +) + + +def _warn(scope, msg): + warnings.warn(f"[{scope}] {msg}") + + +_VIEW_FILE_RE = re.compile(r"(?:^|/)([^/]+)\.view(?:\.ya?ml)?$") +_QUERY_VIEW_FILE_RE = re.compile(r"(?:^|/)([^/]+)\.query\.view(?:\.ya?ml)?$") +_TOPIC_FILE_RE = re.compile(r"(?:^|/)([^/]+)\.topic(?:\.ya?ml)?$") +_MODEL_FILE_RE = re.compile(r"(?:^|/)model(?:\.ya?ml)?$") +_RELS_FILE_RE = re.compile(r"(?:^|/)relationships(?:\.ya?ml)?$") + +# Omni's IDE/API writes each view file with a header naming the identifier the +# rest of the model uses for it -- which is schema-qualified (`schema__table`) +# when the view is outside the connection's default schema, and so differs from +# the file's basename. That header is authoritative; the basename is the +# fallback for hand-laid-out directories (including this converter's exports). +_REF_COMMENT_RE = re.compile(r"^#\s*Reference this view as\s+([A-Za-z_]\w*)\s*$") + + +def _canonical_view_name(text, basename): + for line in text.splitlines()[:5]: + m = _REF_COMMENT_RE.match(line) + if m: + return m.group(1) + if line.strip() and not line.lstrip().startswith("#"): + break + return basename + +# View-file keys the converter maps natively; everything else is stashed +# verbatim in the dataset's `view_extras` (and restored on export). +_VIEW_NATIVE_KEYS = {"schema", "catalog", "table_name", "sql", "description", + "ai_context", "dimensions", "measures", + "custom_compound_primary_key_sql"} + +# Dimension keys mapped natively; the rest stash flat on the field. +_DIM_NATIVE_KEYS = {"sql", "label", "description", "synonyms", "ai_context", + "primary_key"} + +# Measure keys the OSI metric represents natively (given a reconstructible +# aggregate); any other key forces the full-measure stash. +_MEASURE_NATIVE_KEYS = {"sql", "aggregate_type", "description", "synonyms", + "ai_context"} + +# aggregate_type values whose OSI expression the exporter can rebuild exactly. +_SIMPLE_AGGS = {"sum": "SUM", "count": "COUNT", "average": "AVG", "min": "MIN", + "max": "MAX", "median": "MEDIAN", "count_distinct": None} + +# Best-effort ANSI renderings for Omni-only aggregate types. The original +# measure is stashed verbatim, so the Omni -> OSI -> Omni trip stays lossless; +# the expression is what other OSI consumers see. +_EXOTIC_AGGS = {"percentile", "list", "sum_distinct_on", "average_distinct_on", + "median_distinct_on", "percentile_distinct_on"} + +_ON_CLAUSE_RE = re.compile( + r"^\s*\$\{\s*([A-Za-z_][\w]*)\.([A-Za-z_][\w]*)\s*\}" + r"\s*=\s*" + r"\$\{\s*([A-Za-z_][\w]*)\.([A-Za-z_][\w]*)\s*\}\s*$" +) + + +def convert_omni_to_osi(files, model_name=None, topic=None): + """Convert Omni model files ({relative filename: YAML str}) to OSI YAML. + + `model_name` overrides the OSI model name (default: the mapped topic's name, + else 'omni_model'). `topic` names the topic whose description/AI context map + onto the OSI model when the directory holds more than one topic. + """ + if not isinstance(files, dict) or not files: + raise ConversionError("expected a non-empty mapping of {filename: YAML}") + + views, topics = {}, {} + view_meta = {} # canonical name -> (file path, file basename) + topic_paths = {} + unmapped_views = set() # view names present as files but with no OSI dataset + model_yaml = None + rel_entries = [] + extra_files = {} + for fname, text in files.items(): + qv = _QUERY_VIEW_FILE_RE.search(fname) + if qv: + # Query views are backed by a saved query, not a table; OSI has no + # dataset form for them. Preserved verbatim, restored on export. + _warn(f"file '{fname}'", "query views have no OSI dataset form; " + "preserved in custom_extensions only") + unmapped_views.add(_canonical_view_name(text, qv.group(1))) + extra_files[fname] = text + continue + mv = _VIEW_FILE_RE.search(fname) + if mv: + vname = _canonical_view_name(text, mv.group(1)) + parsed = load_yaml(text, fname) or {} + source = join_source( + dict(parsed, table_name=parsed.get("table_name", mv.group(1)))) + if source is None: + # A view with no schema/sql of its own (an `extends`-only view) + # has no standalone OSI dataset form. Preserved verbatim. + _warn(f"view '{vname}'", + "no `schema`/`sql` source (an extends-only view?); " + "preserved in custom_extensions only") + unmapped_views.add(vname) + extra_files[fname] = text + continue + if vname in views: + raise ConversionError( + f"two view files resolve to view '{vname}' " + f"('{view_meta[vname][0]}' and '{fname}')") + views[vname] = parsed + view_meta[vname] = (fname, mv.group(1)) + continue + mt = _TOPIC_FILE_RE.search(fname) + if mt: + topics[mt.group(1)] = load_yaml(text, fname) or {} + topic_paths[mt.group(1)] = fname + continue + if _MODEL_FILE_RE.search(fname): + model_yaml = load_yaml(text, fname) + continue + if _RELS_FILE_RE.search(fname): + parsed = load_yaml(text, fname) or [] + if not isinstance(parsed, list): + raise ConversionError( + f"'{fname}' must be a top-level YAML list of joins") + rel_entries = parsed + continue + _warn(f"file '{fname}'", "unrecognized file; preserved in " + "custom_extensions only") + extra_files[fname] = text + + if not views: + raise ConversionError( + "no convertible view files (*.view.yaml with a schema/sql source) " + "found; nothing to convert") + + # The mapped topic supplies the OSI model's name/description/ai_context. + mapped_name = None + if topic is not None: + if topic not in topics: + raise ConversionError( + f"requested topic '{topic}' not found; topics present: " + f"{sorted(topics) or 'none'}") + mapped_name = topic + elif len(topics) == 1: + mapped_name = next(iter(topics)) + elif len(topics) > 1: + _warn("model", f"{len(topics)} topics found and none chosen with --topic; " + f"topic metadata is preserved in custom_extensions only") + + model = {"name": model_name or mapped_name or "omni_model"} + + mapped_topic = topics.get(mapped_name, {}) + if mapped_topic.get("description"): + model["description"] = mapped_topic["description"] + ai = {} + if mapped_topic.get("ai_context"): + ai["instructions"] = mapped_topic["ai_context"] + if ai: + model["ai_context"] = ai + + datasets = [] + for vname, view in views.items(): + datasets.append(_convert_view(vname, view, view_meta[vname])) + model["datasets"] = datasets + + # A join OSI cannot represent -- one touching a view with no OSI dataset (a + # query view, an extends-only view), or a non-equi/cross join -- is stashed + # verbatim with its position, so export rebuilds relationships.yaml in the + # original order. Malformed entries still raise. + relationships, extra_rels = [], [] + rel_names = set() + for i, entry in enumerate(rel_entries): + endpoints = {entry.get("join_from_view"), entry.get("join_to_view")} + rel = None + if endpoints & unmapped_views: + _warn(f"relationship #{i + 1}", + "references a view with no OSI dataset form; preserved in " + "custom_extensions only") + else: + rel = _convert_relationship(entry, i, views) + if rel is None: + extra_rels.append({"index": i, "entry": entry}) + continue + # OSI relationship names are unique per model; several (aliased) joins + # between one view pair generate the same `_to_` -- suffix + # the repeats. Export never reads the name, so this stays lossless. + base, n, k = rel["name"], rel["name"], 2 + while n in rel_names: + n, k = f"{base}_{k}", k + 1 + rel["name"] = n + rel_names.add(n) + relationships.append(rel) + if relationships: + model["relationships"] = relationships + + base_view = mapped_topic.get("base_view") + metrics = _convert_measures(views, relationships, base_view) + if metrics: + model["metrics"] = metrics + + # Model-level stash: the model file and topics verbatim (minus natively + # mapped topic properties), the mapped topic's identity, the topic's base + # view (so export re-roots the generated join tree identically), and any + # unconvertible files. `topics` is stashed even when empty so a lossless + # re-export does not invent a topic the original model never had. + stash = {"topics": {}} + if model_yaml is not None: + stash["model_file"] = model_yaml + for tname, tdict in topics.items(): + tdict = dict(tdict) + if tname == mapped_name: + tdict.pop("description", None) + tdict.pop("ai_context", None) + stash["topics"][tname] = tdict + topic_files = {t: p for t, p in topic_paths.items() + if p != f"{TOPIC_DIR}/{t}.topic.yaml"} + if topic_files: + stash["topic_files"] = topic_files + if mapped_name is not None: + stash["mapped_topic"] = mapped_name + if base_view: + if base_view not in views: + _warn(f"topic '{mapped_name}'", + f"base_view '{base_view}' is not a view in this model") + else: + stash["base_view"] = base_view + if extra_rels: + stash["extra_relationships"] = extra_rels + if extra_files: + stash["extra_files"] = extra_files + write_stash(model, stash) + + return dump_yaml({"version": OSI_VERSION, "semantic_model": [model]}) + + +def _convert_view(vname, view, meta): + scope = f"view '{vname}'" + fname, basename = meta + ds = {"name": vname} + stash = {} + + # An implicit table_name is the *file's* name -- not the canonical view + # name, which is schema-qualified for a view outside the default schema. + source = join_source(dict(view, table_name=view.get("table_name", basename))) + ds["source"] = source + if str(view.get("table_name", "")) == basename: + # Explicit-but-redundant table_name: the exporter would normalize it + # away, so remember it was spelled out. + stash["table_name"] = view["table_name"] + + if view.get("description"): + ds["description"] = view["description"] + if view.get("ai_context"): + ds["ai_context"] = {"instructions": view["ai_context"]} + + fields = [] + pk_cols = [] + # Omni writes compound-key entries as `${view.field}`/`${field}` references; + # resolve same-view references to plain field names (the original list is + # stashed whenever this normalization changes it). + raw_compound = view.get("custom_compound_primary_key_sql") or [] + compound = [] + for c in raw_compound: + translated, _ = omni_sql_to_osi(str(c), vname) + translated = translated.strip() + compound.append(translated if is_simple_identifier(translated) else str(c)) + # Omni mustache templating ({{# field.filter }} ...) in a field's sql has + # no SQL (or OSI) form at all; such dimensions/measures are stashed whole + # and dropped from the OSI model. Popped here so the later measure pass + # sees only convertible measures. + for kind, key in (("dimensions", "extra_dimensions"), + ("measures", "extra_measures")): + entries = view.get(kind) or {} + templated = {n: e for n, e in entries.items() + if "{{" in str((e or {}).get("sql", ""))} + if templated: + for n in templated: + _warn(f"{kind[:-1]} '{vname}.{n}'", + "sql uses Omni template syntax ('{{'), which has no " + "OSI form; preserved in custom_extensions only") + entries.pop(n) + stash[key] = templated + + dims = view.get("dimensions") or {} + covered = set() # names the exporter can resolve back to a dimension + for dname, dim in dims.items(): + dim = dim or {} + field, col = _convert_dimension(vname, dname, dim) + fields.append(field) + covered.update((dname, col)) + if dim.get("primary_key"): + pk_cols.append(col) + if dname in compound: + compound = [col if c == dname else c for c in compound] + if fields: + ds["fields"] = fields + + # Stash the original compound-key list whenever the exporter could not + # rebuild it from the OSI primary_key alone (a `${view.field}` reference + # that was normalized away, or an entry no dimension covers). + if raw_compound and ( + compound != [str(c) for c in raw_compound] + or any(str(c) not in covered for c in compound)): + stash["custom_compound_primary_key_sql"] = list(raw_compound) + + if compound: + unknown = [c for c in compound if not is_simple_identifier(str(c))] + if unknown: + _warn(scope, f"custom_compound_primary_key_sql entries {unknown} are not " + f"plain field names; using them as-is in primary_key") + ds["primary_key"] = [str(c) for c in compound] + if pk_cols: + _warn(scope, "both a primary_key dimension and " + "custom_compound_primary_key_sql found; using the compound key") + elif len(pk_cols) == 1: + ds["primary_key"] = pk_cols + elif len(pk_cols) > 1: + # Multiple primary_key dimensions form a composite key in Omni. + ds["primary_key"] = pk_cols + + if fname != view_file(vname): + stash["file"] = fname + extras = {k: v for k, v in view.items() if k not in _VIEW_NATIVE_KEYS} + for key in ("description", "ai_context"): + # Present-but-empty metadata has no OSI slot; preserve it as an extra. + if key in view and not view[key]: + extras[key] = view[key] + if extras: + stash["view_extras"] = extras + write_stash(ds, stash) + return ds + + +def _convert_dimension(vname, dname, dim): + """Build one OSI field from an Omni dimension. Returns (field, column) where + `column` is the underlying column used for key resolution (the translated + expression when it is a bare column, else the dimension name).""" + scope = f"dimension '{vname}.{dname}'" + stash = {} + + sql = dim.get("sql") + if sql is None: + expr = dname # schema-layer default: the same-named physical column + else: + sql = str(sql) + expr, changed = omni_sql_to_osi(sql, vname) + if has_timeframe_ref(sql): + _warn(scope, "timeframe reference (${view.field[timeframe]}) has no OSI " + "form; flattened to the base field, original sql stashed") + if changed or sql.strip() == dname: + # Stashed when the OSI expression differs from the Omni sql, and + # also when the sql is an explicit same-named bare column -- which + # the exporter would otherwise normalize to the implicit + # schema-layer default (no `sql:` key). + stash["sql"] = sql + + field = { + "name": dname, + "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, + } + if dim.get("label"): + field["label"] = dim["label"] + if dim.get("description"): + field["description"] = dim["description"] + ai = {} + if dim.get("ai_context"): + ai["instructions"] = dim["ai_context"] + if dim.get("synonyms"): + ai["synonyms"] = list(dim["synonyms"]) + if ai: + field["ai_context"] = ai + if "timeframes" in dim: + field["dimension"] = {"is_time": True} + stash["timeframes"] = dim["timeframes"] + + for key, value in dim.items(): + if key == "timeframes": + continue + if key in _DIM_NATIVE_KEYS: + # A present-but-empty native value (description: '') has no OSI + # slot -- OSI omits empty metadata -- so it rides in the stash. + if not value and not isinstance(value, bool) and key != "sql": + stash[key] = value + continue + stash[key] = value + + write_stash(field, stash) + column = expr.strip() if is_simple_identifier(expr) else dname + return field, column + + +def _convert_relationship(entry, index, views): + what = f"relationship #{index + 1}" + from_view = require_str(entry, "join_from_view", what) + to_view = require_str(entry, "join_to_view", what) + for v in (from_view, to_view): + if v not in views: + raise ConversionError(f"{what}: view '{v}' has no view file") + on_sql = require_str(entry, "on_sql", what) + + # Aliased joins reference the alias in on_sql; accept those names too. + aliases = { + entry.get("join_from_view_as") or from_view: from_view, + entry.get("join_to_view_as") or to_view: to_view, + from_view: from_view, + to_view: to_view, + } + + from_cols, to_cols = [], [] + for clause in re.split(r"\s+AND\s+", on_sql, flags=re.IGNORECASE): + m = _ON_CLAUSE_RE.match(clause) + if not m: + # A valid Omni join OSI cannot express (a range/non-equi or cross + # join). The caller stashes the entry verbatim. + _warn(what, f"('{from_view}' -> '{to_view}'): on_sql clause " + f"'{clause.strip()}' is not an equi-join of two " + f"${{view.field}} references, so it has no OSI " + f"relationship form; preserved in custom_extensions only") + return None + la, lf, ra, rf = m.groups() + if aliases.get(la) == from_view and aliases.get(ra) == to_view: + from_cols.append(_field_column(views[from_view], lf)) + to_cols.append(_field_column(views[to_view], rf)) + elif aliases.get(la) == to_view and aliases.get(ra) == from_view: + from_cols.append(_field_column(views[from_view], rf)) + to_cols.append(_field_column(views[to_view], lf)) + else: + _warn(what, f"on_sql clause '{clause.strip()}' references views other " + f"than '{from_view}'/'{to_view}' (or their aliases); " + f"preserved in custom_extensions only") + return None + + # The declared type/join_type are stashed verbatim whenever present -- even + # when they restate an Omni default -- so export reproduces the exact file. + stash = {} + # Export rebuilds on_sql from the OSI columns in canonical form; when that + # would not reproduce the original (an alias reference, reversed clause + # sides, `and` casing, spacing, a field-to-column translation), the + # original rides in the stash instead. + rebuilt = " AND ".join( + "${" + f"{from_view}.{fc}" + "} = ${" + f"{to_view}.{tc}" + "}" + for fc, tc in zip(from_cols, to_cols)) + if rebuilt != on_sql: + stash["on_sql"] = on_sql + rel_type = str(entry.get("relationship_type") or REL_MANY_TO_ONE) + if "relationship_type" in entry: + stash["relationship_type"] = rel_type + if rel_type == REL_ONE_TO_MANY: + # OSI `from` is always the many side; flip the orientation (the stashed + # type tells export to flip back to the original one_to_many join). + from_view, to_view = to_view, from_view + from_cols, to_cols = to_cols, from_cols + elif rel_type == "many_to_many": + # one_to_one / many_to_many / assumed_many_to_one keep the declared + # orientation. + _warn(what, "many_to_many has no OSI orientation (OSI `to` is the one " + "side); orientation kept as declared, type preserved in " + "custom_extensions") + + rel = {"name": f"{from_view}_to_{to_view}", "from": from_view, "to": to_view, + "from_columns": from_cols, "to_columns": to_cols} + + if entry.get("join_type"): + stash["join_type"] = entry["join_type"] + if entry.get("where_sql"): + stash["where_sql"] = entry["where_sql"] + _warn(what, "where_sql (join filter) has no OSI form; preserved in " + "custom_extensions only") + for key in ("reversible", "id", "join_from_view_as", "join_from_view_as_label", + "join_to_view_as", "join_to_view_as_label"): + if key in entry: + stash[key] = entry[key] + write_stash(rel, stash) + return rel + + +def _field_column(view, fname): + """Resolve a ${view.field} reference to the field's underlying column when the + dimension is a bare column, else keep the field name.""" + dim = (view.get("dimensions") or {}).get(fname) or {} + sql = dim.get("sql") + if sql is None: + return fname + return sql.strip() if is_simple_identifier(str(sql)) else fname + + +def _convert_measures(views, relationships, base_view): + """Turn every view's measures into OSI model-level metrics. + + A metric name is the measure name when globally unique, else + `__` (the original name is stashed either way when it + matters). The full original measure is stashed whenever the OSI expression + alone cannot reconstruct it exactly. + """ + # The exporter re-derives measure placement from the expression (a + # view-qualified aggregate lands on that view, everything else on the base + # view). Compute that default here so placement is only stashed when needed. + effective_base = base_view or _fk_sink(views, relationships) + + counts = {} + for vname, view in views.items(): + for mname in (view.get("measures") or {}): + counts[mname] = counts.get(mname, 0) + 1 + + metrics = [] + seen = set() + for vname, view in views.items(): + for mname, measure in (view.get("measures") or {}).items(): + measure = measure or {} + metric_name = mname if counts[mname] == 1 else f"{vname}__{mname}" + if metric_name in seen: + raise ConversionError( + f"metric name '{metric_name}' derived twice; rename the " + f"colliding measures in Omni") + seen.add(metric_name) + metrics.append( + _convert_measure(vname, mname, metric_name, measure, effective_base)) + return metrics + + +def _convert_measure(vname, mname, metric_name, measure, effective_base): + scope = f"measure '{vname}.{mname}'" + stash = {} + + agg = measure.get("aggregate_type") + sql = measure.get("sql") + expr = None + # Reconstructible = the exporter can rebuild this measure from the OSI + # expression alone: only natively-mapped keys, and a sql that is either free + # of `${...}` references or a lone same-view field reference (which the + # exporter re-derives from the view's dimension names). Anything else keeps + # the original measure in the stash. + reconstructible = set(measure) <= _MEASURE_NATIVE_KEYS and ( + sql is None + or "${" not in str(sql) + or re.fullmatch(r"\s*\$\{\s*[A-Za-z_]\w*\s*\}\s*", str(sql)) is not None + ) + + if agg is None and sql is not None: + # A raw-SQL measure (the aggregate is written out in the sql). OSI + # metrics are model-level, so view qualifiers are kept, not stripped + # (own_view=None below leaves `${view.field}` as `view.field`). + # Always stashed: the exporter would otherwise re-parse a lone + # aggregate call into a structured measure, changing the file. + expr, _ = omni_sql_to_osi(str(sql), own_view=None) + reconstructible = False + elif agg in _SIMPLE_AGGS: + if agg == "count" and sql is None: + expr = "COUNT(*)" + elif sql is None: + raise ConversionError(f"{scope}: aggregate_type '{agg}' requires sql") + else: + inner = _measure_operand(vname, str(sql)) + if agg == "count_distinct": + expr = f"COUNT(DISTINCT {inner})" + else: + expr = f"{_SIMPLE_AGGS[agg]}({inner})" + elif agg in _EXOTIC_AGGS: + # Best-effort ANSI so OSI consumers still see a metric; the verbatim + # measure is stashed, so re-export is exact. + inner = _measure_operand(vname, str(sql or "")) + expr = _exotic_expr(agg, inner, measure) + _warn(scope, f"aggregate_type '{agg}' has no exact ANSI form; emitted a " + f"best-effort expression, original measure preserved in " + f"custom_extensions") + reconstructible = False + else: + raise ConversionError( + f"{scope}: unknown aggregate_type '{agg}'") + + if measure.get("filters"): + _warn(scope, "measure filters have no OSI form; expression emitted " + "unfiltered, original measure preserved in custom_extensions") + reconstructible = False + + # Placement/name recovery: only stashed when the exporter would otherwise + # derive a different view or name. + derived_target = _derived_placement(expr, effective_base) + if not reconstructible: + stashed_measure = { + k: v for k, v in measure.items() + if k not in ("description", "synonyms", "ai_context") + } + stash["measure"] = stashed_measure + stash["view"] = vname + if metric_name != mname: + stash["name"] = mname + else: + if derived_target != vname: + stash["view"] = vname + if metric_name != mname: + stash["name"] = mname + + metric = { + "name": metric_name, + "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, + } + if measure.get("description"): + metric["description"] = measure["description"] + ai = {} + if measure.get("ai_context"): + ai["instructions"] = measure["ai_context"] + if measure.get("synonyms"): + ai["synonyms"] = list(measure["synonyms"]) + if ai: + metric["ai_context"] = ai + write_stash(metric, stash) + return metric + + +def _measure_operand(vname, sql): + """Translate a measure's operand sql to an OSI reference: a same-view field + or bare column becomes `view.name` (the qualified form OSI metrics use); + anything else keeps its view qualifiers and is left as-is.""" + inner, _ = omni_sql_to_osi(sql, own_view=None) + inner = inner.strip() + if is_simple_identifier(inner): + return f"{vname}.{inner}" + return inner + + +def _derived_placement(expr, effective_base): + """Mirror the exporter's placement rule: a metric lands on the single view + its expression references, else on the base view.""" + refs = set(re.findall(r"(?.view.yaml` per dataset, a +`relationships.yaml` join list, one generated `topics/.topic.yaml` +(base view chosen like a fact table), and -- when a prior import stashed one -- +the original `model.yaml`. See README.md for the capability summary. + +Usage (CLI): + osi-omni export -i model.yaml -o omni_model/ [--base-view orders] [--dialect SNOWFLAKE] +""" + +import re +import warnings + +from ._common import ( + ConversionError, + DEFAULT_TIMEFRAMES, + MODEL_FILE, + OSI_VERSION, + REL_MANY_TO_ONE, + REL_ONE_TO_MANY, + RELATIONSHIPS_FILE, + dump_yaml, + foreign_vendor_extensions, + instructions_of, + is_simple_identifier, + load_yaml, + osi_expr_refs_to_omni, + parse_source, + pick_expression, + read_stash, + require_str, + sanitize_name, + synonyms_of, + topic_file, + view_file, + write_stash, # noqa: F401 (re-exported for symmetry in tests) +) + + +def _warn(scope, msg): + warnings.warn(f"[{scope}] {msg}") + + +# Dimension-level stash keys restored verbatim onto the exported dimension. +# `sql` is handled separately (it replaces the derived expression). +_DIM_STASH_PASSTHROUGH_EXCLUDE = {"sql"} + +# One OSI metric aggregate call maps to a structured Omni measure. +_AGG_TO_OMNI = { + "SUM": "sum", + "COUNT": "count", + "AVG": "average", + "MIN": "min", + "MAX": "max", + "MEDIAN": "median", +} + +_AGG_CALL_RE = re.compile( + r"^\s*(SUM|COUNT|AVG|MIN|MAX|MEDIAN)\s*\((.*)\)\s*$", + re.IGNORECASE | re.DOTALL, +) + +# `view.column` -- a dotted reference an OSI metric uses to point into a dataset. +_DOTTED_REF_RE = re.compile( + r"(? 1: + _warn("model", "multiple semantic models found; converting only the first") + + return _convert_model(models[0], base_view, dialect) + + +def _convert_model(model, explicit_base_view, dialect): + name = model.get("name", "") + dataset_list = model.get("datasets", []) or [] + if not dataset_list: + raise ConversionError(f"Model '{name}' has no datasets") + + # Dataset -> Omni view names. Sanitization collisions (and case-insensitive + # duplicates, which sanitize identically) fail loudly rather than merging. + view_names = {} + taken = set() + for d in dataset_list: + ds_name = require_str(d, "name", f"Model '{name}': dataset") + view_names[ds_name] = sanitize_name(ds_name, f"Model '{name}': dataset", taken) + taken.add(view_names[ds_name].lower()) + datasets = {d["name"]: d for d in dataset_list} + relationships = model.get("relationships", []) or [] + for rel in relationships: + scope = f"Model '{name}': relationship '{rel.get('name', '')}'" + if (require_str(rel, "from", scope) not in datasets + or require_str(rel, "to", scope) not in datasets): + raise ConversionError(f"{scope} references an unknown dataset") + + model_stash = read_stash(model) + + files = {} + + # model.yaml: only a stashed original (a fresh model needs no model file; + # model-wide Omni settings have no OSI source to generate from). + if model_stash.get("model_file") is not None: + files[MODEL_FILE] = dump_yaml(model_stash["model_file"]) + + # Views (and the per-view dimension name maps the other stages need). A + # stashed original file path (e.g. `DELIGHTED/response.view`) wins over the + # canonical `views/.view.yaml` layout. + dims_by_view = {} + view_paths = {} + for ds_name, ds in datasets.items(): + vname = view_names[ds_name] + view, dim_names = _convert_dataset(ds, vname, view_names, dialect) + view_paths[vname] = read_stash(ds).get("file") or view_file(vname) + files[view_paths[vname]] = dump_yaml(view) + dims_by_view[vname] = dim_names + + # Relationships: converted OSI relationships in order, with any joins a + # prior import could not map (they touch a query/extends view) reinserted + # verbatim at their original positions. + imported = "topics" in model_stash + rel_entries = [ + _convert_relationship(rel, view_names, imported) for rel in relationships + ] + for item in sorted(model_stash.get("extra_relationships") or [], + key=lambda x: x.get("index", 0)): + rel_entries.insert(min(item.get("index", 0), len(rel_entries)), + item["entry"]) + if rel_entries: + files[RELATIONSHIPS_FILE] = dump_yaml(rel_entries) + + # Base view: explicit flag > stashed original > FK-sink heuristic. Resolved + # lazily -- a model whose topics are stashed and whose metrics all carry (or + # derive) their own placement never needs one, so the multi-root error only + # fires when a base view is genuinely required. + base_hint = explicit_base_view or model_stash.get("base_view") + base_cache = [] + + def resolve_base(): + if not base_cache: + base_cache.append( + view_names[_pick_base_view(name, datasets, relationships, base_hint)]) + return base_cache[0] + + # Measures (from OSI metrics) attach to views. + measures_by_view = {} + for metric in model.get("metrics", []) or []: + placed = _convert_metric(metric, resolve_base, view_names, dims_by_view, + dialect) + if placed is None: + continue + target_view, mname, measure = placed + target = measures_by_view.setdefault(target_view, {}) + if mname.lower() in {m.lower() for m in target}: + raise ConversionError( + f"Model '{name}': two metrics map to measure '{mname}' on view " + f"'{target_view}'; rename one in the OSI model.") + target[mname] = measure + for vname, measures in measures_by_view.items(): + view = load_yaml(files[view_paths[vname]], f"view '{vname}'") or {} + clashes = set(measures) & set(view.get("dimensions") or {}) + for c in sorted(clashes): + _warn(f"measure '{c}'", + f"name collides with a dimension on view '{vname}'; Omni requires " + f"unique field names per view -- rename before use") + existing = view.setdefault("measures", {}) + existing.update(measures) + files[view_paths[vname]] = dump_yaml(view) + + # Topics: stashed originals restore verbatim (with natively-mapped properties + # re-injected on the mapped topic). The `topics` stash key being *present* -- + # even empty -- means the original Omni model's topic set is known, so a + # fresh topic is only generated for hand-authored OSI (no stash). + if "topics" in model_stash: + mapped = model_stash.get("mapped_topic") + topic_paths = model_stash.get("topic_files") or {} + for tname, topic in (model_stash["topics"] or {}).items(): + topic = dict(topic) + if tname == mapped: + if model.get("description"): + topic["description"] = model["description"] + instructions = instructions_of(model.get("ai_context")) + if instructions: + topic["ai_context"] = instructions + files[topic_paths.get(tname) or topic_file(tname)] = dump_yaml(topic) + else: + tname = sanitize_name(name, f"Model '{name}'", set()) + files[topic_file(tname)] = dump_yaml( + _build_topic(model, resolve_base(), view_names, relationships)) + + # Files a prior import could not convert (query views, unrecognized files) + # restore verbatim. + for fname, text in (model_stash.get("extra_files") or {}).items(): + files[fname] = text + + _warn_dropped_model(model) + return files + + +def _convert_dataset(ds, vname, view_names, dialect): + """Build one Omni view dict from an OSI dataset. Returns (view, dims) where + dims = {"cols": {column: dimension name}, "names": set of dimension names} + (used to resolve primary keys and metric references).""" + ds_name = ds["name"] + scope = f"dataset '{ds_name}'" + stash = read_stash(ds) + + view = {} + parsed = parse_source(ds.get("source"), ds_name) + if parsed[0] == "sql": + view["sql"] = parsed[1] + else: + _, catalog, schema, table = parsed + if catalog: + view["catalog"] = catalog + view["schema"] = schema + # table_name defaults to the *file's* name -- the basename of the + # stashed original path when there is one (a schema-folder layout names + # the view `schema__table` but the file `SCHEMA/table.view`), else the + # view name. Emit only when it differs. + default_table = vname + if stash.get("file"): + base = stash["file"].rsplit("/", 1)[-1] + default_table = re.sub(r"\.view(\.ya?ml)?$", "", base) + if "table_name" in stash: # was spelled out even though redundant + view["table_name"] = stash["table_name"] + elif table != default_table: + view["table_name"] = table + if ds.get("description"): + view["description"] = ds["description"] + instructions = instructions_of(ds.get("ai_context")) + if instructions: + view["ai_context"] = instructions + if synonyms_of(ds.get("ai_context")): + _warn(scope, "dataset ai_context synonyms have no Omni view-level home; dropped") + + # Restore stashed Omni view extras (label, hidden, tags, filters:, ...) + # verbatim. Keys the converter derives itself are not stashed on import. + for key, value in (stash.get("view_extras") or {}).items(): + view[key] = value + + dimensions = {} + col_to_dim = {} + taken = set() + for field in ds.get("fields", []) or []: + fname = require_str(field, "name", f"{scope}: field") + fscope = f"field '{fname}'" + dname = sanitize_name(fname, f"{scope}: field", taken) + taken.add(dname.lower()) + expr = pick_expression(field.get("expression"), dialect) + if expr is None: + _warn(fscope, "no usable ANSI_SQL (or preferred-dialect) expression; " + "dropping field") + continue + dim = _convert_field(field, dname, expr, fscope) + dimensions[dname] = dim + if is_simple_identifier(expr): + col_to_dim[expr.strip()] = dname + + # primary_key: mark the matching dimension (creating a hidden one for a key + # column no field covers); a composite key becomes the view-level + # custom_compound_primary_key_sql list of field names -- restored verbatim + # when a prior import stashed the original (`${view.field}`-form) list. + pk = ds.get("primary_key") or [] + if stash.get("custom_compound_primary_key_sql"): + view["custom_compound_primary_key_sql"] = \ + stash["custom_compound_primary_key_sql"] + elif pk: + pk_dims = [] + for col in pk: + # A key entry is either a raw column some dimension covers, or + # already a dimension name (import resolves a computed dimension's + # key to its field name). + dname = col_to_dim.get(col) or (col if col in dimensions else None) + if dname is None: + dname = sanitize_name(col, f"{scope}: primary key column", taken) + taken.add(dname.lower()) + dim = {"hidden": True} + if dname != col: + dim["sql"] = col + dimensions[dname] = dim + col_to_dim[col] = dname + pk_dims.append(dname) + if len(pk_dims) == 1: + dimensions[pk_dims[0]]["primary_key"] = True + else: + view["custom_compound_primary_key_sql"] = pk_dims + + if ds.get("unique_keys"): + # A unique key that merely restates the primary key is redundant, not lost. + extra = [k for k in ds["unique_keys"] if list(k) != list(pk)] + if extra: + _warn(scope, "unique_keys have no Omni home; dropped") + + # Fields a prior import could not convert (Omni template syntax in sql) + # restore verbatim alongside the mapped ones. + dimensions.update(stash.get("extra_dimensions") or {}) + if stash.get("extra_measures"): + view["measures"] = dict(stash["extra_measures"]) + if dimensions: + view["dimensions"] = dimensions + if foreign_vendor_extensions(ds): + _warn(scope, "foreign-vendor custom_extensions dropped") + return view, {"cols": col_to_dim, "names": set(dimensions)} + + +def _convert_field(field, dname, expr, fscope): + stash = read_stash(field) + dim = {} + + # A stashed original Omni `sql` (import rewrote its ${...} refs) wins, so + # Omni -> OSI -> Omni restores the exact expression. + if "sql" in stash: + dim["sql"] = stash["sql"] + elif is_simple_identifier(expr): + if expr.strip() != dname: + dim["sql"] = expr.strip() + # else: the dimension inherits the same-named schema column -- no sql key. + else: + # Raw SQL over the view's own columns is legal in Omni sql. References to + # other views would need ${view.field} form; OSI field expressions are + # dataset-scoped, so this is emitted as-is. + dim["sql"] = expr + + if field.get("label"): + dim["label"] = field["label"] + if field.get("description"): + dim["description"] = field["description"] + syns = synonyms_of(field.get("ai_context")) + if syns: + dim["synonyms"] = syns + instructions = instructions_of(field.get("ai_context")) + if instructions: + dim["ai_context"] = instructions + + if (field.get("dimension") or {}).get("is_time"): + # The stashed list wins even when empty (`timeframes: []` is a real + # Omni value); the default list is only for hand-authored OSI. + dim["timeframes"] = (stash["timeframes"] if "timeframes" in stash + else list(DEFAULT_TIMEFRAMES)) + elif "timeframes" in stash: + dim["timeframes"] = stash["timeframes"] + + for key, value in stash.items(): + if key in ("sql", "timeframes") or key in dim: + continue + dim[key] = value + + if foreign_vendor_extensions(field): + _warn(fscope, "foreign-vendor custom_extensions dropped") + return dim + + +def _convert_relationship(rel, view_names, imported=False): + rname = rel.get("name", "") + scope = f"relationship '{rname}'" + from_cols = rel.get("from_columns") or [] + to_cols = rel.get("to_columns") or [] + if not isinstance(from_cols, list) or not isinstance(to_cols, list) \ + or not from_cols or not to_cols: + raise ConversionError( + f"Relationship '{rname}': from_columns and to_columns are required lists") + if len(from_cols) != len(to_cols): + raise ConversionError( + f"Relationship '{rname}': from_columns ({len(from_cols)}) and " + f"to_columns ({len(to_cols)}) must have the same length") + + stash = read_stash(rel) + from_view, to_view = view_names[rel["from"]], view_names[rel["to"]] + if stash.get("relationship_type") == REL_ONE_TO_MANY: + # The import flipped a one_to_many join so the OSI `from` is the many + # side; flip back to the originally declared orientation. + from_view, to_view = to_view, from_view + from_cols, to_cols = to_cols, from_cols + entry = {"join_from_view": from_view, "join_to_view": to_view} + # Aliased joins (join_*_view_as) are Omni-only; restore before on_sql so the + # stashed on_sql (which may reference the alias) stays consistent. + for key in ("join_from_view_as", "join_from_view_as_label", + "join_to_view_as", "join_to_view_as_label"): + if key in stash: + entry[key] = stash[key] + + if "on_sql" in stash: + entry["on_sql"] = stash["on_sql"] + else: + entry["on_sql"] = " AND ".join( + "${" + f"{from_view}.{fc}" + "} = ${" + f"{to_view}.{tc}" + "}" + for fc, tc in zip(from_cols, to_cols) + ) + # OSI orientation is many(from) -> one(to). The stash holds the declared + # type/join_type verbatim (even an explicit Omni default). An imported join + # with no stashed key genuinely omitted it; fresh OSI states many_to_one. + if "relationship_type" in stash: + entry["relationship_type"] = stash["relationship_type"] + elif not imported: + entry["relationship_type"] = REL_MANY_TO_ONE + if stash.get("join_type"): + entry["join_type"] = stash["join_type"] + for key in ("reversible", "where_sql", "id"): + if key in stash: + entry[key] = stash[key] + + if rel.get("ai_context"): + _warn(scope, "relationship ai_context has no Omni home; dropped") + if foreign_vendor_extensions(rel): + _warn(scope, "foreign-vendor custom_extensions dropped") + return entry + + +def _pick_base_view(model_name, datasets, relationships, hint): + """Choose the topic's base view: an explicit hint if given, else the dataset + that is never a relationship `to` (the FK sink of a many-to-one star).""" + if hint is not None: + if hint not in datasets: + raise ConversionError( + f"Model '{model_name}': requested base view '{hint}' is not a dataset") + return hint + if len(datasets) == 1: + return next(iter(datasets)) + if not relationships: + raise ConversionError( + f"Model '{model_name}': {len(datasets)} datasets but no relationships; " + f"name the topic's base view with --base-view.") + incoming = {name: 0 for name in datasets} + for rel in relationships: + incoming[rel["to"]] += 1 + roots = [n for n in datasets if incoming[n] == 0] + if not roots: + raise ConversionError( + f"Model '{model_name}': every dataset is a relationship target (the " + f"graph has a cycle); name the topic's base view with --base-view.") + if len(roots) > 1: + raise ConversionError( + f"Model '{model_name}': multiple candidate base views {sorted(roots)}; " + f"name the topic's base view with --base-view.") + return roots[0] + + +def _build_topic(model, base_vname, view_names, relationships): + """Generate the topic for a fresh export: base view + the nested join map of + every view reachable from it, plus the model's description/AI context.""" + topic = {"base_view": base_vname} + if model.get("description"): + topic["description"] = model["description"] + instructions = instructions_of(model.get("ai_context")) + if instructions: + topic["ai_context"] = instructions + + # BFS the (undirected) relationship graph out from the base view; each view + # joins once, on its first-discovered (shortest) path. Views left unreachable + # simply stay out of the topic -- they are still exported and joinable, and + # every relationship remains in relationships.yaml either way. + adj = {} + for rel in relationships: + a, b = view_names[rel["from"]], view_names[rel["to"]] + adj.setdefault(a, []).append(b) + adj.setdefault(b, []).append(a) + children = {base_vname: {}} + seen = {base_vname} + queue = [base_vname] + while queue: + cur = queue.pop(0) + for neighbor in adj.get(cur, []): + if neighbor not in seen: + seen.add(neighbor) + children[cur][neighbor] = {} + children[neighbor] = children[cur][neighbor] + queue.append(neighbor) + if children[base_vname]: + topic["joins"] = children[base_vname] + return topic + + +def _convert_metric(metric, resolve_base, view_names, dims_by_view, dialect): + """Map one OSI metric to an Omni measure. Returns (view_name, measure_name, + measure_dict), or None when the metric has no usable expression.""" + mname_raw = require_str(metric, "name", "metric") + scope = f"metric '{mname_raw}'" + stash = read_stash(metric) + # The import qualifies a colliding measure name as `__` and + # stashes the original; restore it, else sanitize the OSI name. + mname = stash.get("name") or sanitize_name(mname_raw, scope, set()) + + if "measure" in stash: + # A prior import stashed the original Omni measure (a filtered, exotic, + # or otherwise non-reconstructible one) -- restore it verbatim and + # re-inject the natively-mapped properties. + measure = dict(stash["measure"]) + _apply_metric_metadata(metric, measure) + return stash.get("view") or resolve_base(), mname, measure + + expr = pick_expression(metric.get("expression"), dialect) + if expr is None: + _warn(scope, "no usable ANSI_SQL (or preferred-dialect) expression; " + "dropping metric") + return None + + sanitized_views = set(view_names.values()) + referenced = { + m.group(1) + for m in _DOTTED_REF_RE.finditer(expr) + if m.group(1) in sanitized_views + } + + measure = None + target = None + m = _AGG_CALL_RE.match(expr) + if m and _balanced(m.group(2)): + func, inner = m.group(1).upper(), m.group(2).strip() + distinct = re.match(r"(?i)^DISTINCT\s+(.+)$", inner, re.DOTALL) + if func == "COUNT" and distinct: + func, inner = "COUNT_DISTINCT", distinct.group(1).strip() + agg = "count_distinct" if func == "COUNT_DISTINCT" else _AGG_TO_OMNI[func] + if func == "COUNT" and inner == "*": + measure, target = {"aggregate_type": "count"}, None + else: + dotted = _DOTTED_REF_RE.fullmatch(inner) + if dotted and dotted.group(1) in sanitized_views: + # AGG(view.name): a structured measure on that view. `name` may be + # a modeled field (referenced as ${name}) or a raw column covered + # by a field; only an unmodeled name stays a raw column ref. + vname, ref = dotted.group(1), dotted.group(2) + dims = dims_by_view.get(vname) or {"cols": {}, "names": set()} + dim = dims["cols"].get(ref) or (ref if ref in dims["names"] else None) + measure = {"sql": "${" + dim + "}" if dim else ref, + "aggregate_type": agg} + target = vname + elif not referenced: + # AGG over the base view's own columns (bare or computed). + measure = {"sql": inner, "aggregate_type": agg} + target = None + elif len(referenced) == 1: + measure = {"sql": osi_expr_refs_to_omni(inner, sanitized_views), + "aggregate_type": agg} + target = next(iter(referenced)) + + if measure is None: + # Anything else (a ratio, a multi-view aggregate, window SQL) becomes a + # raw-SQL measure; `view.col` references switch to `${view.col}` form. + target = next(iter(referenced)) if len(referenced) == 1 else None + measure = {"sql": osi_expr_refs_to_omni(expr, sanitized_views)} + if len(referenced) > 1: + _warn(scope, f"expression spans views {sorted(referenced)}; emitted as a " + f"raw-SQL measure on the base view -- verify the join path") + + # Stashed placement (from import) wins; else the derived view; else base. + target = stash.get("view") or target or resolve_base() + _apply_metric_metadata(metric, measure) + if foreign_vendor_extensions(metric): + _warn(scope, "foreign-vendor custom_extensions dropped") + return target, mname, measure + + +def _apply_metric_metadata(metric, measure): + if metric.get("description"): + measure["description"] = metric["description"] + syns = synonyms_of(metric.get("ai_context")) + if syns: + measure["synonyms"] = syns + instructions = instructions_of(metric.get("ai_context")) + if instructions: + measure["ai_context"] = instructions + + +def _balanced(s): + depth = 0 + for ch in s: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth < 0: + return False + return depth == 0 + + +def _warn_dropped_model(model): + if foreign_vendor_extensions(model): + _warn("model", "foreign-vendor custom_extensions dropped") + ai = model.get("ai_context") + if isinstance(ai, dict): + if ai.get("synonyms"): + _warn("model", "model ai_context synonyms have no Omni home; dropped") + if ai.get("examples"): + _warn("model", "model ai_context examples have no Omni home (Omni " + "sample_queries need a full query definition); dropped") diff --git a/converters/omni/tests/_roundtrip_helpers.py b/converters/omni/tests/_roundtrip_helpers.py new file mode 100644 index 0000000..bf003af --- /dev/null +++ b/converters/omni/tests/_roundtrip_helpers.py @@ -0,0 +1,361 @@ +"""Shared model builders and round-trip assertions for property-based tests. + +This module is deliberately free of any third-party test dependency (no +hypothesis, no pytest) so the generation + assertion logic can run two ways: + + - driven by Hypothesis strategies (see test_roundtrip_properties.py), and + - driven by a plain seeded `random.Random` (RandomRnd below), which is how the + logic is exercised in environments where hypothesis is not installed. + +Both drivers implement the small `Rnd` interface (chance/count/pick/text/colname); +the builders below depend only on that interface, so the generated model space is +identical regardless of driver. + +The builders intentionally generate within the *round-trippable subset* -- the +shapes the converter reproduces exactly. Known normalizations are avoided by +construction (documented inline), e.g.: + - Omni names are generated already valid (lowercase snake_case), so the + sanitizer never renames anything; + - OSI metric expressions reference *field names* (`view.field`), which survive + the `${field}` modeled-reference trip; a raw column that differs from its + field's name would come back as the field name; + - OSI relationship names use the canonical `_to_` form the importer + regenerates. +Name fuzzing (collisions, reserved words) is left to the targeted unit tests, +which assert the converter *rejects* or *warns on* those inputs. +""" + +import random +import string +import warnings + +from osi_omni import convert_omni_to_osi, convert_osi_to_omni +from osi_omni._common import OSI_VERSION, dump_yaml, load_yaml + +from _util import strip_normalized + +_AGGS = ["sum", "average", "min", "max", "median", "count_distinct"] +_OSI_AGGS = ["SUM", "AVG", "MIN", "MAX", "MEDIAN"] + + +# --- Rnd backend for offline (no hypothesis) runs -------------------------------- + +class RandomRnd: + """The `Rnd` interface backed by a seeded `random.Random`.""" + + def __init__(self, seed): + self.r = random.Random(seed) + + def chance(self, p=0.5): + return self.r.random() < p + + def count(self, lo, hi): + return self.r.randint(lo, hi) + + def pick(self, seq): + return self.r.choice(list(seq)) + + def text(self): + # Alphanumeric with optional interior spaces; no leading/trailing space + # and no YAML-special characters, so the value is preserved verbatim + # through a dump/load cycle. + alnum = string.ascii_letters + string.digits + n = self.r.randint(0, 10) + body = "".join(self.r.choice(alnum + " ") for _ in range(n)) + return (self.r.choice(alnum) + body).strip() or "x" + + def colname(self): + first = self.r.choice(string.ascii_lowercase) + rest = "".join( + self.r.choice(string.ascii_lowercase + string.digits + "_") + for _ in range(self.r.randint(0, 7)) + ) + return first + rest + + +class _Names: + """Hands out globally-unique names with a given prefix.""" + + def __init__(self): + self._n = {} + + def next(self, prefix): + i = self._n.get(prefix, 0) + self._n[prefix] = i + 1 + return f"{prefix}{i}" + + +# --- Omni model builder (for Omni -> OSI -> Omni) --------------------------------- + +def _maybe_meta(rnd, target): + if rnd.chance(0.4): + target["description"] = rnd.text() + if rnd.chance(0.3): + target["label"] = rnd.text() + if rnd.chance(0.3): + target["synonyms"] = [rnd.text() for _ in range(rnd.count(1, 3))] + if rnd.chance(0.2): + target["ai_context"] = rnd.text() + + +def _build_dimensions(rnd, names): + dims = {} + for _ in range(rnd.count(1, 4)): + dname = names.next("d") + dim = {} + flavor = rnd.count(0, 3) + if flavor == 1: + dim["sql"] = rnd.colname() # bare raw column + elif flavor == 2: + dim["sql"] = f'"{rnd.colname().upper()}"' # quoted identifier + elif flavor == 3 and dims: + dim["sql"] = "${" + rnd.pick(list(dims)) + "} + 1" # field ref + _maybe_meta(rnd, dim) + if rnd.chance(0.25): + dim["format"] = rnd.pick(["usdcurrency_2", "number_0", "percent_1", "id"]) + if rnd.chance(0.2): + dim["hidden"] = True + if rnd.chance(0.2): + dim["group_label"] = rnd.text() + if rnd.chance(0.2): + dim["timeframes"] = ["raw", "date", "month"] + dims[dname] = dim + if rnd.chance(0.6): + # primary_key on a dimension whose sql is absent or a bare column. + for dname, dim in dims.items(): + sql = dim.get("sql", "") + if "$" not in sql and '"' not in sql: + dim["primary_key"] = True + break + return dims + + +def _build_measures(rnd, names, dims, shared_measure_name): + measures = {} + for _ in range(rnd.count(0, 3)): + mname = names.next("m") + flavor = rnd.count(0, 4) + if flavor == 0: + m = {"aggregate_type": "count"} + elif flavor == 1 and dims: + m = {"sql": "${" + rnd.pick(list(dims)) + "}", + "aggregate_type": rnd.pick(_AGGS)} + elif flavor == 2: + m = {"sql": rnd.colname(), "aggregate_type": rnd.pick(_AGGS)} + elif flavor == 3: + m = {"sql": f"SUM({rnd.colname()}) / 100"} # raw-SQL measure + else: + m = {"sql": rnd.colname(), "aggregate_type": "percentile", + "percentile": rnd.pick([50, 75, 95])} + if rnd.chance(0.3): + m["description"] = rnd.text() + if rnd.chance(0.2): + m["synonyms"] = [rnd.text() for _ in range(rnd.count(1, 2))] + if rnd.chance(0.2): + m["format"] = "usdcurrency_0" + if rnd.chance(0.2) and m.get("aggregate_type") == "count": + m["filters"] = {rnd.colname(): {"is": rnd.text()}} + measures[mname] = m + if shared_measure_name and rnd.chance(0.5): + # The same measure name on several views exercises the name-qualification + # (`view__measure`) and stashed-name restore paths. + measures[shared_measure_name] = {"aggregate_type": "count"} + return measures + + +def build_omni(rnd): + """Generate an Omni model ({filename: YAML str}) in the round-trippable subset.""" + names = _Names() + n_views = rnd.count(1, 4) + view_names = [names.next("v") for _ in range(n_views)] + + files = {} + dims_of = {} + for vname in view_names: + view = {"schema": rnd.colname()} + if rnd.chance(0.4): + view["catalog"] = rnd.colname() + if rnd.chance(0.4): + view["table_name"] = rnd.colname().upper() + if rnd.chance(0.3): + view["description"] = rnd.text() + if rnd.chance(0.25): + view["label"] = rnd.text() # stash-only view extra + if rnd.chance(0.2): + view["tags"] = [rnd.colname()] # stash-only view extra + dims = _build_dimensions(rnd, names) + view["dimensions"] = dims + dims_of[vname] = dims + measures = _build_measures(rnd, names, dims, "count") + if measures: + view["measures"] = measures + files[f"views/{vname}.view.yaml"] = dump_yaml(view) + + rels = [] + for vname in view_names[1:]: + n_cols = rnd.count(1, 2) + clauses = [ + "${" + f"{view_names[0]}.{rnd.colname()}" + "} = ${" + f"{vname}.{rnd.colname()}" + "}" + for _ in range(n_cols) + ] + rel = {"join_from_view": view_names[0], "join_to_view": vname, + "on_sql": " AND ".join(clauses), + "relationship_type": rnd.pick( + ["many_to_one", "many_to_one", "one_to_many", "one_to_one"])} + if rnd.chance(0.3): + rel["join_type"] = rnd.pick(["inner", "full_outer"]) + if rnd.chance(0.2): + rel["reversible"] = True + if rnd.chance(0.2): + rel["where_sql"] = "${" + f"{vname}.{rnd.colname()}" + "}" + rels.append(rel) + if rels: + files["relationships.yaml"] = dump_yaml(rels) + + if rnd.chance(0.7): + topic = {"base_view": view_names[0]} + if rnd.chance(0.5): + topic["description"] = rnd.text() + if rnd.chance(0.4): + topic["ai_context"] = rnd.text() + if rnd.chance(0.4): + topic["label"] = rnd.text() + if len(view_names) > 1 and rnd.chance(0.6): + topic["joins"] = {v: {} for v in view_names[1:]} + if rnd.chance(0.3): + topic["default_filters"] = { + f"{view_names[0]}.{rnd.colname()}": {"is": rnd.text()}} + files[f"topics/{names.next('t')}.topic.yaml"] = dump_yaml(topic) + + if rnd.chance(0.4): + files["model.yaml"] = dump_yaml({ + "week_start_day": rnd.pick(["Sunday", "Monday"]), + "included_schemas": [rnd.colname()], + }) + return files + + +# --- OSI model builder (for OSI -> Omni -> OSI) ----------------------------------- + +def _osi_field(rnd, names): + fname = names.next("f") + flavor = rnd.count(0, 2) + if flavor == 0: + expr = fname # column named like the field + elif flavor == 1: + expr = rnd.colname() # renamed bare column + else: + expr = f"{rnd.colname()} || {rnd.colname()}" # computed expression + field = {"name": fname, + "expression": {"dialects": [{"dialect": "ANSI_SQL", + "expression": expr}]}} + if rnd.chance(0.4): + field["description"] = rnd.text() + if rnd.chance(0.3): + field["label"] = rnd.text() + ai = {} + if rnd.chance(0.3): + ai["synonyms"] = [rnd.text() for _ in range(rnd.count(1, 2))] + if rnd.chance(0.2): + ai["instructions"] = rnd.text() + if ai: + field["ai_context"] = ai + if rnd.chance(0.2): + field["dimension"] = {"is_time": True} + return field, expr + + +def build_osi(rnd): + """Generate an OSI model dict in the round-trippable subset.""" + names = _Names() + fact = names.next("fact") + dim_names = [names.next("dim") for _ in range(rnd.count(0, 3))] + + datasets = [] + fields_of = {} + for ds_name in [fact] + dim_names: + ds = {"name": ds_name, + "source": f"{rnd.colname()}.{rnd.colname()}.{rnd.colname()}"} + fields, simple_fields = [], [] + for _ in range(rnd.count(1, 4)): + field, expr = _osi_field(rnd, names) + fields.append(field) + if expr == field["name"]: + simple_fields.append(field["name"]) + if rnd.chance(0.3): + ds["description"] = rnd.text() + ds["fields"] = fields + # A primary key over a field whose column matches its name survives the + # dimension round-trip byte-for-byte. + if simple_fields and rnd.chance(0.5): + ds["primary_key"] = [simple_fields[0]] + datasets.append(ds) + fields_of[ds_name] = [f["name"] for f in fields] + + relationships = [] + for dname in dim_names: + n = rnd.count(1, 2) + relationships.append({ + "name": f"{fact}_to_{dname}", + "from": fact, "to": dname, + "from_columns": [f"fk{i}_{rnd.colname()}" for i in range(n)], + "to_columns": [f"pk{i}_{rnd.colname()}" for i in range(n)], + }) + + metrics = [] + for _ in range(rnd.count(0, 3)): + mname = names.next("metric") + flavor = rnd.count(0, 2) + target = rnd.pick([fact] + dim_names) + if flavor == 0: + expr = "COUNT(*)" + elif flavor == 1 and fields_of[target]: + expr = f"{rnd.pick(_OSI_AGGS)}({target}.{rnd.pick(fields_of[target])})" + else: + expr = f"SUM({fact}.{fields_of[fact][0]}) / COUNT(*)" + metric = {"name": mname, + "expression": {"dialects": [{"dialect": "ANSI_SQL", + "expression": expr}]}} + if rnd.chance(0.4): + metric["description"] = rnd.text() + if rnd.chance(0.3): + metric["ai_context"] = {"synonyms": [rnd.text()]} + metrics.append(metric) + + model = {"name": names.next("model")} + if rnd.chance(0.4): + model["description"] = rnd.text() + if rnd.chance(0.3): + model["ai_context"] = {"instructions": rnd.text()} + model["datasets"] = datasets + if relationships: + model["relationships"] = relationships + if metrics: + model["metrics"] = metrics + return {"version": OSI_VERSION, "semantic_model": [model]} + + +# --- Round-trip assertions ------------------------------------------------------- + +def _quiet(fn, *args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return fn(*args, **kwargs) + + +def assert_omni_roundtrip(files): + """An Omni model survives Omni -> OSI -> Omni with every file identical + (structurally -- YAML key order and formatting aside).""" + osi = _quiet(convert_omni_to_osi, files) + files2 = _quiet(convert_osi_to_omni, osi) + parsed1 = {name: load_yaml(text, name) for name, text in files.items()} + parsed2 = {name: load_yaml(text, name) for name, text in files2.items()} + assert parsed2 == parsed1 + + +def assert_osi_roundtrip(osi): + """An OSI model dict survives OSI -> Omni -> OSI up to the documented + normalizations (see _util.strip_normalized).""" + files = _quiet(convert_osi_to_omni, dump_yaml(osi)) + osi2 = load_yaml(_quiet(convert_omni_to_osi, files)) + assert strip_normalized(osi2) == strip_normalized(osi) diff --git a/converters/omni/tests/_util.py b/converters/omni/tests/_util.py new file mode 100644 index 0000000..f0e2812 --- /dev/null +++ b/converters/omni/tests/_util.py @@ -0,0 +1,116 @@ +"""Shared test helpers: fixture loading and structural normalization.""" + +import copy +import json +import pathlib + +from osi_omni._common import load_yaml # src is on sys.path via conftest.py + +FIXTURES = pathlib.Path(__file__).resolve().parent / "fixtures" +REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] + + +def load_fixture(name): + with open(FIXTURES / name) as fh: + return fh.read() + + +def load_fixture_dir(name): + """Read a fixture Omni model directory as {relative posix path: text}.""" + root = FIXTURES / name + files = {} + for path in sorted(root.rglob("*")): + if path.is_file(): + files[path.relative_to(root).as_posix()] = path.read_text() + return files + + +def parse(yaml_str): + return load_yaml(yaml_str) + + +def parse_files(files): + """Parse every YAML file of an Omni model dict for structural comparison.""" + return {name: load_yaml(text, name) for name, text in files.items()} + + +def canon(obj): + """Deep-copy with every `custom_extensions[].data` JSON string parsed into a + dict, so comparisons are insensitive to JSON key order / whitespace.""" + obj = copy.deepcopy(obj) + + def walk(node): + if isinstance(node, dict): + for ext in node.get("custom_extensions") or []: + if isinstance(ext.get("data"), str): + ext["data"] = json.loads(ext["data"]) + for v in node.values(): + walk(v) + elif isinstance(node, list): + for v in node: + walk(v) + + walk(obj) + return obj + + +def strip_normalized(osi): + """Normalize away what the OSI -> Omni -> OSI trip changes by design, so a + round-trip comparison reflects the documented behavior: + + - `custom_extensions` everywhere: the import adds OMNI stashes (topic set, + timeframe lists, ...) that a hand-authored source model does not carry. + - `unique_keys`: no Omni home (dropped with a warning on export). + - relationship `name`: regenerated as `_to_` on import. + - relationship `ai_context`: no Omni home (dropped with a warning). + - model `ai_context` synonyms/examples: no Omni home; only the + instructions text maps (onto the topic). + - `dimension: {is_time: false}`: equivalent to an absent `dimension` + (only `is_time: true` has an Omni form -- `timeframes`). + - a primary-key column no field covers materializes as a hidden + dimension on export, so it comes back as an extra (stash-only) field. + """ + osi = copy.deepcopy(osi) + for model in osi.get("semantic_model", []): + model.pop("custom_extensions", None) + ai = model.get("ai_context") + if isinstance(ai, dict): + ai.pop("synonyms", None) + ai.pop("examples", None) + if not ai: + model.pop("ai_context") + for ds in model.get("datasets", []): + ds.pop("custom_extensions", None) + ds.pop("unique_keys", None) + ai = ds.get("ai_context") + if isinstance(ai, dict): + ai.pop("synonyms", None) + if not ai: + ds.pop("ai_context") + fields = ds.get("fields", []) or [] + for field in fields: + field.pop("custom_extensions", None) + if field.get("dimension") == {"is_time": False}: + field.pop("dimension") + # Drop backfilled key fields: a bare-column field named after a + # primary_key column, carrying nothing but its expression. + pk = set(ds.get("primary_key") or []) + ds_fields = [ + f for f in fields + if not (f["name"] in pk and set(f) <= {"name", "expression"}) + ] + if ds_fields: + ds["fields"] = ds_fields + else: + ds.pop("fields", None) + for rel in model.get("relationships", []) or []: + rel["name"] = f"{rel['from']}_to_{rel['to']}" + rel.pop("ai_context", None) + rel.pop("custom_extensions", None) + for metric in model.get("metrics", []) or []: + metric.pop("custom_extensions", None) + # Metric order is not semantic; the import regroups metrics by the view + # their measure lives on. + if model.get("metrics"): + model["metrics"].sort(key=lambda m: m["name"]) + return osi diff --git a/converters/omni/tests/conftest.py b/converters/omni/tests/conftest.py new file mode 100644 index 0000000..282f013 --- /dev/null +++ b/converters/omni/tests/conftest.py @@ -0,0 +1,6 @@ +import pathlib +import sys + +# Make the converter modules in ../src importable from the tests. +_SRC = pathlib.Path(__file__).resolve().parent.parent / "src" +sys.path.insert(0, str(_SRC)) diff --git a/converters/omni/tests/fixtures/fixtureA_omni/relationships.yaml b/converters/omni/tests/fixtures/fixtureA_omni/relationships.yaml new file mode 100644 index 0000000..04f7b21 --- /dev/null +++ b/converters/omni/tests/fixtures/fixtureA_omni/relationships.yaml @@ -0,0 +1,4 @@ +- join_from_view: orders + join_to_view: customer + on_sql: ${orders.o_custkey} = ${customer.c_custkey} + relationship_type: many_to_one diff --git a/converters/omni/tests/fixtures/fixtureA_omni/topics/sales.topic.yaml b/converters/omni/tests/fixtures/fixtureA_omni/topics/sales.topic.yaml new file mode 100644 index 0000000..6324461 --- /dev/null +++ b/converters/omni/tests/fixtures/fixtureA_omni/topics/sales.topic.yaml @@ -0,0 +1,5 @@ +base_view: orders +description: Sales orders with customer attributes +ai_context: Use this model for order analysis. +joins: + customer: {} diff --git a/converters/omni/tests/fixtures/fixtureA_omni/views/customer.view.yaml b/converters/omni/tests/fixtures/fixtureA_omni/views/customer.view.yaml new file mode 100644 index 0000000..5706a4d --- /dev/null +++ b/converters/omni/tests/fixtures/fixtureA_omni/views/customer.view.yaml @@ -0,0 +1,7 @@ +catalog: samples +schema: tpch +dimensions: + c_custkey: + primary_key: true + c_name: + description: Customer name diff --git a/converters/omni/tests/fixtures/fixtureA_omni/views/orders.view.yaml b/converters/omni/tests/fixtures/fixtureA_omni/views/orders.view.yaml new file mode 100644 index 0000000..f61582a --- /dev/null +++ b/converters/omni/tests/fixtures/fixtureA_omni/views/orders.view.yaml @@ -0,0 +1,32 @@ +catalog: samples +schema: tpch +description: One row per order +dimensions: + o_orderkey: + description: Order identifier + primary_key: true + o_custkey: {} + o_orderdate: + label: Order Date + synonyms: + - order date + - date + timeframes: + - raw + - date + - week + - month + - quarter + - year + o_totalprice: {} +measures: + total_revenue: + sql: ${o_totalprice} + aggregate_type: sum + description: Total order revenue + synonyms: + - revenue + - sales + order_count: + aggregate_type: count + description: Number of orders diff --git a/converters/omni/tests/fixtures/fixtureA_osi.yaml b/converters/omni/tests/fixtures/fixtureA_osi.yaml new file mode 100644 index 0000000..b6e6863 --- /dev/null +++ b/converters/omni/tests/fixtures/fixtureA_osi.yaml @@ -0,0 +1,83 @@ +# yaml-language-server: $schema=../../../../core-spec/osi-schema.json +# +# Fixture A -- all-native round-trip (OSI -> Omni -> OSI). +# Star schema; every construct maps to a native Omni slot. Documented +# normalizations on the OSI -> Omni -> OSI trip: relationship names are +# regenerated (`_to_`), and unique_keys that merely restate the +# primary key are absorbed by it. + +version: "0.2.0.dev0" + +semantic_model: + - name: sales + description: Sales orders with customer attributes + ai_context: + instructions: "Use this model for order analysis." + datasets: + - name: orders # fact: no incoming relationship -> topic base_view + source: samples.tpch.orders + primary_key: [o_orderkey] + description: One row per order + fields: + - name: o_orderkey + expression: + dialects: + - dialect: ANSI_SQL + expression: o_orderkey + description: Order identifier + - name: o_custkey + expression: + dialects: + - dialect: ANSI_SQL + expression: o_custkey + - name: o_orderdate + expression: + dialects: + - dialect: ANSI_SQL + expression: o_orderdate + label: Order Date + dimension: + is_time: true + ai_context: + synonyms: [order date, date] + - name: o_totalprice + expression: + dialects: + - dialect: ANSI_SQL + expression: o_totalprice + - name: customer + source: samples.tpch.customer + primary_key: [c_custkey] + fields: + - name: c_custkey + expression: + dialects: + - dialect: ANSI_SQL + expression: c_custkey + - name: c_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_name + description: Customer name + relationships: + - name: orders_to_customer + from: orders + to: customer + from_columns: [o_custkey] + to_columns: [c_custkey] + metrics: + - name: total_revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.o_totalprice) + description: Total order revenue + ai_context: + synonyms: [revenue, sales] + - name: order_count + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(*) + description: Number of orders diff --git a/converters/omni/tests/fixtures/fixtureB_omni/model.yaml b/converters/omni/tests/fixtures/fixtureB_omni/model.yaml new file mode 100644 index 0000000..d265896 --- /dev/null +++ b/converters/omni/tests/fixtures/fixtureB_omni/model.yaml @@ -0,0 +1,10 @@ +# Fixture B -- stash round-trip (Omni -> OSI -> Omni, lossless). +# The model file has no OSI home and round-trips via the model-level stash. +included_schemas: +- ecomm +week_start_day: Sunday +access_grants: + pii_access: + user_attribute: role + allowed_values: + - admin diff --git a/converters/omni/tests/fixtures/fixtureB_omni/relationships.yaml b/converters/omni/tests/fixtures/fixtureB_omni/relationships.yaml new file mode 100644 index 0000000..cd91d26 --- /dev/null +++ b/converters/omni/tests/fixtures/fixtureB_omni/relationships.yaml @@ -0,0 +1,12 @@ +# Exercises: non-default join_type, reversible, a composite-key one_to_many +# join (flipped to OSI's many-side-first orientation and flipped back on export). +- join_from_view: order_items + join_to_view: users + on_sql: ${order_items.user_id} = ${users.id} + relationship_type: many_to_one + join_type: inner + reversible: true +- join_from_view: users + join_to_view: orders + on_sql: ${users.id} = ${orders.user_id} AND ${users.region} = ${orders.region} + relationship_type: one_to_many diff --git a/converters/omni/tests/fixtures/fixtureB_omni/topics/order_analysis.topic.yaml b/converters/omni/tests/fixtures/fixtureB_omni/topics/order_analysis.topic.yaml new file mode 100644 index 0000000..57d0eae --- /dev/null +++ b/converters/omni/tests/fixtures/fixtureB_omni/topics/order_analysis.topic.yaml @@ -0,0 +1,15 @@ +# Topic curation (label, joins, default_filters, fields) has no OSI home and +# round-trips via the model-level stash; description/ai_context map natively. +base_view: order_items +label: Order Analysis +description: Line-item order analysis +ai_context: You are an ecommerce analyst. +joins: + users: + orders: {} +default_filters: + users.state: + is: California +fields: +- all_views.* +- -users.state diff --git a/converters/omni/tests/fixtures/fixtureB_omni/views/order_items.view.yaml b/converters/omni/tests/fixtures/fixtureB_omni/views/order_items.view.yaml new file mode 100644 index 0000000..ca84597 --- /dev/null +++ b/converters/omni/tests/fixtures/fixtureB_omni/views/order_items.view.yaml @@ -0,0 +1,46 @@ +# Exercises the dimension/measure stash: format, hidden, group_label, +# timeframes, convert_tz, a quoted-identifier sql, a ${field} reference, a +# filtered measure, and a percentile measure. +schema: ecomm +table_name: ORDER_ITEMS +label: Order Items +description: One row per line item +dimensions: + id: + primary_key: true + format: id + hidden: true + user_id: {} + sale_price: + sql: '"SALE_PRICE"' + format: usdcurrency_2 + group_label: Money + created_at: + timeframes: + - raw + - date + - month + convert_tz: false + full_price: + sql: ${sale_price} * 1.1 + label: Full Price +measures: + count: + aggregate_type: count + total_revenue: + sql: ${sale_price} + aggregate_type: sum + format: usdcurrency_0 + description: Total revenue + synonyms: + - sales + - revenue + completed: + aggregate_type: count + filters: + status: + is: complete + p95_price: + sql: ${sale_price} + aggregate_type: percentile + percentile: 95 diff --git a/converters/omni/tests/fixtures/fixtureB_omni/views/orders.view.yaml b/converters/omni/tests/fixtures/fixtureB_omni/views/orders.view.yaml new file mode 100644 index 0000000..3264995 --- /dev/null +++ b/converters/omni/tests/fixtures/fixtureB_omni/views/orders.view.yaml @@ -0,0 +1,6 @@ +schema: ecomm +dimensions: + order_id: + primary_key: true + user_id: {} + region: {} diff --git a/converters/omni/tests/fixtures/fixtureB_omni/views/users.view.yaml b/converters/omni/tests/fixtures/fixtureB_omni/views/users.view.yaml new file mode 100644 index 0000000..5b42d8b --- /dev/null +++ b/converters/omni/tests/fixtures/fixtureB_omni/views/users.view.yaml @@ -0,0 +1,9 @@ +schema: ecomm +dimensions: + id: + primary_key: true + region: {} + state: {} +measures: + count: + aggregate_type: count diff --git a/converters/omni/tests/fixtures/tpcds_omni/relationships.yaml b/converters/omni/tests/fixtures/tpcds_omni/relationships.yaml new file mode 100644 index 0000000..4a70f2d --- /dev/null +++ b/converters/omni/tests/fixtures/tpcds_omni/relationships.yaml @@ -0,0 +1,16 @@ +- join_from_view: store_sales + join_to_view: date_dim + on_sql: ${store_sales.ss_sold_date_sk} = ${date_dim.d_date_sk} + relationship_type: many_to_one +- join_from_view: store_sales + join_to_view: customer + on_sql: ${store_sales.ss_customer_sk} = ${customer.c_customer_sk} + relationship_type: many_to_one +- join_from_view: store_sales + join_to_view: item + on_sql: ${store_sales.ss_item_sk} = ${item.i_item_sk} + relationship_type: many_to_one +- join_from_view: store_sales + join_to_view: store + on_sql: ${store_sales.ss_store_sk} = ${store.s_store_sk} + relationship_type: many_to_one diff --git a/converters/omni/tests/fixtures/tpcds_omni/topics/tpcds_retail_model.topic.yaml b/converters/omni/tests/fixtures/tpcds_omni/topics/tpcds_retail_model.topic.yaml new file mode 100644 index 0000000..51fcb06 --- /dev/null +++ b/converters/omni/tests/fixtures/tpcds_omni/topics/tpcds_retail_model.topic.yaml @@ -0,0 +1,11 @@ +base_view: store_sales +description: TPC-DS retail semantic model for sales and customer analytics +ai_context: Use this semantic model for retail analytics. It provides comprehensive + sales, customer, product, and store data from the TPC-DS benchmark. The model supports + time-based analysis, customer segmentation, product performance, and store operations + metrics. +joins: + date_dim: {} + customer: {} + item: {} + store: {} diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/customer.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/customer.view.yaml new file mode 100644 index 0000000..c70c8c0 --- /dev/null +++ b/converters/omni/tests/fixtures/tpcds_omni/views/customer.view.yaml @@ -0,0 +1,27 @@ +catalog: tpcds +schema: public +description: Customer dimension with demographic information +dimensions: + c_customer_sk: + description: Surrogate key for customer + primary_key: true + c_customer_id: + description: Business key for customer + synonyms: + - customer ID + - customer number + c_first_name: + description: Customer first name + c_last_name: + description: Customer last name + customer_full_name: + sql: c_first_name || ' ' || c_last_name + description: Customer full name (computed field) + synonyms: + - full name + - customer name + c_email_address: + description: Customer email address + synonyms: + - email + - contact diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/date_dim.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/date_dim.view.yaml new file mode 100644 index 0000000..dcc7160 --- /dev/null +++ b/converters/omni/tests/fixtures/tpcds_omni/views/date_dim.view.yaml @@ -0,0 +1,53 @@ +catalog: tpcds +schema: public +description: Date dimension with calendar attributes +dimensions: + d_date_sk: + description: Surrogate key for date + primary_key: true + d_date: + description: Actual date value + synonyms: + - date + - calendar date + timeframes: + - raw + - date + - week + - month + - quarter + - year + d_year: + description: Year + synonyms: + - year + timeframes: + - raw + - date + - week + - month + - quarter + - year + d_quarter_name: + description: Quarter name (e.g., 2024Q1) + synonyms: + - quarter + - fiscal quarter + timeframes: + - raw + - date + - week + - month + - quarter + - year + d_month_name: + description: Month name + synonyms: + - month + timeframes: + - raw + - date + - week + - month + - quarter + - year diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/item.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/item.view.yaml new file mode 100644 index 0000000..c4fa8c2 --- /dev/null +++ b/converters/omni/tests/fixtures/tpcds_omni/views/item.view.yaml @@ -0,0 +1,33 @@ +catalog: tpcds +schema: public +description: Item/Product dimension with product attributes +dimensions: + i_item_sk: + description: Surrogate key for item + primary_key: true + i_item_id: + description: Business key for item + synonyms: + - item ID + - product ID + - SKU + i_item_desc: + description: Item description + synonyms: + - product description + - item name + i_brand: + description: Brand name + synonyms: + - brand + - manufacturer + i_category: + description: Item category + synonyms: + - product category + - department + i_current_price: + description: Current price of the item + synonyms: + - price + - list price diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/store.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/store.view.yaml new file mode 100644 index 0000000..5a9520d --- /dev/null +++ b/converters/omni/tests/fixtures/tpcds_omni/views/store.view.yaml @@ -0,0 +1,32 @@ +catalog: tpcds +schema: public +description: Store dimension with location and store attributes +dimensions: + s_store_sk: + description: Surrogate key for store + primary_key: true + s_store_id: + description: Business key for store + synonyms: + - store ID + - store number + s_store_name: + description: Store name + synonyms: + - store name + - location name + s_city: + description: City where store is located + synonyms: + - city + - location + s_state: + description: State where store is located + synonyms: + - state + - region + s_number_employees: + description: Number of employees at the store + synonyms: + - employee count + - staff size diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/store_sales.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/store_sales.view.yaml new file mode 100644 index 0000000..0a21118 --- /dev/null +++ b/converters/omni/tests/fixtures/tpcds_omni/views/store_sales.view.yaml @@ -0,0 +1,90 @@ +catalog: tpcds +schema: public +description: Fact table containing all store sales transactions +custom_compound_primary_key_sql: +- ss_item_sk +- ss_ticket_number +dimensions: + ss_sold_date_sk: + description: Foreign key to date dimension + synonyms: + - sale date + - transaction date + ss_item_sk: + description: Foreign key to item dimension + synonyms: + - product + - item + ss_customer_sk: + description: Foreign key to customer dimension + synonyms: + - customer + - buyer + ss_store_sk: + description: Foreign key to store dimension + synonyms: + - store + - location + ss_quantity: + description: Quantity of items sold + synonyms: + - units sold + - quantity + ss_sales_price: + description: Sales price per unit + synonyms: + - unit price + - price + ss_ext_sales_price: + description: Extended sales price (quantity * price) + synonyms: + - total price + - line total + ss_net_profit: + description: Net profit from the sale + synonyms: + - profit + - margin + ss_ticket_number: + hidden: true +measures: + total_sales: + sql: ${ss_ext_sales_price} + aggregate_type: sum + description: Total sales revenue across all transactions + synonyms: + - total revenue + - gross sales + - sales amount + total_profit: + sql: ${ss_net_profit} + aggregate_type: sum + description: Total net profit from store sales + synonyms: + - net profit + - total earnings + - profit + customer_lifetime_value: + sql: SUM(${store_sales.ss_ext_sales_price}) / COUNT(DISTINCT ${customer.c_customer_sk}) + description: Average lifetime sales value per customer + synonyms: + - CLV + - LTV + - customer value + - lifetime revenue + sales_by_brand: + sql: ${ss_ext_sales_price} + aggregate_type: sum + description: Total sales by brand (requires grouping by item.i_brand) + synonyms: + - brand sales + - brand performance + - brand revenue + store_productivity: + sql: SUM(${store_sales.ss_ext_sales_price}) / NULLIF(SUM(${store.s_number_employees}), + 0) + description: Sales per employee across stores + synonyms: + - sales per employee + - employee productivity + - revenue per employee diff --git a/converters/omni/tests/test_omni_to_osi.py b/converters/omni/tests/test_omni_to_osi.py new file mode 100644 index 0000000..b124808 --- /dev/null +++ b/converters/omni/tests/test_omni_to_osi.py @@ -0,0 +1,376 @@ +"""Unit tests for the Omni -> OSI importer.""" + +import json +import warnings + +import pytest + +from osi_omni import ConversionError, convert_omni_to_osi +from osi_omni._common import dump_yaml +from _util import load_fixture_dir, parse + + +def imp(files, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return parse(convert_omni_to_osi(files, **kwargs))["semantic_model"][0] + + +def minimal_files(**view_overrides): + view = {"schema": "sch", "dimensions": {"id": {"primary_key": True}, + "amount": {}}} + view.update(view_overrides) + return {"views/orders.view.yaml": dump_yaml(view)} + + +def stash_of(obj): + for ext in obj.get("custom_extensions") or []: + if ext.get("vendor_name") == "OMNI": + return json.loads(ext["data"]) + return {} + + +def dataset(model, name): + return next(d for d in model["datasets"] if d["name"] == name) + + +def field(ds, name): + return next(f for f in ds.get("fields", []) if f["name"] == name) + + +def expr_of(obj): + return obj["expression"]["dialects"][0]["expression"] + + +# --- structure -------------------------------------------------------------- + +def test_view_becomes_dataset_with_source(): + model = imp(minimal_files()) + ds = dataset(model, "orders") + assert ds["source"] == "sch.orders" # table_name defaults to the view name + assert ds["primary_key"] == ["id"] + assert expr_of(field(ds, "amount")) == "amount" + + +def test_catalog_and_table_name_join_into_source(): + model = imp(minimal_files(catalog="db", table_name="RAW_ORDERS")) + assert dataset(model, "orders")["source"] == "db.sch.RAW_ORDERS" + + +def test_sql_view_becomes_subquery_source(): + files = {"views/v.view.yaml": dump_yaml({"sql": "SELECT 1 AS x", + "dimensions": {"x": {}}})} + assert dataset(imp(files), "v")["source"] == "SELECT 1 AS x" + + +def test_view_without_schema_or_sql_is_stashed_not_converted(): + # An extends-only view has no standalone dataset form; it is preserved in + # the model stash (and a model with nothing else convertible is an error). + files = dict(minimal_files()) + files["views/v.view.yaml"] = dump_yaml({"extends": ["orders"], + "dimensions": {"x": {}}}) + model = imp(files) + assert [d["name"] for d in model["datasets"]] == ["orders"] + assert "views/v.view.yaml" in stash_of(model)["extra_files"] + + with pytest.raises(ConversionError, match="no convertible view files"): + imp({"views/v.view.yaml": dump_yaml({"dimensions": {"x": {}}})}) + + +def test_no_view_files_rejected(): + with pytest.raises(ConversionError, match="no convertible view files"): + imp({"model.yaml": "{}\n"}) + + +def test_dimension_metadata_maps(): + files = minimal_files(dimensions={ + "amount": {"label": "Amount", "description": "The amount", + "synonyms": ["value"], "ai_context": "Prefer this."}}) + f = field(dataset(imp(files), "orders"), "amount") + assert f["label"] == "Amount" + assert f["description"] == "The amount" + assert f["ai_context"] == {"instructions": "Prefer this.", + "synonyms": ["value"]} + + +def test_timeframes_map_to_is_time_and_stash(): + files = minimal_files(dimensions={"created_at": {"timeframes": ["raw", "date"]}}) + f = field(dataset(imp(files), "orders"), "created_at") + assert f["dimension"] == {"is_time": True} + assert stash_of(f)["timeframes"] == ["raw", "date"] + + +def test_field_reference_sql_translates_and_stashes_original(): + files = minimal_files(dimensions={ + "sale_price": {}, + "full_price": {"sql": "${sale_price} * 1.1"}}) + f = field(dataset(imp(files), "orders"), "full_price") + assert expr_of(f) == "sale_price * 1.1" + assert stash_of(f)["sql"] == "${sale_price} * 1.1" + + +def test_table_ref_sql_translates(): + files = minimal_files(dimensions={"x": {"sql": "${TABLE}.raw_x"}}) + f = field(dataset(imp(files), "orders"), "x") + assert expr_of(f) == "raw_x" + + +def test_omni_only_dimension_params_stash(): + files = minimal_files(dimensions={ + "amount": {"format": "usdcurrency_2", "hidden": True, + "group_label": "Money"}}) + f = field(dataset(imp(files), "orders"), "amount") + assert stash_of(f) == {"_v": 1, "format": "usdcurrency_2", "hidden": True, + "group_label": "Money"} + + +def test_compound_primary_key_resolves_to_columns(): + files = minimal_files( + custom_compound_primary_key_sql=["id", "line"], + dimensions={"id": {}, "line": {"sql": "line_no"}}) + assert dataset(imp(files), "orders")["primary_key"] == ["id", "line_no"] + + +def test_view_extras_stash(): + files = minimal_files(label="Orders", hidden=True, tags=["fact"]) + ds = dataset(imp(files), "orders") + assert stash_of(ds)["view_extras"] == {"label": "Orders", "hidden": True, + "tags": ["fact"]} + + +# --- relationships ---------------------------------------------------------- + +def _two_view_files(rels): + return { + "views/orders.view.yaml": dump_yaml( + {"schema": "s", "dimensions": {"id": {"primary_key": True}, + "user_id": {}}}), + "views/users.view.yaml": dump_yaml( + {"schema": "s", "dimensions": {"id": {"primary_key": True}}}), + "relationships.yaml": dump_yaml(rels), + } + + +def test_many_to_one_join_decomposes(): + model = imp(_two_view_files([ + {"join_from_view": "orders", "join_to_view": "users", + "on_sql": "${orders.user_id} = ${users.id}", + "relationship_type": "many_to_one"}])) + rel = model["relationships"][0] + assert (rel["from"], rel["to"]) == ("orders", "users") + assert rel["from_columns"] == ["user_id"] + assert rel["to_columns"] == ["id"] + assert rel["name"] == "orders_to_users" + + +def test_one_to_many_join_flips_to_many_side_first(): + model = imp(_two_view_files([ + {"join_from_view": "users", "join_to_view": "orders", + "on_sql": "${users.id} = ${orders.user_id}", + "relationship_type": "one_to_many"}])) + rel = model["relationships"][0] + assert (rel["from"], rel["to"]) == ("orders", "users") + assert rel["from_columns"] == ["user_id"] + assert rel["to_columns"] == ["id"] + assert stash_of(rel)["relationship_type"] == "one_to_many" + + +def test_composite_on_sql_decomposes_in_order(): + model = imp(_two_view_files([ + {"join_from_view": "orders", "join_to_view": "users", + "on_sql": "${orders.a} = ${users.b} AND ${orders.c} = ${users.d}", + "relationship_type": "many_to_one"}])) + rel = model["relationships"][0] + assert rel["from_columns"] == ["a", "c"] + assert rel["to_columns"] == ["b", "d"] + + +def test_field_reference_in_on_sql_resolves_to_column(): + files = { + "views/orders.view.yaml": dump_yaml( + {"schema": "s", + "dimensions": {"user_key": {"sql": "user_id_raw"}}}), + "views/users.view.yaml": dump_yaml( + {"schema": "s", "dimensions": {"id": {}}}), + "relationships.yaml": dump_yaml([ + {"join_from_view": "orders", "join_to_view": "users", + "on_sql": "${orders.user_key} = ${users.id}", + "relationship_type": "many_to_one"}]), + } + rel = imp(files)["relationships"][0] + assert rel["from_columns"] == ["user_id_raw"] + + +def test_non_equi_join_is_stashed_not_converted(): + # A range join is valid in Omni but has no OSI relationship form; it is + # preserved verbatim (with its position) rather than failing the import. + entry = {"join_from_view": "orders", "join_to_view": "users", + "on_sql": "${orders.total} >= ${users.threshold}", + "relationship_type": "many_to_one"} + model = imp(_two_view_files([entry])) + assert "relationships" not in model + assert stash_of(model)["extra_relationships"] == [ + {"index": 0, "entry": entry}] + + +def test_join_referencing_third_view_is_stashed_not_converted(): + entry = {"join_from_view": "orders", "join_to_view": "users", + "on_sql": "${orders.x} = ${ghost.y}", + "relationship_type": "many_to_one"} + model = imp(_two_view_files([entry])) + assert "relationships" not in model + assert stash_of(model)["extra_relationships"] == [ + {"index": 0, "entry": entry}] + + +def test_join_to_missing_view_rejected(): + with pytest.raises(ConversionError, match="has no view file"): + imp({"views/orders.view.yaml": dump_yaml({"schema": "s"}), + "relationships.yaml": dump_yaml([ + {"join_from_view": "orders", "join_to_view": "ghost", + "on_sql": "${orders.x} = ${ghost.y}", + "relationship_type": "many_to_one"}])}) + + +def test_join_type_and_where_sql_stash(): + model = imp(_two_view_files([ + {"join_from_view": "orders", "join_to_view": "users", + "on_sql": "${orders.user_id} = ${users.id}", + "relationship_type": "many_to_one", "join_type": "inner", + "where_sql": "${users.active}", "reversible": True}])) + stash = stash_of(model["relationships"][0]) + assert stash["join_type"] == "inner" + assert stash["where_sql"] == "${users.active}" + assert stash["reversible"] is True + + +# --- measures --------------------------------------------------------------- + +def test_count_measure_becomes_count_star_metric(): + files = minimal_files(measures={"count": {"aggregate_type": "count"}}) + metric = imp(files)["metrics"][0] + assert expr_of(metric) == "COUNT(*)" + assert metric["name"] == "count" + + +def test_sum_measure_qualifies_operand(): + files = minimal_files(measures={ + "total": {"sql": "${amount}", "aggregate_type": "sum", + "description": "Total", "synonyms": ["revenue"]}}) + metric = imp(files)["metrics"][0] + assert expr_of(metric) == "SUM(orders.amount)" + assert metric["description"] == "Total" + assert metric["ai_context"] == {"synonyms": ["revenue"]} + + +def test_count_distinct_measure(): + files = minimal_files(measures={ + "n_users": {"sql": "${amount}", "aggregate_type": "count_distinct"}}) + assert expr_of(imp(files)["metrics"][0]) == "COUNT(DISTINCT orders.amount)" + + +def test_raw_sql_measure_keeps_view_qualifiers(): + files = minimal_files(measures={ + "ratio": {"sql": "SUM(${orders.amount}) / COUNT(*)"}}) + assert expr_of(imp(files)["metrics"][0]) == "SUM(orders.amount) / COUNT(*)" + + +def test_filtered_measure_stashes_original(): + files = minimal_files(measures={ + "done": {"aggregate_type": "count", + "filters": {"status": {"is": "complete"}}}}) + metric = imp(files)["metrics"][0] + assert expr_of(metric) == "COUNT(*)" + stash = stash_of(metric) + assert stash["measure"]["filters"] == {"status": {"is": "complete"}} + assert stash["view"] == "orders" + + +def test_percentile_measure_best_effort_expression(): + files = minimal_files(measures={ + "p95": {"sql": "${amount}", "aggregate_type": "percentile", + "percentile": 95}}) + metric = imp(files)["metrics"][0] + assert expr_of(metric) == ( + "PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY orders.amount)") + assert stash_of(metric)["measure"]["aggregate_type"] == "percentile" + + +def test_unknown_aggregate_type_rejected(): + files = minimal_files(measures={"x": {"aggregate_type": "mystery"}}) + with pytest.raises(ConversionError, match="unknown aggregate_type"): + imp(files) + + +def test_colliding_measure_names_qualify_with_view(): + files = { + "views/a.view.yaml": dump_yaml( + {"schema": "s", "measures": {"count": {"aggregate_type": "count"}}}), + "views/b.view.yaml": dump_yaml( + {"schema": "s", "measures": {"count": {"aggregate_type": "count"}}}), + } + names = sorted(m["name"] for m in imp(files)["metrics"]) + assert names == ["a__count", "b__count"] + + +# --- topics and the model file ---------------------------------------------- + +def test_topic_maps_onto_model(): + model = imp(load_fixture_dir("fixtureB_omni")) + assert model["name"] == "order_analysis" + assert model["description"] == "Line-item order analysis" + assert model["ai_context"] == {"instructions": "You are an ecommerce analyst."} + stash = stash_of(model) + assert stash["mapped_topic"] == "order_analysis" + assert stash["base_view"] == "order_items" + topic_stash = stash["topics"]["order_analysis"] + assert "description" not in topic_stash # natively mapped, not duplicated + assert topic_stash["label"] == "Order Analysis" + assert topic_stash["joins"] == {"users": {"orders": {}}} + + +def test_model_file_stashes_verbatim(): + model = imp(load_fixture_dir("fixtureB_omni")) + model_file = stash_of(model)["model_file"] + assert model_file["week_start_day"] == "Sunday" + assert model_file["included_schemas"] == ["ecomm"] + + +def test_no_topics_stashes_empty_topic_set(): + model = imp(minimal_files()) + assert stash_of(model)["topics"] == {} + + +def test_topic_flag_selects_mapped_topic(): + files = dict(load_fixture_dir("fixtureB_omni")) + files["topics/second.topic.yaml"] = dump_yaml( + {"base_view": "users", "description": "Second"}) + model = imp(files, topic="second") + assert model["description"] == "Second" + + +def test_unknown_topic_flag_rejected(): + with pytest.raises(ConversionError, match="not found"): + imp(load_fixture_dir("fixtureB_omni"), topic="ghost") + + +def test_model_name_flag_overrides_topic_name(): + model = imp(load_fixture_dir("fixtureB_omni"), model_name="my_model") + assert model["name"] == "my_model" + + +def test_query_view_preserved_as_extra_file(): + files = minimal_files() + files["views/facts.query.view.yaml"] = "schema: s\nsql: SELECT 1\n" + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") + model = parse(convert_omni_to_osi(files))["semantic_model"][0] + assert any("query views" in str(w.message) for w in ws) + assert "views/facts.query.view.yaml" in stash_of(model)["extra_files"] + + +def test_relationships_must_be_a_list(): + files = minimal_files() + files["relationships.yaml"] = dump_yaml({"not": "a list"}) + with pytest.raises(ConversionError, match="top-level YAML list"): + imp(files) diff --git a/converters/omni/tests/test_osi_to_omni.py b/converters/omni/tests/test_osi_to_omni.py new file mode 100644 index 0000000..ba0fb61 --- /dev/null +++ b/converters/omni/tests/test_osi_to_omni.py @@ -0,0 +1,334 @@ +"""Unit tests for the OSI -> Omni exporter.""" + +import warnings + +import pytest + +from osi_omni import ConversionError, convert_osi_to_omni +from osi_omni._common import dump_yaml +from _util import REPO_ROOT, load_fixture, load_fixture_dir, parse, parse_files + + +def export(osi_yaml, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return convert_osi_to_omni(osi_yaml, **kwargs) + + +def minimal(**model_overrides): + model = { + "name": "m", + "datasets": [{"name": "orders", "source": "db.sch.orders", + "fields": [_field("amount")]}], + } + model.update(model_overrides) + return dump_yaml({"version": "0.2.0.dev0", "semantic_model": [model]}) + + +def _field(name, expr=None, dialect="ANSI_SQL", **extra): + f = {"name": name, + "expression": {"dialects": [{"dialect": dialect, + "expression": expr or name}]}} + f.update(extra) + return f + + +# --- fixtures --------------------------------------------------------------- + +def test_fixtureA_export_matches_expected(): + files = export(load_fixture("fixtureA_osi.yaml")) + assert parse_files(files) == parse_files(load_fixture_dir("fixtureA_omni")) + + +def test_tpcds_export_matches_expected(): + with open(REPO_ROOT / "examples" / "tpcds_semantic_model.yaml") as fh: + files = export(fh.read()) + assert parse_files(files) == parse_files(load_fixture_dir("tpcds_omni")) + + +# --- structure -------------------------------------------------------------- + +def test_three_part_source_splits_into_catalog_schema_table(): + files = export(minimal()) + view = parse(files["views/orders.view.yaml"]) + assert view["catalog"] == "db" + assert view["schema"] == "sch" + assert "table_name" not in view # table matches the view name + + +def test_table_name_emitted_when_it_differs(): + files = export(minimal(datasets=[{"name": "orders", "source": "sch.raw_orders"}])) + view = parse(files["views/orders.view.yaml"]) + assert view == {"schema": "sch", "table_name": "raw_orders"} + + +def test_subquery_source_becomes_sql_view(): + files = export(minimal(datasets=[ + {"name": "orders", "source": "SELECT * FROM t"}])) + assert parse(files["views/orders.view.yaml"])["sql"] == "SELECT * FROM t" + + +def test_one_part_source_rejected(): + with pytest.raises(ConversionError, match="no schema part"): + export(minimal(datasets=[{"name": "orders", "source": "orders"}])) + + +def test_field_same_named_bare_column_gets_no_sql(): + files = export(minimal()) + assert parse(files["views/orders.view.yaml"])["dimensions"]["amount"] == {} + + +def test_field_renamed_bare_column_gets_sql(): + files = export(minimal(datasets=[ + {"name": "orders", "source": "db.sch.orders", + "fields": [_field("total", "amount")]}])) + dims = parse(files["views/orders.view.yaml"])["dimensions"] + assert dims["total"] == {"sql": "amount"} + + +def test_complex_expression_emitted_verbatim(): + files = export(minimal(datasets=[ + {"name": "orders", "source": "db.sch.orders", + "fields": [_field("full_name", "first || ' ' || last")]}])) + dims = parse(files["views/orders.view.yaml"])["dimensions"] + assert dims["full_name"]["sql"] == "first || ' ' || last" + + +def test_is_time_maps_to_default_timeframes(): + files = export(minimal(datasets=[ + {"name": "orders", "source": "db.sch.orders", + "fields": [_field("created_at", dimension={"is_time": True})]}])) + dims = parse(files["views/orders.view.yaml"])["dimensions"] + assert dims["created_at"]["timeframes"] == [ + "raw", "date", "week", "month", "quarter", "year"] + + +def test_single_primary_key_marks_dimension(): + files = export(minimal(datasets=[ + {"name": "orders", "source": "db.sch.orders", "primary_key": ["id"], + "fields": [_field("id")]}])) + assert parse(files["views/orders.view.yaml"])["dimensions"]["id"][ + "primary_key"] is True + + +def test_composite_primary_key_uses_compound_key(): + files = export(minimal(datasets=[ + {"name": "orders", "source": "db.sch.orders", + "primary_key": ["id", "line"], "fields": [_field("id")]}])) + view = parse(files["views/orders.view.yaml"]) + assert view["custom_compound_primary_key_sql"] == ["id", "line"] + # The uncovered key column materialized as a hidden dimension. + assert view["dimensions"]["line"] == {"hidden": True} + + +def test_field_metadata_maps(): + files = export(minimal(datasets=[ + {"name": "orders", "source": "db.sch.orders", + "fields": [_field("amount", label="Amount", description="The amount", + ai_context={"synonyms": ["value"], + "instructions": "Prefer this."})]}])) + dim = parse(files["views/orders.view.yaml"])["dimensions"]["amount"] + assert dim["label"] == "Amount" + assert dim["description"] == "The amount" + assert dim["synonyms"] == ["value"] + assert dim["ai_context"] == "Prefer this." + + +def test_relationship_becomes_join_entry(): + files = export(minimal(datasets=[ + {"name": "orders", "source": "db.sch.orders", + "fields": [_field("user_id")]}, + {"name": "users", "source": "db.sch.users", "fields": [_field("id")]}, + ], relationships=[ + {"name": "r", "from": "orders", "to": "users", + "from_columns": ["user_id"], "to_columns": ["id"]}])) + rels = parse(files["relationships.yaml"]) + assert rels == [{ + "join_from_view": "orders", "join_to_view": "users", + "on_sql": "${orders.user_id} = ${users.id}", + "relationship_type": "many_to_one"}] + + +def test_composite_relationship_joins_with_and(): + files = export(minimal(datasets=[ + {"name": "orders", "source": "db.sch.orders"}, + {"name": "users", "source": "db.sch.users"}, + ], relationships=[ + {"name": "r", "from": "orders", "to": "users", + "from_columns": ["a", "b"], "to_columns": ["c", "d"]}])) + on_sql = parse(files["relationships.yaml"])[0]["on_sql"] + assert on_sql == "${orders.a} = ${users.c} AND ${orders.b} = ${users.d}" + + +def test_topic_generated_with_fk_sink_base_and_join_tree(): + files = export(load_fixture("fixtureA_osi.yaml")) + topic = parse(files["topics/sales.topic.yaml"]) + assert topic["base_view"] == "orders" + assert topic["joins"] == {"customer": {}} + assert topic["description"] == "Sales orders with customer attributes" + assert topic["ai_context"] == "Use this model for order analysis." + + +def test_explicit_base_view_overrides_heuristic(): + files = export(load_fixture("fixtureA_osi.yaml"), base_view="customer") + topic = parse(files["topics/sales.topic.yaml"]) + assert topic["base_view"] == "customer" + assert topic["joins"] == {"orders": {}} + + +def test_unknown_base_view_rejected(): + with pytest.raises(ConversionError, match="not a dataset"): + export(minimal(), base_view="nope") + + +def test_multiple_roots_require_base_view(): + osi = minimal(datasets=[{"name": "a", "source": "db.s.a"}, + {"name": "b", "source": "db.s.b"}]) + with pytest.raises(ConversionError, match="--base-view"): + export(osi) + + +def test_metric_aggregate_becomes_structured_measure(): + files = export(minimal(metrics=[ + {"name": "total", "expression": {"dialects": [ + {"dialect": "ANSI_SQL", "expression": "SUM(orders.amount)"}]}}])) + measures = parse(files["views/orders.view.yaml"])["measures"] + assert measures["total"] == {"sql": "${amount}", "aggregate_type": "sum"} + + +def test_count_star_becomes_count_measure_on_base_view(): + files = export(minimal(metrics=[ + {"name": "n", "expression": {"dialects": [ + {"dialect": "ANSI_SQL", "expression": "COUNT(*)"}]}}])) + assert parse(files["views/orders.view.yaml"])["measures"]["n"] == { + "aggregate_type": "count"} + + +def test_count_distinct_metric(): + files = export(minimal(metrics=[ + {"name": "n", "expression": {"dialects": [ + {"dialect": "ANSI_SQL", + "expression": "COUNT(DISTINCT orders.amount)"}]}}])) + assert parse(files["views/orders.view.yaml"])["measures"]["n"] == { + "sql": "${amount}", "aggregate_type": "count_distinct"} + + +def test_ratio_metric_becomes_raw_sql_measure(): + files = export(minimal(metrics=[ + {"name": "avg_x", "expression": {"dialects": [ + {"dialect": "ANSI_SQL", + "expression": "SUM(orders.amount) / COUNT(*)"}]}}])) + measure = parse(files["views/orders.view.yaml"])["measures"]["avg_x"] + assert measure == {"sql": "SUM(${orders.amount}) / COUNT(*)"} + + +def test_dialect_preference(): + field = {"name": "x", "expression": {"dialects": [ + {"dialect": "ANSI_SQL", "expression": "ansi_col"}, + {"dialect": "SNOWFLAKE", "expression": "snow_col"}]}} + files = export(minimal(datasets=[ + {"name": "orders", "source": "db.sch.orders", "fields": [field]}]), + dialect="SNOWFLAKE") + assert parse(files["views/orders.view.yaml"])["dimensions"]["x"][ + "sql"] == "snow_col" + + +def test_names_are_sanitized(): + files = export(dump_yaml({"version": "0.2.0.dev0", "semantic_model": [{ + "name": "My Model", + "datasets": [{"name": "Order Items", "source": "db.sch.t", + "fields": [_field("Total Price", "p")]}]}]})) + assert "views/order_items.view.yaml" in files + assert "topics/my_model.topic.yaml" in files + dims = parse(files["views/order_items.view.yaml"])["dimensions"] + assert dims == {"total_price": {"sql": "p"}} + + +def test_sanitization_collision_rejected(): + osi = minimal(datasets=[{"name": "Orders", "source": "db.s.a"}, + {"name": "orders", "source": "db.s.b"}]) + with pytest.raises(ConversionError, match="collides"): + export(osi) + + +def test_duplicate_metric_names_rejected(): + metric = {"name": "total", "expression": {"dialects": [ + {"dialect": "ANSI_SQL", "expression": "SUM(orders.amount)"}]}} + with pytest.raises(ConversionError, match="two metrics"): + export(minimal(metrics=[metric, dict(metric, name="Total")])) + + +def test_relationship_column_count_mismatch_rejected(): + osi = minimal(datasets=[{"name": "a", "source": "db.s.a"}, + {"name": "b", "source": "db.s.b"}], + relationships=[{"name": "r", "from": "a", "to": "b", + "from_columns": ["x"], "to_columns": ["y", "z"]}]) + with pytest.raises(ConversionError, match="same length"): + export(osi) + + +def test_unknown_relationship_dataset_rejected(): + osi = minimal(relationships=[{"name": "r", "from": "orders", "to": "ghost", + "from_columns": ["x"], "to_columns": ["y"]}]) + with pytest.raises(ConversionError, match="unknown dataset"): + export(osi) + + +def test_unsupported_version_rejected(): + with pytest.raises(ConversionError, match="Unsupported OSI version"): + export(dump_yaml({"version": "9.9.9", "semantic_model": [{"name": "m"}]})) + + +# --- warnings --------------------------------------------------------------- + +def _warnings_of(osi_yaml, **kwargs): + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") + convert_osi_to_omni(osi_yaml, **kwargs) + return [str(w.message) for w in ws] + + +def test_unique_keys_warn(): + msgs = _warnings_of(minimal(datasets=[ + {"name": "orders", "source": "db.s.t", "primary_key": ["id"], + "unique_keys": [["email"]], "fields": [_field("id")]}])) + assert any("unique_keys" in m for m in msgs) + + +def test_unique_keys_restating_primary_key_do_not_warn(): + msgs = _warnings_of(minimal(datasets=[ + {"name": "orders", "source": "db.s.t", "primary_key": ["id"], + "unique_keys": [["id"]], "fields": [_field("id")]}])) + assert not any("unique_keys" in m for m in msgs) + + +def test_field_without_usable_dialect_warns_and_drops(): + osi = minimal(datasets=[ + {"name": "orders", "source": "db.s.t", + "fields": [_field("x", dialect="MDX")]}]) + msgs = _warnings_of(osi) + assert any("no usable" in m for m in msgs) + assert "dimensions" not in parse(export(osi)["views/orders.view.yaml"]) + + +def test_relationship_ai_context_warns(): + msgs = _warnings_of(minimal(datasets=[ + {"name": "a", "source": "db.s.a"}, {"name": "b", "source": "db.s.b"}], + relationships=[{"name": "r", "from": "a", "to": "b", + "from_columns": ["x"], "to_columns": ["y"], + "ai_context": {"synonyms": ["join"]}}])) + assert any("relationship ai_context" in m for m in msgs) + + +def test_foreign_vendor_extensions_warn(): + msgs = _warnings_of(minimal(custom_extensions=[ + {"vendor_name": "DBT", "data": "{}"}])) + assert any("foreign-vendor" in m for m in msgs) + + +def test_multiple_models_warn_and_first_converts(): + osi = parse(minimal()) + osi["semantic_model"].append({"name": "second", "datasets": [ + {"name": "x", "source": "db.s.x"}]}) + msgs = _warnings_of(dump_yaml(osi)) + assert any("multiple semantic models" in m for m in msgs) diff --git a/converters/omni/tests/test_real_world_layout.py b/converters/omni/tests/test_real_world_layout.py new file mode 100644 index 0000000..b58a159 --- /dev/null +++ b/converters/omni/tests/test_real_world_layout.py @@ -0,0 +1,170 @@ +"""Round-trip tests for the model shape Omni's API/IDE actually emits. + +A model pulled from a real instance (`omni models yaml-get`) differs from the +canonical `views/.view.yaml` layout these converters write: + + - view files live in per-schema folders (`DELIGHTED/response.view`) and the + model refers to them by a schema-qualified name (`delighted__response`) + recorded in a `# Reference this view as ...` header comment; + - joins restate Omni defaults (`join_type: always_left`, + `relationship_type: many_to_one`) and write compound keys / on_sql with + `${view.field}` references, mixed-case AND, and alias references; + - extends-only views, non-equi joins, template-syntax sql, empty-string + metadata, `timeframes: []`, camelCase/underscore-edged identifiers, and + schema names with spaces all occur. + +Every one of these was found in production models; the round trip through OSI +must reproduce the original files exactly (same paths, same parsed content). +""" + +import warnings + +import pytest + +from osi_omni import ConversionError, convert_omni_to_osi, convert_osi_to_omni +from osi_omni._common import dump_yaml, load_yaml + + +REAL_FILES = { + "model": dump_yaml({"included_schemas": ["DELIGHTED", "GITHUB"]}), + "relationships": dump_yaml([ + # Omni defaults restated explicitly; ${view.field} refs; declared type. + {"join_from_view": "delighted__response", + "join_to_view": "delighted__person", + "join_type": "always_left", + "on_sql": "${delighted__response.person_id} = ${delighted__person.id}", + "relationship_type": "assumed_many_to_one"}, + # Non-equi (range) join: valid Omni, no OSI form -- stashed. + {"join_from_view": "delighted__response", + "join_to_view": "github__team_membership", + "join_type": "always_left", + "on_sql": "${delighted__response.created_at} >= " + "${github__team_membership.valid_from}", + "relationship_type": "many_to_one"}, + # Aliased join: on_sql references the alias, lowercase `and`. + {"join_from_view": "github__team_membership", + "join_to_view": "delighted__person", + "join_to_view_as": "membership_owner", + "on_sql": "${github__team_membership.user_id} = ${membership_owner.id} " + "and ${github__team_membership.org} = ${membership_owner.org}", + "relationship_type": "many_to_one"}, + # Second join between the same pair (name dedup on the OSI side). + {"join_from_view": "github__team_membership", + "join_to_view": "delighted__person", + "on_sql": "${github__team_membership.approver_id} = " + "${delighted__person.id}", + "relationship_type": "many_to_one"}, + ]), + "DELIGHTED/response.view": ( + "# Reference this view as delighted__response\n" + dump_yaml({ + "schema": "DELIGHTED", "table_name": "RESPONSE", + "description": "", # present-but-empty metadata survives + "custom_compound_primary_key_sql": [ + "${delighted__response.id}", "${delighted__response.org}"], + "dimensions": { + "id": {"sql": '"ID"'}, + "org": {"sql": '"ORG"'}, + "person_id": {"sql": '"PERSON_ID"'}, + "created_at": {"sql": '"CREATED_AT"', "timeframes": []}, + "self_named": {"sql": "self_named"}, # explicit same-named col + "_fivetran_id": {"sql": '"_FIVETRAN_ID"'}, + "payload_modelId": { # camelCase (JSON-flattened) name + "sql": "CAST(${delighted__response.payload}['modelId'] " + "AS VARCHAR)"}, + "templated": {"sql": "CASE WHEN {{# delighted__response.f.filter }}" + " ${id} {{/ delighted__response.f.filter }} " + "THEN 1 ELSE 0 END"}, + }, + "measures": {"count": {"aggregate_type": "count"}}, + })), + "DELIGHTED/person.view": ( + "# Reference this view as delighted__person\n" + dump_yaml({ + "schema": "DELIGHTED", # table_name implicit = file name + "dimensions": {"id": {"sql": '"ID"', "primary_key": True}, + "org": {"sql": '"ORG"'}}, + })), + "GITHUB/team_membership.view": ( + "# Reference this view as github__team_membership\n" + dump_yaml({ + "schema": "GITHUB", "table_name": "TEAM_MEMBERSHIP", + "dimensions": {"user_id": {}, "approver_id": {}, "org": {}, + "valid_from": {}}, + })), + # Extends-only view: no schema/sql of its own -- preserved, not converted. + "DELIGHTED/response_ext.view": ( + "# Reference this view as delighted__response_ext\n" + dump_yaml({ + "extends": ["delighted__response"], + "dimensions": {"extra": {"sql": "1"}}, + })), + # Uploaded-CSV schema with a space in its name. + "Omni Views/upload.view": ( + "# Reference this view as upload\n" + dump_yaml({ + "schema": "Omni Views", + "uploaded_table_name": "upload.csv::abc", + "dimensions": {"user_id": {}}, + })), + "Marketing/insights.topic": dump_yaml({ + "base_view": "delighted__response", + "description": "Survey insights."}), +} + + +def _roundtrip(files): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + osi_yaml = convert_omni_to_osi(files) + return osi_yaml, convert_osi_to_omni(osi_yaml) + + +def test_api_layout_roundtrip_is_lossless(): + _, exported = _roundtrip(REAL_FILES) + original = {k if "." in k.split("/")[-1] else k + ".yaml": v + for k, v in REAL_FILES.items()} + assert set(exported) == set(original) + for fname in original: + assert load_yaml(exported[fname]) == load_yaml(original[fname]), fname + + +def test_qualified_view_names_come_from_reference_comment(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + osi = load_yaml(convert_omni_to_osi(REAL_FILES))["semantic_model"][0] + names = {d["name"] for d in osi["datasets"]} + assert "delighted__response" in names + assert "upload" in names # comment name wins even with a folder + # The qualified name resolves relationship references and same-view + # ${view.field} compound-key entries. + rel = osi["relationships"][0] + assert rel["from"] == "delighted__response" + assert rel["to"] == "delighted__person" + response = next(d for d in osi["datasets"] + if d["name"] == "delighted__response") + assert response["primary_key"] == ["id", "org"] + assert response["source"] == "DELIGHTED.RESPONSE" + upload = next(d for d in osi["datasets"] if d["name"] == "upload") + assert upload["source"] == '"Omni Views".upload' + + +def test_unmappable_omni_features_drop_out_of_osi_but_survive(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + osi = load_yaml(convert_omni_to_osi(REAL_FILES))["semantic_model"][0] + # The extends view is not a dataset; the templated dimension not a field. + assert "delighted__response_ext" not in {d["name"] for d in osi["datasets"]} + response = next(d for d in osi["datasets"] + if d["name"] == "delighted__response") + assert "templated" not in {f["name"] for f in response["fields"]} + # The non-equi join is not an OSI relationship; the two same-pair joins get + # unique OSI names. + names = [r["name"] for r in osi["relationships"]] + assert len(names) == len(set(names)) == 3 + + +def test_duplicate_canonical_view_names_rejected(): + files = { + "A/orders.view": "# Reference this view as orders\nschema: A\n", + "B/orders.view": "# Reference this view as orders\nschema: B\n", + } + with pytest.raises(ConversionError, match="two view files"): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + convert_omni_to_osi(files) diff --git a/converters/omni/tests/test_roundtrip.py b/converters/omni/tests/test_roundtrip.py new file mode 100644 index 0000000..baccb52 --- /dev/null +++ b/converters/omni/tests/test_roundtrip.py @@ -0,0 +1,65 @@ +"""Fixture-based round-trip tests. + +- Omni -> OSI -> Omni must be lossless (the stash carries everything). +- OSI -> Omni -> OSI must be identical up to the documented normalizations + (see _util.strip_normalized). +- Every OSI document the importer emits must validate against the core-spec + JSON schema (skipped when jsonschema is not installed). +""" + +import warnings + +import pytest + +from osi_omni import convert_omni_to_osi, convert_osi_to_omni +from _util import ( + REPO_ROOT, + load_fixture, + load_fixture_dir, + parse, + parse_files, + strip_normalized, +) + + +def _quiet(fn, *args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return fn(*args, **kwargs) + + +@pytest.mark.parametrize("fixture", ["fixtureA_omni", "fixtureB_omni", "tpcds_omni"]) +def test_omni_roundtrip_is_lossless(fixture): + files = load_fixture_dir(fixture) + osi = _quiet(convert_omni_to_osi, files) + files2 = _quiet(convert_osi_to_omni, osi) + assert parse_files(files2) == parse_files(files) + + +@pytest.mark.parametrize("path", [ + "fixtureA_osi.yaml", + str(REPO_ROOT / "examples" / "tpcds_semantic_model.yaml"), +]) +def test_osi_roundtrip_up_to_documented_normalizations(path): + if "/" in path: + with open(path) as fh: + osi_yaml = fh.read() + else: + osi_yaml = load_fixture(path) + files = _quiet(convert_osi_to_omni, osi_yaml) + osi2 = _quiet(convert_omni_to_osi, files) + original = strip_normalized(parse(osi_yaml)) + # Foreign-vendor extensions are dropped on export (documented); OSI-side + # normalization strips all custom_extensions from both sides. + assert strip_normalized(parse(osi2)) == original + + +@pytest.mark.parametrize("fixture", ["fixtureA_omni", "fixtureB_omni", "tpcds_omni"]) +def test_imported_osi_validates_against_core_spec_schema(fixture): + jsonschema = pytest.importorskip("jsonschema") + import json + + with open(REPO_ROOT / "core-spec" / "osi-schema.json") as fh: + schema = json.load(fh) + osi = _quiet(convert_omni_to_osi, load_fixture_dir(fixture)) + jsonschema.validate(parse(osi), schema) diff --git a/converters/omni/tests/test_roundtrip_properties.py b/converters/omni/tests/test_roundtrip_properties.py new file mode 100644 index 0000000..445c1e2 --- /dev/null +++ b/converters/omni/tests/test_roundtrip_properties.py @@ -0,0 +1,83 @@ +"""Property-based round-trip tests. + +Two drivers over the same builders (see _roundtrip_helpers): + + - Hypothesis, when installed: minimal failing examples and example databases. + - A plain seeded random.Random sweep otherwise, so the property logic still + runs in minimal environments. +""" + +import pytest + +from _roundtrip_helpers import ( + RandomRnd, + assert_omni_roundtrip, + assert_osi_roundtrip, + build_omni, + build_osi, +) + +try: + from hypothesis import given, settings, strategies as st + + HAVE_HYPOTHESIS = True +except ImportError: # pragma: no cover + HAVE_HYPOTHESIS = False + + +if HAVE_HYPOTHESIS: + + class _HypRnd: + """The Rnd interface driven by hypothesis' `data` strategy.""" + + def __init__(self, data): + self.data = data + + def chance(self, p=0.5): + return self.data.draw(st.floats(0, 1)) < p + + def count(self, lo, hi): + return self.data.draw(st.integers(lo, hi)) + + def pick(self, seq): + return self.data.draw(st.sampled_from(list(seq))) + + def text(self): + alnum = st.characters(whitelist_categories=("Lu", "Ll", "Nd"), + max_codepoint=0x7A) + head = self.data.draw(st.text(alphabet=alnum, min_size=1, max_size=1)) + body = self.data.draw(st.text( + alphabet=st.sampled_from( + list("abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 ")), + max_size=10)) + return (head + body).strip() + + def colname(self): + first = self.data.draw(st.sampled_from( + list("abcdefghijklmnopqrstuvwxyz"))) + rest = self.data.draw(st.text( + alphabet=st.sampled_from( + list("abcdefghijklmnopqrstuvwxyz0123456789_")), + max_size=7)) + return first + rest + + @given(st.data()) + @settings(max_examples=50, deadline=None) + def test_omni_roundtrip_property(data): + assert_omni_roundtrip(build_omni(_HypRnd(data))) + + @given(st.data()) + @settings(max_examples=50, deadline=None) + def test_osi_roundtrip_property(data): + assert_osi_roundtrip(build_osi(_HypRnd(data))) + +else: # pragma: no cover - exercised only without hypothesis + + @pytest.mark.parametrize("seed", range(50)) + def test_omni_roundtrip_seeded(seed): + assert_omni_roundtrip(build_omni(RandomRnd(seed))) + + @pytest.mark.parametrize("seed", range(50)) + def test_osi_roundtrip_seeded(seed): + assert_osi_roundtrip(build_osi(RandomRnd(seed))) diff --git a/validation/validate.py b/validation/validate.py index 5965115..258d34f 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -55,7 +55,7 @@ try: import sqlglot - from sqlglot.errors import ParseError + from sqlglot.errors import ParseError, TokenError SQLGLOT_AVAILABLE = True except ImportError: SQLGLOT_AVAILABLE = False @@ -163,14 +163,14 @@ def validate_sql_expression(expr: str, dialect: str, context: str) -> str | None # Try parsing as expression first (for field expressions like "column_name") sqlglot.parse_one(expr, dialect=sqlglot_dialect) return None - except ParseError: + except (ParseError, TokenError): pass try: # Try wrapping in SELECT for simple column references sqlglot.parse_one(f"SELECT {expr}", dialect=sqlglot_dialect) return None - except ParseError as e: + except (ParseError, TokenError) as e: return f"[SQL] {context}: {str(e).split(chr(10))[0]}" From e05b9cd98ab1740f0cba1b63d62d5adbfd83b622 Mon Sep 17 00:00:00 2001 From: Yong Zheng Date: Thu, 16 Jul 2026 00:51:38 -0500 Subject: [PATCH 73/89] Fix converter dbt pytest (#210) --- converters/dbt/pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/converters/dbt/pyproject.toml b/converters/dbt/pyproject.toml index 812b80d..3666f02 100644 --- a/converters/dbt/pyproject.toml +++ b/converters/dbt/pyproject.toml @@ -51,5 +51,10 @@ dev-dependencies = [ "syrupy>=4.0", ] +# apache-ossie is not yet published to PyPI; resolve it from the in-repo +# package for now. Remove this block once apache-ossie published to PyPI. +[tool.uv.sources] +apache-ossie = { path = "../../python", editable = true} + [tool.pytest.ini_options] testpaths = ["tests"] From d6e1ce0eb0636a3875929fb3f0f3c3247b675748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 16 Jul 2026 20:38:57 +0200 Subject: [PATCH 74/89] Add PR template to "help" the contributor and the reviewers (#218) --- .github/PULL_REQUEST_TEMPLATE.md | 61 ++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..45d0ca8 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,61 @@ + + +## Summary + + + +## Related Issues + + + +## Checklist + +### Specification +- [ ] Spec changes are included in `core-spec/` and follow the existing structure +- [ ] Spec changes have been discussed on the mailing list or in a linked issue +- [ ] Breaking changes to the spec are clearly called out in the summary + +### Ontology +- [ ] Ontology changes in `ontology/` are consistent with spec changes +- [ ] New or modified terms are defined and documented + +### Converters +- [ ] Converter logic in `converters/` is updated to reflect spec or ontology changes +- [ ] New converters include tests under the converter's test directory + +### Validation +- [ ] Validation rules in `validation/` are updated if the spec changed +- [ ] New validation cases are covered by tests + +### Documentation +- [ ] `docs/` is updated to reflect any user-facing changes +- [ ] New features or behaviors are documented with examples where appropriate +- [ ] `CONTRIBUTING.md` is updated if the contribution process changed + +### Examples +- [ ] `examples/` are added or updated for any new spec constructs or converter support + +### Tests +- [ ] All existing tests pass (`pytest` / CI green) +- [ ] New functionality is covered by tests + +### Compliance +- [ ] ASF license headers are present on all new source files +- [ ] No third-party dependencies are added without PMC/IPMC approval From c394e98899f06d5bf190ce686569a6f760fc3959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Fri, 17 Jul 2026 06:40:46 +0200 Subject: [PATCH 75/89] Add ASF header and fix LICENSE header (#217) --- LICENSE | 1 - converters/honeydew/pyproject.toml | 17 +++++++++++++++++ .../honeydew/src/honeydew_osi_converter.py | 17 +++++++++++++++++ .../tests/test_honeydew_osi_converter.py | 17 +++++++++++++++++ converters/omni/pyproject.toml | 17 +++++++++++++++++ converters/omni/src/osi_omni/__init__.py | 17 +++++++++++++++++ converters/omni/src/osi_omni/_common.py | 17 +++++++++++++++++ converters/omni/src/osi_omni/cli.py | 17 +++++++++++++++++ converters/omni/src/osi_omni/omni_to_osi.py | 17 +++++++++++++++++ converters/omni/src/osi_omni/osi_to_omni.py | 17 +++++++++++++++++ converters/omni/tests/_roundtrip_helpers.py | 17 +++++++++++++++++ converters/omni/tests/_util.py | 17 +++++++++++++++++ converters/omni/tests/conftest.py | 17 +++++++++++++++++ .../fixtures/fixtureA_omni/relationships.yaml | 17 +++++++++++++++++ .../fixtureA_omni/topics/sales.topic.yaml | 17 +++++++++++++++++ .../fixtureA_omni/views/customer.view.yaml | 17 +++++++++++++++++ .../fixtureA_omni/views/orders.view.yaml | 17 +++++++++++++++++ .../omni/tests/fixtures/fixtureA_osi.yaml | 17 +++++++++++++++++ .../tests/fixtures/fixtureB_omni/model.yaml | 17 +++++++++++++++++ .../fixtures/fixtureB_omni/relationships.yaml | 17 +++++++++++++++++ .../topics/order_analysis.topic.yaml | 17 +++++++++++++++++ .../fixtureB_omni/views/order_items.view.yaml | 17 +++++++++++++++++ .../fixtureB_omni/views/orders.view.yaml | 17 +++++++++++++++++ .../fixtureB_omni/views/users.view.yaml | 17 +++++++++++++++++ .../fixtures/tpcds_omni/relationships.yaml | 17 +++++++++++++++++ .../topics/tpcds_retail_model.topic.yaml | 17 +++++++++++++++++ .../tpcds_omni/views/customer.view.yaml | 17 +++++++++++++++++ .../tpcds_omni/views/date_dim.view.yaml | 17 +++++++++++++++++ .../fixtures/tpcds_omni/views/item.view.yaml | 17 +++++++++++++++++ .../fixtures/tpcds_omni/views/store.view.yaml | 17 +++++++++++++++++ .../tpcds_omni/views/store_sales.view.yaml | 17 +++++++++++++++++ converters/omni/tests/test_omni_to_osi.py | 17 +++++++++++++++++ converters/omni/tests/test_osi_to_omni.py | 17 +++++++++++++++++ converters/omni/tests/test_real_world_layout.py | 17 +++++++++++++++++ converters/omni/tests/test_roundtrip.py | 17 +++++++++++++++++ .../omni/tests/test_roundtrip_properties.py | 17 +++++++++++++++++ converters/orionbelt/pyproject.toml | 17 +++++++++++++++++ .../orionbelt/src/ossie_orionbelt/__init__.py | 17 +++++++++++++++++ .../orionbelt/src/ossie_orionbelt/_common.py | 17 +++++++++++++++++ converters/orionbelt/src/ossie_orionbelt/cli.py | 17 +++++++++++++++++ .../orionbelt/src/ossie_orionbelt/converter.py | 16 ++++++++++++++++ .../src/ossie_orionbelt/obml_to_osi.py | 17 +++++++++++++++++ .../orionbelt/src/ossie_orionbelt/ontology.py | 17 +++++++++++++++++ .../src/ossie_orionbelt/osi_to_obml.py | 17 +++++++++++++++++ .../orionbelt/src/ossie_orionbelt/validation.py | 17 +++++++++++++++++ .../orionbelt/tests/fixtures/obml_as_osi.yaml | 17 +++++++++++++++++ .../orionbelt/tests/fixtures/tpcds_as_obml.yaml | 17 +++++++++++++++++ .../orionbelt/tests/fixtures/tpcds_osi.yaml | 17 +++++++++++++++++ .../tests/fixtures/tpcds_semantic_model.yaml | 17 +++++++++++++++++ .../tests/test_osi_converter_cumulative.py | 17 +++++++++++++++++ .../tests/test_osi_converter_filters.py | 17 +++++++++++++++++ .../test_osi_converter_measure_overrides.py | 17 +++++++++++++++++ .../tests/test_osi_converter_ontology.py | 17 +++++++++++++++++ .../orionbelt/tests/test_osi_converter_pop.py | 17 +++++++++++++++++ .../tests/test_osi_converter_properties.py | 17 +++++++++++++++++ .../tests/test_osi_converter_trend_v26.py | 17 +++++++++++++++++ .../tests/test_osi_converter_vendors.py | 17 +++++++++++++++++ .../tests/test_osi_metric_no_silent_loss.py | 17 +++++++++++++++++ .../orionbelt/tests/test_osi_tpcds_baseline.py | 17 +++++++++++++++++ .../orionbelt/tests/test_osi_v02_compat.py | 17 +++++++++++++++++ 60 files changed, 1002 insertions(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index d645695..261eeb9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,3 @@ - Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ diff --git a/converters/honeydew/pyproject.toml b/converters/honeydew/pyproject.toml index 82df220..34447ae 100644 --- a/converters/honeydew/pyproject.toml +++ b/converters/honeydew/pyproject.toml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [project] name = "honeydew-osi" version = "0.2.0.dev0" diff --git a/converters/honeydew/src/honeydew_osi_converter.py b/converters/honeydew/src/honeydew_osi_converter.py index 31f64bf..5bfe200 100644 --- a/converters/honeydew/src/honeydew_osi_converter.py +++ b/converters/honeydew/src/honeydew_osi_converter.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Bidirectional converter between OSI and Honeydew semantic model formats. diff --git a/converters/honeydew/tests/test_honeydew_osi_converter.py b/converters/honeydew/tests/test_honeydew_osi_converter.py index 08c2265..4d09ae6 100644 --- a/converters/honeydew/tests/test_honeydew_osi_converter.py +++ b/converters/honeydew/tests/test_honeydew_osi_converter.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Tests for the bidirectional OSI ↔ Honeydew converter.""" import json diff --git a/converters/omni/pyproject.toml b/converters/omni/pyproject.toml index b4be76f..e8bc633 100644 --- a/converters/omni/pyproject.toml +++ b/converters/omni/pyproject.toml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/converters/omni/src/osi_omni/__init__.py b/converters/omni/src/osi_omni/__init__.py index 8eab424..ca2ffdf 100644 --- a/converters/omni/src/osi_omni/__init__.py +++ b/converters/omni/src/osi_omni/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Bidirectional converter between OSI semantic models and Omni semantic model files (model.yaml / relationships.yaml / views/*.view.yaml / topics/*.topic.yaml). Pure offline transforms: OSI YAML string <-> {relative filename: YAML string}. diff --git a/converters/omni/src/osi_omni/_common.py b/converters/omni/src/osi_omni/_common.py index 571b525..ad799bc 100644 --- a/converters/omni/src/osi_omni/_common.py +++ b/converters/omni/src/osi_omni/_common.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Shared helpers for the OSI <-> Omni converters. Both directions are pure offline YAML transforms. The cross-cutting concerns live diff --git a/converters/omni/src/osi_omni/cli.py b/converters/omni/src/osi_omni/cli.py index 70fc7a1..17af022 100644 --- a/converters/omni/src/osi_omni/cli.py +++ b/converters/omni/src/osi_omni/cli.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Command-line interface for the OSI <-> Omni converter. osi-omni export -i model.yaml -o omni_model/ [--base-view orders] [--dialect SNOWFLAKE] diff --git a/converters/omni/src/osi_omni/omni_to_osi.py b/converters/omni/src/osi_omni/omni_to_osi.py index b627509..75a1d3b 100644 --- a/converters/omni/src/osi_omni/omni_to_osi.py +++ b/converters/omni/src/osi_omni/omni_to_osi.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Convert Omni semantic model files to an OSI semantic model. Pure offline conversion. Accepts the Omni model-directory layout as a mapping of diff --git a/converters/omni/src/osi_omni/osi_to_omni.py b/converters/omni/src/osi_omni/osi_to_omni.py index bcbb351..ffda2b8 100644 --- a/converters/omni/src/osi_omni/osi_to_omni.py +++ b/converters/omni/src/osi_omni/osi_to_omni.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Convert an OSI semantic model to Omni semantic model files. Pure offline conversion -- no Omni connection required. Produces the Omni diff --git a/converters/omni/tests/_roundtrip_helpers.py b/converters/omni/tests/_roundtrip_helpers.py index bf003af..3c55d65 100644 --- a/converters/omni/tests/_roundtrip_helpers.py +++ b/converters/omni/tests/_roundtrip_helpers.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Shared model builders and round-trip assertions for property-based tests. This module is deliberately free of any third-party test dependency (no diff --git a/converters/omni/tests/_util.py b/converters/omni/tests/_util.py index f0e2812..2cb1599 100644 --- a/converters/omni/tests/_util.py +++ b/converters/omni/tests/_util.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Shared test helpers: fixture loading and structural normalization.""" import copy diff --git a/converters/omni/tests/conftest.py b/converters/omni/tests/conftest.py index 282f013..0bcde53 100644 --- a/converters/omni/tests/conftest.py +++ b/converters/omni/tests/conftest.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pathlib import sys diff --git a/converters/omni/tests/fixtures/fixtureA_omni/relationships.yaml b/converters/omni/tests/fixtures/fixtureA_omni/relationships.yaml index 04f7b21..75e293b 100644 --- a/converters/omni/tests/fixtures/fixtureA_omni/relationships.yaml +++ b/converters/omni/tests/fixtures/fixtureA_omni/relationships.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + - join_from_view: orders join_to_view: customer on_sql: ${orders.o_custkey} = ${customer.c_custkey} diff --git a/converters/omni/tests/fixtures/fixtureA_omni/topics/sales.topic.yaml b/converters/omni/tests/fixtures/fixtureA_omni/topics/sales.topic.yaml index 6324461..f3df13b 100644 --- a/converters/omni/tests/fixtures/fixtureA_omni/topics/sales.topic.yaml +++ b/converters/omni/tests/fixtures/fixtureA_omni/topics/sales.topic.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + base_view: orders description: Sales orders with customer attributes ai_context: Use this model for order analysis. diff --git a/converters/omni/tests/fixtures/fixtureA_omni/views/customer.view.yaml b/converters/omni/tests/fixtures/fixtureA_omni/views/customer.view.yaml index 5706a4d..6ebe1e1 100644 --- a/converters/omni/tests/fixtures/fixtureA_omni/views/customer.view.yaml +++ b/converters/omni/tests/fixtures/fixtureA_omni/views/customer.view.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + catalog: samples schema: tpch dimensions: diff --git a/converters/omni/tests/fixtures/fixtureA_omni/views/orders.view.yaml b/converters/omni/tests/fixtures/fixtureA_omni/views/orders.view.yaml index f61582a..77cda90 100644 --- a/converters/omni/tests/fixtures/fixtureA_omni/views/orders.view.yaml +++ b/converters/omni/tests/fixtures/fixtureA_omni/views/orders.view.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + catalog: samples schema: tpch description: One row per order diff --git a/converters/omni/tests/fixtures/fixtureA_osi.yaml b/converters/omni/tests/fixtures/fixtureA_osi.yaml index b6e6863..0cb830f 100644 --- a/converters/omni/tests/fixtures/fixtureA_osi.yaml +++ b/converters/omni/tests/fixtures/fixtureA_osi.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # yaml-language-server: $schema=../../../../core-spec/osi-schema.json # # Fixture A -- all-native round-trip (OSI -> Omni -> OSI). diff --git a/converters/omni/tests/fixtures/fixtureB_omni/model.yaml b/converters/omni/tests/fixtures/fixtureB_omni/model.yaml index d265896..b9602ec 100644 --- a/converters/omni/tests/fixtures/fixtureB_omni/model.yaml +++ b/converters/omni/tests/fixtures/fixtureB_omni/model.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Fixture B -- stash round-trip (Omni -> OSI -> Omni, lossless). # The model file has no OSI home and round-trips via the model-level stash. included_schemas: diff --git a/converters/omni/tests/fixtures/fixtureB_omni/relationships.yaml b/converters/omni/tests/fixtures/fixtureB_omni/relationships.yaml index cd91d26..6dbc77e 100644 --- a/converters/omni/tests/fixtures/fixtureB_omni/relationships.yaml +++ b/converters/omni/tests/fixtures/fixtureB_omni/relationships.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Exercises: non-default join_type, reversible, a composite-key one_to_many # join (flipped to OSI's many-side-first orientation and flipped back on export). - join_from_view: order_items diff --git a/converters/omni/tests/fixtures/fixtureB_omni/topics/order_analysis.topic.yaml b/converters/omni/tests/fixtures/fixtureB_omni/topics/order_analysis.topic.yaml index 57d0eae..9107f42 100644 --- a/converters/omni/tests/fixtures/fixtureB_omni/topics/order_analysis.topic.yaml +++ b/converters/omni/tests/fixtures/fixtureB_omni/topics/order_analysis.topic.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Topic curation (label, joins, default_filters, fields) has no OSI home and # round-trips via the model-level stash; description/ai_context map natively. base_view: order_items diff --git a/converters/omni/tests/fixtures/fixtureB_omni/views/order_items.view.yaml b/converters/omni/tests/fixtures/fixtureB_omni/views/order_items.view.yaml index ca84597..07f479c 100644 --- a/converters/omni/tests/fixtures/fixtureB_omni/views/order_items.view.yaml +++ b/converters/omni/tests/fixtures/fixtureB_omni/views/order_items.view.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Exercises the dimension/measure stash: format, hidden, group_label, # timeframes, convert_tz, a quoted-identifier sql, a ${field} reference, a # filtered measure, and a percentile measure. diff --git a/converters/omni/tests/fixtures/fixtureB_omni/views/orders.view.yaml b/converters/omni/tests/fixtures/fixtureB_omni/views/orders.view.yaml index 3264995..f20a220 100644 --- a/converters/omni/tests/fixtures/fixtureB_omni/views/orders.view.yaml +++ b/converters/omni/tests/fixtures/fixtureB_omni/views/orders.view.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + schema: ecomm dimensions: order_id: diff --git a/converters/omni/tests/fixtures/fixtureB_omni/views/users.view.yaml b/converters/omni/tests/fixtures/fixtureB_omni/views/users.view.yaml index 5b42d8b..4cf2572 100644 --- a/converters/omni/tests/fixtures/fixtureB_omni/views/users.view.yaml +++ b/converters/omni/tests/fixtures/fixtureB_omni/views/users.view.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + schema: ecomm dimensions: id: diff --git a/converters/omni/tests/fixtures/tpcds_omni/relationships.yaml b/converters/omni/tests/fixtures/tpcds_omni/relationships.yaml index 4a70f2d..33c0abb 100644 --- a/converters/omni/tests/fixtures/tpcds_omni/relationships.yaml +++ b/converters/omni/tests/fixtures/tpcds_omni/relationships.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + - join_from_view: store_sales join_to_view: date_dim on_sql: ${store_sales.ss_sold_date_sk} = ${date_dim.d_date_sk} diff --git a/converters/omni/tests/fixtures/tpcds_omni/topics/tpcds_retail_model.topic.yaml b/converters/omni/tests/fixtures/tpcds_omni/topics/tpcds_retail_model.topic.yaml index 51fcb06..783d4cf 100644 --- a/converters/omni/tests/fixtures/tpcds_omni/topics/tpcds_retail_model.topic.yaml +++ b/converters/omni/tests/fixtures/tpcds_omni/topics/tpcds_retail_model.topic.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + base_view: store_sales description: TPC-DS retail semantic model for sales and customer analytics ai_context: Use this semantic model for retail analytics. It provides comprehensive diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/customer.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/customer.view.yaml index c70c8c0..c195c6d 100644 --- a/converters/omni/tests/fixtures/tpcds_omni/views/customer.view.yaml +++ b/converters/omni/tests/fixtures/tpcds_omni/views/customer.view.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + catalog: tpcds schema: public description: Customer dimension with demographic information diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/date_dim.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/date_dim.view.yaml index dcc7160..068c808 100644 --- a/converters/omni/tests/fixtures/tpcds_omni/views/date_dim.view.yaml +++ b/converters/omni/tests/fixtures/tpcds_omni/views/date_dim.view.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + catalog: tpcds schema: public description: Date dimension with calendar attributes diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/item.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/item.view.yaml index c4fa8c2..08436b3 100644 --- a/converters/omni/tests/fixtures/tpcds_omni/views/item.view.yaml +++ b/converters/omni/tests/fixtures/tpcds_omni/views/item.view.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + catalog: tpcds schema: public description: Item/Product dimension with product attributes diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/store.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/store.view.yaml index 5a9520d..342a480 100644 --- a/converters/omni/tests/fixtures/tpcds_omni/views/store.view.yaml +++ b/converters/omni/tests/fixtures/tpcds_omni/views/store.view.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + catalog: tpcds schema: public description: Store dimension with location and store attributes diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/store_sales.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/store_sales.view.yaml index 0a21118..084dc55 100644 --- a/converters/omni/tests/fixtures/tpcds_omni/views/store_sales.view.yaml +++ b/converters/omni/tests/fixtures/tpcds_omni/views/store_sales.view.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + catalog: tpcds schema: public description: Fact table containing all store sales transactions diff --git a/converters/omni/tests/test_omni_to_osi.py b/converters/omni/tests/test_omni_to_osi.py index b124808..704d544 100644 --- a/converters/omni/tests/test_omni_to_osi.py +++ b/converters/omni/tests/test_omni_to_osi.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Unit tests for the Omni -> OSI importer.""" import json diff --git a/converters/omni/tests/test_osi_to_omni.py b/converters/omni/tests/test_osi_to_omni.py index ba0fb61..e6d1d0a 100644 --- a/converters/omni/tests/test_osi_to_omni.py +++ b/converters/omni/tests/test_osi_to_omni.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Unit tests for the OSI -> Omni exporter.""" import warnings diff --git a/converters/omni/tests/test_real_world_layout.py b/converters/omni/tests/test_real_world_layout.py index b58a159..c78864d 100644 --- a/converters/omni/tests/test_real_world_layout.py +++ b/converters/omni/tests/test_real_world_layout.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Round-trip tests for the model shape Omni's API/IDE actually emits. A model pulled from a real instance (`omni models yaml-get`) differs from the diff --git a/converters/omni/tests/test_roundtrip.py b/converters/omni/tests/test_roundtrip.py index baccb52..6c4f5dd 100644 --- a/converters/omni/tests/test_roundtrip.py +++ b/converters/omni/tests/test_roundtrip.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Fixture-based round-trip tests. - Omni -> OSI -> Omni must be lossless (the stash carries everything). diff --git a/converters/omni/tests/test_roundtrip_properties.py b/converters/omni/tests/test_roundtrip_properties.py index 445c1e2..4d2973d 100644 --- a/converters/omni/tests/test_roundtrip_properties.py +++ b/converters/omni/tests/test_roundtrip_properties.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Property-based round-trip tests. Two drivers over the same builders (see _roundtrip_helpers): diff --git a/converters/orionbelt/pyproject.toml b/converters/orionbelt/pyproject.toml index d039657..56a675e 100644 --- a/converters/orionbelt/pyproject.toml +++ b/converters/orionbelt/pyproject.toml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/converters/orionbelt/src/ossie_orionbelt/__init__.py b/converters/orionbelt/src/ossie_orionbelt/__init__.py index 5dc24fa..3471599 100644 --- a/converters/orionbelt/src/ossie_orionbelt/__init__.py +++ b/converters/orionbelt/src/ossie_orionbelt/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ossie-orionbelt: bidirectional OBML <-> OSI converter. Converts between OrionBelt Markup Language (OBML) semantic models and Open diff --git a/converters/orionbelt/src/ossie_orionbelt/_common.py b/converters/orionbelt/src/ossie_orionbelt/_common.py index 3460381..f0e73a3 100644 --- a/converters/orionbelt/src/ossie_orionbelt/_common.py +++ b/converters/orionbelt/src/ossie_orionbelt/_common.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Shared constants and mapping tables for the OSI ↔ OBML converter. These module-level constants are used by more than one of the converter diff --git a/converters/orionbelt/src/ossie_orionbelt/cli.py b/converters/orionbelt/src/ossie_orionbelt/cli.py index 196722f..ec70aef 100644 --- a/converters/orionbelt/src/ossie_orionbelt/cli.py +++ b/converters/orionbelt/src/ossie_orionbelt/cli.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Command-line entry point for the OBML <-> OSI converter. A single ``ossie-orionbelt`` command with two format-named subcommands, mirroring diff --git a/converters/orionbelt/src/ossie_orionbelt/converter.py b/converters/orionbelt/src/ossie_orionbelt/converter.py index 9d08ff8..ae3f5d7 100644 --- a/converters/orionbelt/src/ossie_orionbelt/converter.py +++ b/converters/orionbelt/src/ossie_orionbelt/converter.py @@ -1,4 +1,20 @@ #!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. """ OSI ↔ OBML Bidirectional Converter =================================== diff --git a/converters/orionbelt/src/ossie_orionbelt/obml_to_osi.py b/converters/orionbelt/src/ossie_orionbelt/obml_to_osi.py index 9f3daca..db9ce97 100644 --- a/converters/orionbelt/src/ossie_orionbelt/obml_to_osi.py +++ b/converters/orionbelt/src/ossie_orionbelt/obml_to_osi.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """OBML → OSI conversion (the :class:`OBMLtoOSI` direction). Extracted verbatim from ``converter.py``; see that module for the package-level diff --git a/converters/orionbelt/src/ossie_orionbelt/ontology.py b/converters/orionbelt/src/ossie_orionbelt/ontology.py index 86886c2..9715c5b 100644 --- a/converters/orionbelt/src/ossie_orionbelt/ontology.py +++ b/converters/orionbelt/src/ossie_orionbelt/ontology.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """OBML → OSI ontology derivation (the :class:`OBMLtoOSIOntology` direction). Extracted verbatim from ``converter.py``; reuses :class:`OBMLtoOSI` for the diff --git a/converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py b/converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py index cc3e4fe..52a8c23 100644 --- a/converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py +++ b/converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """OSI → OBML conversion (the :class:`OSItoOBML` direction). Extracted verbatim from ``converter.py``; see that module for the package-level diff --git a/converters/orionbelt/src/ossie_orionbelt/validation.py b/converters/orionbelt/src/ossie_orionbelt/validation.py index 64c9f51..4f0c137 100644 --- a/converters/orionbelt/src/ossie_orionbelt/validation.py +++ b/converters/orionbelt/src/ossie_orionbelt/validation.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Validation helpers for OBML / OSI / OSI-ontology documents. Extracted verbatim from ``converter.py``. Schema files are vendored beside the diff --git a/converters/orionbelt/tests/fixtures/obml_as_osi.yaml b/converters/orionbelt/tests/fixtures/obml_as_osi.yaml index 7d8fe87..ef055ac 100644 --- a/converters/orionbelt/tests/fixtures/obml_as_osi.yaml +++ b/converters/orionbelt/tests/fixtures/obml_as_osi.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + version: '0.1.1' semantic_model: - name: sales_analytics diff --git a/converters/orionbelt/tests/fixtures/tpcds_as_obml.yaml b/converters/orionbelt/tests/fixtures/tpcds_as_obml.yaml index acb8f0f..94a92ca 100644 --- a/converters/orionbelt/tests/fixtures/tpcds_as_obml.yaml +++ b/converters/orionbelt/tests/fixtures/tpcds_as_obml.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + version: 1.0 dataObjects: store_sales: diff --git a/converters/orionbelt/tests/fixtures/tpcds_osi.yaml b/converters/orionbelt/tests/fixtures/tpcds_osi.yaml index b038f3e..f59b7f7 100644 --- a/converters/orionbelt/tests/fixtures/tpcds_osi.yaml +++ b/converters/orionbelt/tests/fixtures/tpcds_osi.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # yaml-language-server: $schema=../core-spec/osi-schema.json # TPC-DS Semantic Model Example diff --git a/converters/orionbelt/tests/fixtures/tpcds_semantic_model.yaml b/converters/orionbelt/tests/fixtures/tpcds_semantic_model.yaml index 7dd2785..a677c82 100644 --- a/converters/orionbelt/tests/fixtures/tpcds_semantic_model.yaml +++ b/converters/orionbelt/tests/fixtures/tpcds_semantic_model.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # yaml-language-server: $schema=../core-spec/osi-schema.json # TPC-DS Semantic Model Example diff --git a/converters/orionbelt/tests/test_osi_converter_cumulative.py b/converters/orionbelt/tests/test_osi_converter_cumulative.py index c19fa80..1897ac0 100644 --- a/converters/orionbelt/tests/test_osi_converter_cumulative.py +++ b/converters/orionbelt/tests/test_osi_converter_cumulative.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Tests for OSI ↔ OBML cumulative metric conversion. Validates that cumulative metrics survive the OBML → OSI → OBML roundtrip diff --git a/converters/orionbelt/tests/test_osi_converter_filters.py b/converters/orionbelt/tests/test_osi_converter_filters.py index 91be10a..ccc5ef5 100644 --- a/converters/orionbelt/tests/test_osi_converter_filters.py +++ b/converters/orionbelt/tests/test_osi_converter_filters.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Tests for OSI ↔ OBML static filter roundtrip. Validates that model-level static filters survive the OBML → OSI → OBML diff --git a/converters/orionbelt/tests/test_osi_converter_measure_overrides.py b/converters/orionbelt/tests/test_osi_converter_measure_overrides.py index c610dc9..0f69c29 100644 --- a/converters/orionbelt/tests/test_osi_converter_measure_overrides.py +++ b/converters/orionbelt/tests/test_osi_converter_measure_overrides.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Tests for OSI ↔ OBML roundtrip of measure grain and filterContext properties. Validates that grain override and filterContext configurations survive the diff --git a/converters/orionbelt/tests/test_osi_converter_ontology.py b/converters/orionbelt/tests/test_osi_converter_ontology.py index 7669387..b4b4d51 100644 --- a/converters/orionbelt/tests/test_osi_converter_ontology.py +++ b/converters/orionbelt/tests/test_osi_converter_ontology.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Tests for the OBML → OSI **ontology** converter (OBMLtoOSIOntology). Validates that the derived ontology document passes the converter's semantic diff --git a/converters/orionbelt/tests/test_osi_converter_pop.py b/converters/orionbelt/tests/test_osi_converter_pop.py index c1fdcf3..f060159 100644 --- a/converters/orionbelt/tests/test_osi_converter_pop.py +++ b/converters/orionbelt/tests/test_osi_converter_pop.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Tests for OSI ↔ OBML period-over-period metric conversion. Validates that PoP metrics survive the OBML → OSI → OBML roundtrip diff --git a/converters/orionbelt/tests/test_osi_converter_properties.py b/converters/orionbelt/tests/test_osi_converter_properties.py index 0e33d0b..4d07303 100644 --- a/converters/orionbelt/tests/test_osi_converter_properties.py +++ b/converters/orionbelt/tests/test_osi_converter_properties.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Tests for OSI ↔ OBML roundtrip of data types, settings, owner, and column properties. Validates that recently added OBML properties survive the OBML → OSI → OBML diff --git a/converters/orionbelt/tests/test_osi_converter_trend_v26.py b/converters/orionbelt/tests/test_osi_converter_trend_v26.py index 0d950e0..f9f1d79 100644 --- a/converters/orionbelt/tests/test_osi_converter_trend_v26.py +++ b/converters/orionbelt/tests/test_osi_converter_trend_v26.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """OSI ↔ OBML roundtrip for v2.6 trend-analysis fields. Validates that: diff --git a/converters/orionbelt/tests/test_osi_converter_vendors.py b/converters/orionbelt/tests/test_osi_converter_vendors.py index 05eae9b..79b00ce 100644 --- a/converters/orionbelt/tests/test_osi_converter_vendors.py +++ b/converters/orionbelt/tests/test_osi_converter_vendors.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Vendor-identity rules for OSI custom_extensions. - OBML -> OSI tags OrionBelt-proprietary payloads as ``ORIONBELT``. diff --git a/converters/orionbelt/tests/test_osi_metric_no_silent_loss.py b/converters/orionbelt/tests/test_osi_metric_no_silent_loss.py index f843d75..3a9d1d3 100644 --- a/converters/orionbelt/tests/test_osi_metric_no_silent_loss.py +++ b/converters/orionbelt/tests/test_osi_metric_no_silent_loss.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Third-party OSI metrics must never be silently lost on import. Regression coverage for the review on OSI PR #153: the converter only emits diff --git a/converters/orionbelt/tests/test_osi_tpcds_baseline.py b/converters/orionbelt/tests/test_osi_tpcds_baseline.py index 3165b08..43ca284 100644 --- a/converters/orionbelt/tests/test_osi_tpcds_baseline.py +++ b/converters/orionbelt/tests/test_osi_tpcds_baseline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """OSI TPC-DS baseline (the converter-contributing step-5 requirement). Exercises the conceptual conversion flow from the OSI converters guide against diff --git a/converters/orionbelt/tests/test_osi_v02_compat.py b/converters/orionbelt/tests/test_osi_v02_compat.py index f5c110b..144b2b1 100644 --- a/converters/orionbelt/tests/test_osi_v02_compat.py +++ b/converters/orionbelt/tests/test_osi_v02_compat.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """OSI v0.2 compatibility tests. Covers the v2.6 spec bump from OSI v0.1.1 → v0.2.0.dev0: From 09f8c83c479085c0f66ba1ce3754da04018afdff Mon Sep 17 00:00:00 2001 From: Ralfo Becher Date: Fri, 17 Jul 2026 07:32:22 +0200 Subject: [PATCH 76/89] orionbelt converter: fix round-trip fidelity + validation robustness (#206) Mirrors orionbelt-semantic-layer#201 (canonical source) into the ossie converter: - Metric round-trip: OSItoOBML resolves metric references by BOTH the OSI dataset/field name and the physical code (source-table code and field expression, quoted or bare), so metrics emitted against physical codes (e.g. SUM(fact_orders.amount)) round-trip instead of dropping when a data object's code differs from its display name. Resolved refs are bracket-quoted so display names with spaces stay intact through the dataset.column parsers. - Dimension collision: _extract_dimensions qualifies a field name that occurs in more than one dataset (Orders.date / Invoices.date) with its data object and warns, instead of silently overwriting the earlier dimension. - Validation robustness: validate_osi guards its semantic loops against malformed structures (datasets/fields that are not lists of dicts) and returns schema errors instead of raising AttributeError. Adds tests/test_osi_converter_roundtrip_robustness.py. 154 converter tests pass; ruff clean; mypy clean with the dev extra. --- .../src/ossie_orionbelt/osi_to_obml.py | 122 ++++++-- .../src/ossie_orionbelt/validation.py | 28 +- ...test_osi_converter_roundtrip_robustness.py | 296 ++++++++++++++++++ 3 files changed, 415 insertions(+), 31 deletions(-) create mode 100644 converters/orionbelt/tests/test_osi_converter_roundtrip_robustness.py diff --git a/converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py b/converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py index 52a8c23..6ca635d 100644 --- a/converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py +++ b/converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py @@ -37,6 +37,12 @@ OSI_TO_OBML_TYPE, ) +# A dataset/column identifier in a resolved metric expression: either a bare SQL +# word or a bracket-quoted token. ``_resolve_column_refs`` bracket-quotes any +# canonical name that is not a bare word (e.g. a display name with spaces) so the +# downstream ``dataset.column`` parsers below never split it on whitespace. +_RESOLVED_IDENT = r"(?:\w+|\[[^\]]+\])" + class OSItoOBML: """Convert an OSI semantic model YAML to OBML format.""" @@ -595,7 +601,23 @@ def _extract_dimensions(self, datasets: list) -> dict: except (json.JSONDecodeError, TypeError): pass break - dimensions[field_name] = dim_def + # Dimension names must be unique across the model. When the same + # field name occurs in more than one data object (e.g. Orders.date + # and Invoices.date), qualify the later one with its data object + # instead of silently overwriting the earlier dimension. + key = field_name + if key in dimensions and dimensions[key].get("dataObject") != ds_name: + key = f"{ds_name} {field_name}" + suffix = 2 + while key in dimensions: + key = f"{ds_name} {field_name} {suffix}" + suffix += 1 + self.warnings.append( + f"Dimension name '{field_name}' occurs in multiple data " + f"objects; emitted '{ds_name}.{field_name}' as dimension " + f"'{key}' to avoid a collision." + ) + dimensions[key] = dim_def return dimensions def _convert_metrics(self, osi_metrics: list, ds_map: dict) -> tuple[dict, dict]: @@ -617,16 +639,35 @@ def _convert_metrics(self, osi_metrics: list, ds_map: dict) -> tuple[dict, dict] # Case-insensitive dataset/field index for resolving SQL identifiers # back to their canonical OSI names (Snowflake/Databricks expressions - # commonly upper-case or quote them). - ds_lc = {name.lower(): name for name in ds_map} - fields_lc = { - name: { - f["name"].lower(): f["name"] - for f in ds.get("fields", []) or [] - if isinstance(f, dict) and f.get("name") - } - for name, ds in ds_map.items() - } + # commonly upper-case or quote them). Identifiers resolve by BOTH the + # OSI name and the physical code (source-table code / bare-identifier + # field expression): our own OBML -> OSI emitter writes metric SQL + # against the physical code (e.g. SUM(fact_orders.amount)), so resolving + # names only would drop such metrics on the return trip. Names take + # precedence over codes on any collision. + ds_lc: dict[str, str] = {} + for ds_name in ds_map: + ds_lc.setdefault(ds_name.lower(), ds_name) + for ds_name, ds in ds_map.items(): + # Unquote so a quoted source table (Snowflake/Databricks style, + # e.g. WH.PUBLIC."fact_orders") is indexed by its bare code. + table_code = self._unquote_identifier(self._parse_source(ds.get("source", ds_name))[2]) + if table_code: + ds_lc.setdefault(table_code.lower(), ds_name) + + fields_lc: dict[str, dict[str, str]] = {} + for ds_name, ds in ds_map.items(): + fmap: dict[str, str] = {} + osi_fields = ds.get("fields", []) or [] + for f in osi_fields: + if isinstance(f, dict) and f.get("name"): + fmap.setdefault(f["name"].lower(), f["name"]) + for f in osi_fields: + if isinstance(f, dict) and f.get("name"): + code = self._field_expr_identifier(f) + if code: + fmap.setdefault(code.lower(), f["name"]) + fields_lc[ds_name] = fmap for m in osi_metrics: name = m["name"] @@ -990,6 +1031,33 @@ def _unquote_identifier(ident: str) -> str: return ident[1:-1] return ident + @staticmethod + def _q_ident(name: str) -> str: + """Bracket-quote a canonical name that is not a bare SQL word, so the + downstream ``dataset.column`` parsers (which split on ``\\w+``) do not + break on display names containing spaces or other punctuation.""" + return name if re.fullmatch(r"\w+", name) else f"[{name}]" + + @staticmethod + def _field_expr_identifier(field: dict) -> str | None: + """Physical column code of a field when its expression is a single + (optionally quoted) identifier, so code-based metric references (e.g. + ``fact_orders.amount`` or a Snowflake ``"net_amount"``) resolve back to + the field. Returns ``None`` for computed expressions with no single + column code. + """ + expr = field.get("expression") + if not isinstance(expr, dict): + return None + for dialect in expr.get("dialects", []) or []: + if isinstance(dialect, dict): + text = dialect.get("expression") + if isinstance(text, str): + candidate = OSItoOBML._unquote_identifier(text) + if re.fullmatch(r"[A-Za-z_]\w*", candidate): + return candidate + return None + def _resolve_column_refs( self, expr: str, ds_lc: dict[str, str], fields_lc: dict[str, dict[str, str]] ) -> str | None: @@ -1016,7 +1084,9 @@ def repl(match: re.Match[str]) -> str: if col_real is None: unresolved = True return match.group(0) - return f"{ds_real}.{col_real}" + # Bracket-quote names that are not bare words so the downstream + # dataset.column parsers keep multi-word display names intact. + return f"{self._q_ident(ds_real)}.{self._q_ident(col_real)}" rewritten = _COLUMN_REF_RE.sub(repl, expr) return None if unresolved else rewritten @@ -1044,13 +1114,13 @@ def _parse_simple_agg(self, expr: str) -> tuple | None: import re expr = expr.strip() - pattern = r"^(\w+)\(\s*(DISTINCT\s+)?(\w+)\.(\w+)\s*\)$" + pattern = rf"^(\w+)\(\s*(DISTINCT\s+)?({_RESOLVED_IDENT})\.({_RESOLVED_IDENT})\s*\)$" match = re.match(pattern, expr, re.IGNORECASE) if match: agg = match.group(1) is_distinct = match.group(2) is not None - dataset = match.group(3) - column = match.group(4) + dataset = self._unquote_identifier(match.group(3)) + column = self._unquote_identifier(match.group(4)) return agg, dataset, column, is_distinct return None @@ -1087,10 +1157,10 @@ def _parse_expr_agg(self, expr: str) -> tuple | None: return None # Unmatched open paren # Must contain dataset.column references - if not re.search(r"\w+\.\w+", inner): + if not re.search(rf"{_RESOLVED_IDENT}\.{_RESOLVED_IDENT}", inner): return None # Must NOT be a simple dataset.column (already handled by _parse_simple_agg) - if re.match(r"^(DISTINCT\s+)?\w+\.\w+$", inner, re.IGNORECASE): + if re.match(rf"^(DISTINCT\s+)?{_RESOLVED_IDENT}\.{_RESOLVED_IDENT}$", inner, re.IGNORECASE): return None # Must NOT contain nested aggregation calls (those are complex metrics) if re.search(r"\b(" + "|".join(agg_funcs) + r")\s*\(", inner, re.IGNORECASE): @@ -1106,7 +1176,14 @@ def _sql_refs_to_obml(self, sql_expr: str) -> str: (resolved upstream), so the bare-identifier branch is what fires. """ return _COLUMN_REF_RE.sub( - lambda m: "{[" + m.group("ds") + "].[" + m.group("col") + "]}", sql_expr + lambda m: ( + "{[" + + self._unquote_identifier(m.group("ds")) + + "].[" + + self._unquote_identifier(m.group("col")) + + "]}" + ), + sql_expr, ) def _decompose_complex_metric(self, name: str, expr: str) -> tuple[str, dict]: @@ -1163,12 +1240,13 @@ def _decompose_complex_metric(self, name: str, expr: str) -> tuple[str, dict]: inner_clean = inner[dm.end() :].strip() # Is it a simple dataset.column? - simple = re.match(r"^(\w+)\.(\w+)$", inner_clean) + simple = re.match(rf"^({_RESOLVED_IDENT})\.({_RESOLVED_IDENT})$", inner_clean) if simple: - dataset = simple.group(1) - column = simple.group(2) + dataset = self._unquote_identifier(simple.group(1)) + column = self._unquote_identifier(simple.group(2)) suffix = "_distinct" if is_distinct else "" - measure_key = f"_{dataset}_{column}_{agg.lower()}{suffix}" + key_stub = re.sub(r"\W+", "_", f"{dataset}_{column}") + measure_key = f"_{key_stub}_{agg.lower()}{suffix}" measure_def: dict[str, Any] = { "columns": [{"dataObject": dataset, "column": column}], "resultType": "float", diff --git a/converters/orionbelt/src/ossie_orionbelt/validation.py b/converters/orionbelt/src/ossie_orionbelt/validation.py index 4f0c137..e9ec7eb 100644 --- a/converters/orionbelt/src/ossie_orionbelt/validation.py +++ b/converters/orionbelt/src/ossie_orionbelt/validation.py @@ -206,13 +206,22 @@ def validate_osi(osi_dict: dict[str, Any], schema_path: Path | None = None) -> V # 1. JSON Schema validation (OSI uses Draft 2020-12) _validate_json_schema(osi_dict, schema_path or _OSI_SCHEMA_PATH, result, draft="draft2020") + # The semantic checks below assume a well-formed structure (lists of dicts). + # JSON Schema validation above already reports structural errors, so guard + # every level here rather than raising on malformed input. + def _as_dict_list(value: Any) -> list[dict[str, Any]]: + return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] + + models = _as_dict_list(osi_dict.get("semantic_model", [])) + # 2. Unique name checks - for model in osi_dict.get("semantic_model", []): + for model in models: model_name = model.get("name", "") + datasets = _as_dict_list(model.get("datasets", [])) # Unique dataset names dataset_names: list[str] = [] - for ds in model.get("datasets", []): + for ds in datasets: name = ds.get("name", "") if name in dataset_names: result.semantic_errors.append( @@ -221,10 +230,10 @@ def validate_osi(osi_dict: dict[str, Any], schema_path: Path | None = None) -> V dataset_names.append(name) # Unique field names within each dataset - for ds in model.get("datasets", []): + for ds in datasets: ds_name = ds.get("name", "") field_names: list[str] = [] - for field in ds.get("fields", []): + for field in _as_dict_list(ds.get("fields", [])): fname = field.get("name", "") if fname in field_names: result.semantic_errors.append( @@ -234,7 +243,7 @@ def validate_osi(osi_dict: dict[str, Any], schema_path: Path | None = None) -> V # Unique metric names metric_names: list[str] = [] - for m in model.get("metrics", []): + for m in _as_dict_list(model.get("metrics", [])): mname = m.get("name", "") if mname in metric_names: result.semantic_errors.append( @@ -244,7 +253,7 @@ def validate_osi(osi_dict: dict[str, Any], schema_path: Path | None = None) -> V # Unique relationship names rel_names: list[str] = [] - for r in model.get("relationships", []): + for r in _as_dict_list(model.get("relationships", [])): rname = r.get("name", "") if rname in rel_names: result.semantic_errors.append( @@ -254,9 +263,10 @@ def validate_osi(osi_dict: dict[str, Any], schema_path: Path | None = None) -> V rel_names.append(rname) # 3. Reference checks — relationships reference existing datasets - for model in osi_dict.get("semantic_model", []): - ds_name_set = {ds.get("name") for ds in model.get("datasets", []) if ds.get("name")} - for rel in model.get("relationships", []): + for model in models: + datasets = _as_dict_list(model.get("datasets", [])) + ds_name_set = {ds.get("name") for ds in datasets if ds.get("name")} + for rel in _as_dict_list(model.get("relationships", [])): rel_name = rel.get("name", "") from_ds = rel.get("from") to_ds = rel.get("to") diff --git a/converters/orionbelt/tests/test_osi_converter_roundtrip_robustness.py b/converters/orionbelt/tests/test_osi_converter_roundtrip_robustness.py new file mode 100644 index 0000000..02c79f2 --- /dev/null +++ b/converters/orionbelt/tests/test_osi_converter_roundtrip_robustness.py @@ -0,0 +1,296 @@ +"""Regression tests for issue #201: OSI<->OBML converter round-trip fidelity +and validation robustness. + +Covers three bugs found during the apache/ossie#153 review: + +- P1: metric references broke the OBML -> OSI -> OBML round trip when a data + object's physical ``code`` differs from its display name (the emitter writes + the physical code, the importer resolved only by dataset name). +- P2: dimensions extracted from OSI collided when the same field name occurred + in more than one dataset (the second silently overwrote the first). +- P2: ``validate_osi`` raised ``AttributeError`` on malformed input instead of + returning a result with schema errors. +""" + +from __future__ import annotations + +import json +from typing import Any + +import ossie_orionbelt.converter as conv + +# --------------------------------------------------------------------------- +# P1: metric round-trip when dataObject.code != display name +# --------------------------------------------------------------------------- + + +class TestMetricRoundTripCodeVsName: + _OBML: dict[str, Any] = { + "version": 1.0, + "dataObjects": { + "Orders": { + "code": "fact_orders", # physical table code differs from name + "database": "WH", + "schema": "PUBLIC", + "columns": {"Amount": {"code": "amount", "abstractType": "float"}}, + } + }, + "measures": { + "Revenue": { + "columns": [{"dataObject": "Orders", "column": "Amount"}], + "resultType": "float", + "aggregation": "sum", + } + }, + } + + def test_emitter_uses_physical_code(self) -> None: + osi = conv.OBMLtoOSI(self._OBML).convert() + sql = osi["semantic_model"][0]["metrics"][0]["expression"]["dialects"][0]["expression"] + assert "fact_orders" in sql # confirms the emit side uses the code + + def test_measure_survives_round_trip(self) -> None: + osi = conv.OBMLtoOSI(self._OBML).convert() + obml = conv.OSItoOBML(osi).convert() + + # The measure must come back as a real measure, not be stashed as an + # unconverted metric. + assert "Revenue" in obml.get("measures", {}), obml.get("measures") + rev = obml["measures"]["Revenue"] + assert rev["aggregation"] == "sum" + assert rev["columns"] == [{"dataObject": "Orders", "column": "amount"}] + + # Nothing about Revenue should have leaked into an unconverted-metric stash. + stashed = json.dumps(obml.get("customExtensions", [])) + assert "Revenue" not in stashed + + def test_third_party_field_name_differs_from_code(self) -> None: + # Snowflake-style OSI: field display name != physical column code, and + # the metric references the physical code. Must resolve, not drop. + osi = { + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "sales", + "datasets": [ + { + "name": "Orders", + "source": "WH.PUBLIC.fact_orders", + "fields": [ + { + "name": "Amount", + "expression": { + "dialects": [ + {"dialect": "ANSI_SQL", "expression": "amount"} + ] + }, + "data_type": "number", + } + ], + } + ], + "metrics": [ + { + "name": "Revenue", + "expression": { + "dialects": [ + {"dialect": "ANSI_SQL", "expression": "SUM(fact_orders.amount)"} + ] + }, + } + ], + } + ], + } + obml = conv.OSItoOBML(osi).convert() + assert "Revenue" in obml.get("measures", {}), obml.get("measures") + assert obml["measures"]["Revenue"]["columns"] == [ + {"dataObject": "Orders", "column": "Amount"} + ] + + def test_physical_code_maps_to_field_name_with_space(self) -> None: + # OSI field display name contains a space; the metric references the + # physical column code. Resolution must produce a valid OBML measure + # (column "Net Amount"), not a broken "{[Orders].[Net]} Amount" that + # fails query resolution. + osi = { + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "sales", + "datasets": [ + { + "name": "Orders", + "source": "WH.PUBLIC.fact_orders", + "fields": [ + { + "name": "Net Amount", + "expression": { + "dialects": [ + {"dialect": "ANSI_SQL", "expression": "net_amount"} + ] + }, + "data_type": "number", + } + ], + } + ], + "metrics": [ + { + "name": "Net Revenue", + "expression": { + "dialects": [ + { + "dialect": "ANSI_SQL", + "expression": "SUM(fact_orders.net_amount)", + } + ] + }, + } + ], + } + ], + } + obml = conv.OSItoOBML(osi).convert() + assert "Net Revenue" in obml.get("measures", {}), obml.get("measures") + m = obml["measures"]["Net Revenue"] + assert m["aggregation"] == "sum" + assert m["columns"] == [{"dataObject": "Orders", "column": "Net Amount"}] + # It resolved to a clean single-column measure, not a dangling expression. + assert "expression" not in m + + def test_quoted_physical_identifiers_resolve(self) -> None: + # Snowflake/Databricks style: the source table and the field expression + # are quoted identifiers. A metric referencing the bare physical code + # must still resolve to a queryable measure, not fall through to LOSSY. + osi = { + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "sales", + "datasets": [ + { + "name": "Orders", + "source": 'WH.PUBLIC."fact_orders"', + "fields": [ + { + "name": "Amount", + "expression": { + "dialects": [ + {"dialect": "ANSI_SQL", "expression": '"net_amount"'} + ] + }, + "data_type": "number", + } + ], + } + ], + "metrics": [ + { + "name": "Revenue", + "expression": { + "dialects": [ + { + "dialect": "ANSI_SQL", + "expression": "SUM(fact_orders.net_amount)", + } + ] + }, + } + ], + } + ], + } + obml = conv.OSItoOBML(osi).convert() + assert "Revenue" in obml.get("measures", {}), obml.get("measures") + assert obml["measures"]["Revenue"]["columns"] == [ + {"dataObject": "Orders", "column": "Amount"} + ] + + +# --------------------------------------------------------------------------- +# P2: same field name in two datasets must not collide into one dimension +# --------------------------------------------------------------------------- + + +def _dim_dataset(name: str, source: str) -> dict[str, Any]: + return { + "name": name, + "source": source, + "fields": [ + { + "name": "order_date", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "order_date"}]}, + "data_type": "date", + "dimension": {}, + } + ], + } + + +class TestDimensionNameCollision: + def test_both_dimensions_survive(self) -> None: + osi = { + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "sales", + "datasets": [ + _dim_dataset("Orders", "WH.PUBLIC.orders"), + _dim_dataset("Invoices", "WH.PUBLIC.invoices"), + ], + } + ], + } + c = conv.OSItoOBML(osi) + obml = c.convert() + dims = obml.get("dimensions", {}) + + # Two distinct dimensions, one per data object — neither dropped. + assert len(dims) == 2, dims + assert {d["dataObject"] for d in dims.values()} == {"Orders", "Invoices"} + # The collision was disambiguated, not silent. + assert any("multiple data objects" in w for w in c.warnings) + + +# --------------------------------------------------------------------------- +# P2: validate_osi robustness on malformed input +# --------------------------------------------------------------------------- + + +class TestValidateOsiRobustness: + def test_malformed_datasets_does_not_raise(self) -> None: + # datasets is a string, not a list — must return a result, not raise. + r = conv.validate_osi( + {"version": "0.1.1", "semantic_model": [{"name": "x", "datasets": "not-an-array"}]} + ) + assert r is not None + # No garbage semantic errors from iterating a string char-by-char. + assert all("DUPLICATE" not in e for e in r.semantic_errors) + + def test_malformed_fields_does_not_raise(self) -> None: + r = conv.validate_osi( + { + "version": "0.2.0.dev0", + "semantic_model": [{"name": "x", "datasets": [{"name": "D", "fields": "nope"}]}], + } + ) + assert r is not None + + def test_still_flags_duplicate_datasets(self) -> None: + # The robustness guards must not suppress real semantic errors. + r = conv.validate_osi( + { + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "m", + "datasets": [ + {"name": "D", "source": "a.b.d", "fields": []}, + {"name": "D", "source": "a.b.d2", "fields": []}, + ], + } + ], + } + ) + assert any("DUPLICATE_DATASET" in e for e in r.semantic_errors) From cdf95904db78a2857560ca6979a24f7c92be0ed7 Mon Sep 17 00:00:00 2001 From: Dragos Crintea Date: Fri, 17 Jul 2026 06:39:53 +0100 Subject: [PATCH 77/89] add semantido as a ossie vendor (#207) --- python/src/ossie/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index b0db1cc..1c8b646 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -43,6 +43,7 @@ class OSIVendor(str, Enum): DBT = "DBT" DATABRICKS = "DATABRICKS" GOODDATA = "GOODDATA" + SEMANTIDO = "SEMANTIDO" class OSIAIContextObject(BaseModel): From 710da1026df2f08ce3f5490b7aae427676e4d979 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Fri, 17 Jul 2026 11:01:08 -0500 Subject: [PATCH 78/89] feat(cli): scaffold ossie CLI (#151) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(cli): scaffold go module with cobra command tree The OSI converters currently require manual environment setup and per-converter invocation. This lays the foundation for a unified CLI (osi) that will discover and invoke converters as plugins via a stdin/stdout JSON protocol. Creates cli/ at the repo root as a self-contained Go module (github.com/open-semantic-interchange/osi/cli). All commands are stubbed — no logic is wired in this commit. Design decisions: - Go chosen for static binary distribution; no runtime dependency for end users (brew/apt installable) - internal/osidir owns ~/.osi/plugins/ initialization and respects $OSI_PLUGIN_DIR for override; uses os.UserHomeDir() rather than $HOME for Windows portability - PersistentPreRunE on the root command ensures dir init runs before every subcommand; commented caveat that Cobra does not chain this automatically if a subcommand defines its own - MarkFlagsMutuallyExclusive("from", "to") handles the both-set case on osi convert; the neither-set case is validated manually in RunE since Cobra only guards against both being provided Build pipeline (Makefile, .goreleaser.yaml) and CI follow in separate commits. * chore(cli): add makefile and goreleaser build pipeline Enables local builds and cross-platform release artifacts for the OSI CLI. Makefile provides standard targets for day-to-day development: build, test, lint, release-dry-run, and clean. All targets are designed to run from within cli/. GoReleaser config targets linux/amd64, linux/arm64, darwin/amd64, darwin/arm64, and windows/amd64. windows/arm64 is excluded — no widely available CI runner and negligible current demand. CGO_ENABLED=0 is set for fully static binaries, enabling cross-compilation from any host without a C toolchain. Version, commit, and date are injected at build time via ldflags from the vars declared in main.go. * ci(cli): add github actions workflow for build and test Runs go build, go vet, and go test on every push and pull request that touches cli/ or the workflow file itself. The paths filter prevents CLI changes from triggering unrelated workflows in this polyglot repo and vice versa. Go version is derived from go-version-file: cli/go.mod so the workflow automatically picks up any future toolchain bumps without a separate workflow edit. defaults.run.working-directory avoids repeating cd cli/ on every step. Cross-platform build testing is not included — CGO_ENABLED=0 means the linux/amd64 build is representative of all targets. GoReleaser snapshot builds are deferred to a future release workflow. * test(cli): add unit tests for internal/osidir Covers the only logic in F1 that warrants testing: $OSI_PLUGIN_DIR env var override, default path construction via os.UserHomeDir(), directory creation, and idempotent re-invocation of EnsurePluginDir. t.Setenv is used throughout so env var mutations are automatically restored after each test. t.TempDir is used for filesystem tests so no cleanup is needed and tests are safe to run in parallel. * chore(cli): rename osi to ossie throughout cli scaffold Project branding has changed from OSI to OSSIE. Updates all user-facing and internal references in the CLI: - Binary name: osi → ossie - Go module path: .../osi/cli → .../ossie/cli - Default plugin directory: ~/.osi → ~/.ossie - Environment variable: OSI_PLUGIN_DIR → OSSIE_PLUGIN_DIR - GoReleaser project name and archive ids - All command descriptions and flag help text - Output directory default: ./osi-output → ./ossie-output * chore(cli): rename internal osidir package to ossiedir Completes the OSI → OSSIE rename by updating the internal package directory, package declaration, and import reference in cmd/root.go. * fix(ossiedir): rename defaultOSIDir constant to defaultOssieDir The osi → ossie rename missed this unexported constant. The value (.ossie) was already correct; only the identifier name was stale. * chore(cli): fix .gitignore to ignore cli/ossie build artifact The osi → ossie rename missed the gitignore entry, so the local build artifact cli/ossie would no longer be ignored by git. * chore(ossiedir): rename osidir.go and osidir_test.go to ossiedir File names were inconsistent with the package directory name (ossiedir/). Pure rename — no code changes. * Swap references to Open Semantic Interchange to Apache Ossie has moved from the Open Semantic Interchange to incubation with the Apache Software Foundation. As such, the references in this PR needed to be updated accordingly. * chore(cli): align go.mod version with .tool-versions pin cli/go.mod declared `go 1.22` while cli/.tool-versions pins golang 1.26.2, giving contributors two conflicting sources of truth for which Go version this module targets. Flagged in review by khush-bhatia on PR #151. There's no existing compatibility requirement forcing a lower minimum, so align go.mod to the same version .tool-versions already pins rather than introduce a toolchain directive or lower the asdf pin. * refactor(cli): let cobra enforce --from/--to via MarkFlagsOneRequired convert.go hand-rolled a check for the "neither --from nor --to set" case alongside MarkFlagsMutuallyExclusive, which already covers the "both set" case. Suggested by khush-bhatia on PR #151 that cobra's MarkFlagsOneRequired covers this directly. Combining MarkFlagsMutuallyExclusive with MarkFlagsOneRequired on the same flag set gives "exactly one of" semantics entirely through cobra's flag annotations, so the manual check and its now-unneeded fmt.Errorf branch in runConvert are removed. * Apply suggestions from code review Co-authored-by: Khushboo Bhatia * chore(cli): stop tracking .tool-versions, gitignore for local use cli/go.mod and cli/.tool-versions each pinned the Go version independently — even after b834d75 aligned their values, nothing forced the two to move together, so they could drift apart again on the next bump. Removed cli/.tool-versions from the repo so go.mod is the only in-repo source of truth. Verified this doesn't break tooling: CI already resolves its Go version from go.mod via actions/setup-go's `go-version-file: cli/go.mod` (.github/workflows/cli-ci.yml), and asdf's golang plugin (~/.asdf/plugins/golang/bin/parse-legacy-file) reads the version straight out of go.mod's `go` directive when no .tool-versions is present. Rather than deleting the file with no path back, gitignored cli/.tool-versions so contributors whose version manager still wants its own pin file can keep one locally without risk of it getting committed and re-introducing a second tracked source of truth. Treats the toolchain-manager pin as personal environment config, not project config — the same rationale as gitignoring editor-specific config instead of committing it. Caveats: - Relies on asdf's legacy-version-file lookup, which is off by default and must be enabled per-user via `legacy_version_file = yes` in ~/.asdfrc (or ASDF_LEGACY_VERSION_FILE=yes) for `asdf install`/`asdf current` to auto-detect the version from go.mod. Not adding a repo-level workaround since asdf has no per-project way to set this. - No automated enforcement that a contributor's local pin matches go.mod; a stale local .tool-versions can still silently drift, as demonstrated on this machine when the global asdf fallback (golang 1.20.1) was too old to parse a 3-component go directive. Accepted as the tradeoff for not owning tooling-manager config in the repo. --------- Co-authored-by: Khushboo Bhatia --- .github/workflows/cli-ci.yml | 38 +++++++++++++++ .gitignore | 6 +++ cli/.goreleaser.yaml | 48 +++++++++++++++++++ cli/Makefile | 19 ++++++++ cli/cmd/convert.go | 32 +++++++++++++ cli/cmd/plugin/install.go | 22 +++++++++ cli/cmd/plugin/list.go | 18 +++++++ cli/cmd/plugin/plugin.go | 16 +++++++ cli/cmd/plugin/remove.go | 19 ++++++++ cli/cmd/root.go | 38 +++++++++++++++ cli/cmd/validate.go | 24 ++++++++++ cli/go.mod | 10 ++++ cli/go.sum | 10 ++++ cli/internal/ossiedir/ossiedir.go | 40 ++++++++++++++++ cli/internal/ossiedir/ossiedir_test.go | 65 ++++++++++++++++++++++++++ cli/main.go | 21 +++++++++ 16 files changed, 426 insertions(+) create mode 100644 .github/workflows/cli-ci.yml create mode 100644 cli/.goreleaser.yaml create mode 100644 cli/Makefile create mode 100644 cli/cmd/convert.go create mode 100644 cli/cmd/plugin/install.go create mode 100644 cli/cmd/plugin/list.go create mode 100644 cli/cmd/plugin/plugin.go create mode 100644 cli/cmd/plugin/remove.go create mode 100644 cli/cmd/root.go create mode 100644 cli/cmd/validate.go create mode 100644 cli/go.mod create mode 100644 cli/go.sum create mode 100644 cli/internal/ossiedir/ossiedir.go create mode 100644 cli/internal/ossiedir/ossiedir_test.go create mode 100644 cli/main.go diff --git a/.github/workflows/cli-ci.yml b/.github/workflows/cli-ci.yml new file mode 100644 index 0000000..e729089 --- /dev/null +++ b/.github/workflows/cli-ci.yml @@ -0,0 +1,38 @@ +name: CLI CI + +on: + push: + paths: + - 'cli/**' + - '.github/workflows/cli-ci.yml' + pull_request: + paths: + - 'cli/**' + - '.github/workflows/cli-ci.yml' + +jobs: + build-and-test: + name: Build and test + runs-on: ubuntu-latest + defaults: + run: + working-directory: cli + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: cli/go.mod + cache-dependency-path: cli/go.sum + + - name: go build + run: go build ./... + + - name: go vet + run: go vet ./... + + - name: go test + run: go test ./... diff --git a/.gitignore b/.gitignore index fdd3f16..77ca2ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,14 @@ **/__pycache__/ **/.venv/ + **/.pytest_cache/ **/.ruff_cache/ **/.coverage **/htmlcov/ **/dist/ **/target/ + +# Go CLI +cli/dist/ +cli/ossie +cli/.tool-versions diff --git a/cli/.goreleaser.yaml b/cli/.goreleaser.yaml new file mode 100644 index 0000000..3a0a17b --- /dev/null +++ b/cli/.goreleaser.yaml @@ -0,0 +1,48 @@ +version: 2 + +project_name: ossie + +before: + hooks: + - go mod tidy + +builds: + - id: ossie + main: . + binary: ossie + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + ignore: + - goos: windows + goarch: arm64 + env: + - CGO_ENABLED=0 + ldflags: + - -s -w + - -X main.version={{.Version}} + - -X main.commit={{.Commit}} + - -X main.date={{.Date}} + +archives: + - id: ossie + format: tar.gz + format_overrides: + - goos: windows + format: zip + name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + +checksum: + name_template: "checksums.txt" + +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" + - "^chore:" diff --git a/cli/Makefile b/cli/Makefile new file mode 100644 index 0000000..9ecbcb7 --- /dev/null +++ b/cli/Makefile @@ -0,0 +1,19 @@ +BINARY_NAME := ossie +BUILD_DIR := dist + +.PHONY: build test lint release-dry-run clean + +build: + go build -o $(BUILD_DIR)/$(BINARY_NAME) . + +test: + go test ./... + +lint: + go vet ./... + +release-dry-run: + goreleaser release --snapshot --clean --config .goreleaser.yaml + +clean: + rm -rf $(BUILD_DIR) diff --git a/cli/cmd/convert.go b/cli/cmd/convert.go new file mode 100644 index 0000000..dd7d360 --- /dev/null +++ b/cli/cmd/convert.go @@ -0,0 +1,32 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var convertCmd = &cobra.Command{ + Use: "convert --from --input | --to --input ", + Short: "Convert a semantic model between Ossie and a platform format", + RunE: runConvert, +} + +func init() { + convertCmd.Flags().String("from", "", "Source platform — converts platform → Ossie") + convertCmd.Flags().String("to", "", "Target platform — converts Ossie → platform") + convertCmd.Flags().StringP("input", "i", "", "Input file or directory path (required)") + convertCmd.Flags().StringP("output", "o", "", "Output directory path (default: ./ossie-output//)") + convertCmd.Flags().String("plugin", "", "Path to plugin directory (bypasses name-based discovery)") + convertCmd.Flags().Int("timeout", 60, "Plugin invocation timeout in seconds") + convertCmd.Flags().String("max-input-size", "100MB", "Maximum total input size (e.g. 500MB)") + + _ = convertCmd.MarkFlagRequired("input") + convertCmd.MarkFlagsMutuallyExclusive("from", "to") + convertCmd.MarkFlagsOneRequired("from", "to") +} + +func runConvert(cmd *cobra.Command, args []string) error { + fmt.Fprintln(cmd.OutOrStdout(), "not yet implemented") + return nil +} diff --git a/cli/cmd/plugin/install.go b/cli/cmd/plugin/install.go new file mode 100644 index 0000000..aabde2d --- /dev/null +++ b/cli/cmd/plugin/install.go @@ -0,0 +1,22 @@ +package plugin + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var installCmd = &cobra.Command{ + Use: "install [name[@version] | url]", + Short: "Install a plugin from the registry or a URL", + RunE: runPluginInstall, +} + +func init() { + installCmd.Flags().Bool("all", false, "Install the latest version of all registry plugins") +} + +func runPluginInstall(cmd *cobra.Command, args []string) error { + fmt.Fprintln(cmd.OutOrStdout(), "not yet implemented") + return nil +} diff --git a/cli/cmd/plugin/list.go b/cli/cmd/plugin/list.go new file mode 100644 index 0000000..f69e8bb --- /dev/null +++ b/cli/cmd/plugin/list.go @@ -0,0 +1,18 @@ +package plugin + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var listCmd = &cobra.Command{ + Use: "list", + Short: "List installed and available plugins", + RunE: runPluginList, +} + +func runPluginList(cmd *cobra.Command, args []string) error { + fmt.Fprintln(cmd.OutOrStdout(), "not yet implemented") + return nil +} diff --git a/cli/cmd/plugin/plugin.go b/cli/cmd/plugin/plugin.go new file mode 100644 index 0000000..c19e37d --- /dev/null +++ b/cli/cmd/plugin/plugin.go @@ -0,0 +1,16 @@ +package plugin + +import "github.com/spf13/cobra" + +// Cmd is the parent "ossie plugin" command. It is exported so cmd/root.go can +// register it. Invoking it bare prints help. +var Cmd = &cobra.Command{ + Use: "plugin", + Short: "Manage Ossie plugins", +} + +func init() { + Cmd.AddCommand(listCmd) + Cmd.AddCommand(installCmd) + Cmd.AddCommand(removeCmd) +} diff --git a/cli/cmd/plugin/remove.go b/cli/cmd/plugin/remove.go new file mode 100644 index 0000000..1b88669 --- /dev/null +++ b/cli/cmd/plugin/remove.go @@ -0,0 +1,19 @@ +package plugin + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var removeCmd = &cobra.Command{ + Use: "remove ", + Short: "Remove an installed plugin", + Args: cobra.ExactArgs(1), + RunE: runPluginRemove, +} + +func runPluginRemove(cmd *cobra.Command, args []string) error { + fmt.Fprintln(cmd.OutOrStdout(), "not yet implemented") + return nil +} diff --git a/cli/cmd/root.go b/cli/cmd/root.go new file mode 100644 index 0000000..fe213a5 --- /dev/null +++ b/cli/cmd/root.go @@ -0,0 +1,38 @@ +package cmd + +import ( + "github.com/apache/ossie/cli/cmd/plugin" + "github.com/apache/ossie/cli/internal/ossiedir" + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "ossie", + Short: "Apache Ossie (incubating) CLI", + Long: `ossie is the command-line tool for the Apache Ossie (incubating) project.`, + // NOTE: Cobra does NOT automatically chain PersistentPreRunE from parent to + // child. If any subcommand defines its own PersistentPreRunE or PreRunE, this + // function will not run for that subcommand. Future subcommands that define + // their own must call ossiedir.EnsurePluginDir() explicitly. + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return ossiedir.EnsurePluginDir() + }, +} + +// Execute runs the root command. Called by main. +func Execute() error { + return rootCmd.Execute() +} + +// SetVersion sets the version string reported by `ossie --version`. +func SetVersion(v string) { + rootCmd.Version = v +} + +func init() { + rootCmd.PersistentFlags().BoolP("verbose", "v", false, "Enable verbose output (shows plugin stderr)") + + rootCmd.AddCommand(convertCmd) + rootCmd.AddCommand(validateCmd) + rootCmd.AddCommand(plugin.Cmd) +} diff --git a/cli/cmd/validate.go b/cli/cmd/validate.go new file mode 100644 index 0000000..b11d00a --- /dev/null +++ b/cli/cmd/validate.go @@ -0,0 +1,24 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var validateCmd = &cobra.Command{ + Use: "validate [flags] [...]", + Short: "Validate one or more OSSIE YAML or JSON files", + Args: cobra.MinimumNArgs(1), + RunE: runValidate, +} + +func init() { + validateCmd.Flags().Bool("strict", false, "Promote warnings to errors") + validateCmd.Flags().String("output", "text", "Output format: text or json") +} + +func runValidate(cmd *cobra.Command, args []string) error { + fmt.Fprintln(cmd.OutOrStdout(), "not yet implemented") + return nil +} diff --git a/cli/go.mod b/cli/go.mod new file mode 100644 index 0000000..c57048e --- /dev/null +++ b/cli/go.mod @@ -0,0 +1,10 @@ +module github.com/apache/ossie/cli + +go 1.26.2 + +require github.com/spf13/cobra v1.10.2 + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect +) diff --git a/cli/go.sum b/cli/go.sum new file mode 100644 index 0000000..a6ee3e0 --- /dev/null +++ b/cli/go.sum @@ -0,0 +1,10 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/cli/internal/ossiedir/ossiedir.go b/cli/internal/ossiedir/ossiedir.go new file mode 100644 index 0000000..051a8bd --- /dev/null +++ b/cli/internal/ossiedir/ossiedir.go @@ -0,0 +1,40 @@ +package ossiedir + +import ( + "fmt" + "os" + "path/filepath" +) + +const ( + defaultOssieDir = ".ossie" + pluginsSubdir = "plugins" + envVar = "OSSIE_PLUGIN_DIR" +) + +// PluginDir returns the resolved plugin directory path. +// It respects $OSSIE_PLUGIN_DIR if set, otherwise defaults to ~/.ossie/plugins/. +func PluginDir() (string, error) { + if override := os.Getenv(envVar); override != "" { + return override, nil + } + // Use os.UserHomeDir rather than $HOME for Windows portability. + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("could not determine home directory: %w", err) + } + return filepath.Join(home, defaultOssieDir, pluginsSubdir), nil +} + +// EnsurePluginDir ensures the plugin directory exists, creating it if needed. +// It is safe to call multiple times — os.MkdirAll is idempotent. +func EnsurePluginDir() error { + dir, err := PluginDir() + if err != nil { + return err + } + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("could not create plugin directory %s: %w", dir, err) + } + return nil +} diff --git a/cli/internal/ossiedir/ossiedir_test.go b/cli/internal/ossiedir/ossiedir_test.go new file mode 100644 index 0000000..76a8981 --- /dev/null +++ b/cli/internal/ossiedir/ossiedir_test.go @@ -0,0 +1,65 @@ +package ossiedir + +import ( + "os" + "path/filepath" + "testing" +) + +func TestPluginDir_envOverride(t *testing.T) { + want := "/custom/plugin/dir" + t.Setenv(envVar, want) + + got, err := PluginDir() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +func TestPluginDir_default(t *testing.T) { + t.Setenv(envVar, "") + + home, err := os.UserHomeDir() + if err != nil { + t.Fatalf("could not determine home dir: %v", err) + } + want := filepath.Join(home, defaultOssieDir, pluginsSubdir) + + got, err := PluginDir() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +func TestEnsurePluginDir_createsDirectory(t *testing.T) { + tmp := t.TempDir() + target := filepath.Join(tmp, "plugins") + t.Setenv(envVar, target) + + if err := EnsurePluginDir(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if _, err := os.Stat(target); os.IsNotExist(err) { + t.Errorf("expected directory %q to exist, but it does not", target) + } +} + +func TestEnsurePluginDir_idempotent(t *testing.T) { + tmp := t.TempDir() + target := filepath.Join(tmp, "plugins") + t.Setenv(envVar, target) + + if err := EnsurePluginDir(); err != nil { + t.Fatalf("first call failed: %v", err) + } + if err := EnsurePluginDir(); err != nil { + t.Fatalf("second call failed: %v", err) + } +} diff --git a/cli/main.go b/cli/main.go new file mode 100644 index 0000000..305d92d --- /dev/null +++ b/cli/main.go @@ -0,0 +1,21 @@ +package main + +import ( + "os" + + "github.com/apache/ossie/cli/cmd" +) + +// version, commit, and date are set at build time by GoReleaser via ldflags. +var ( + version = "dev" + commit = "none" + date = "unknown" +) + +func main() { + cmd.SetVersion(version) + if err := cmd.Execute(); err != nil { + os.Exit(1) + } +} From 07be0176e48af67f0b46e0957a87a154586abf38 Mon Sep 17 00:00:00 2001 From: Emil Sadek Date: Fri, 17 Jul 2026 15:28:32 -0700 Subject: [PATCH 79/89] chore: add EditorConfig tab indentation for Go files, go.mod, and Makefile (#225) Co-authored-by: Emil Sadek --- .editorconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.editorconfig b/.editorconfig index 71498de..021c172 100644 --- a/.editorconfig +++ b/.editorconfig @@ -30,3 +30,6 @@ indent_size = 2 [*.{py,java,toml}] indent_style = space indent_size = 4 + +[{Makefile,go.mod,*.go}] +indent_style = tab From cbee3243e6cb40b2e6c80339b4483d2d405845c1 Mon Sep 17 00:00:00 2001 From: Yong Zheng Date: Mon, 20 Jul 2026 00:35:58 -0500 Subject: [PATCH 80/89] Add support for quoted dot-separated identifier in snowflake (#233) --- .../src/osi_to_snowflake_yaml_converter.py | 22 ++++++++++++++----- .../test_osi_to_snowflake_yaml_converter.py | 8 +++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/converters/snowflake/src/osi_to_snowflake_yaml_converter.py b/converters/snowflake/src/osi_to_snowflake_yaml_converter.py index 17ef387..196a1d3 100644 --- a/converters/snowflake/src/osi_to_snowflake_yaml_converter.py +++ b/converters/snowflake/src/osi_to_snowflake_yaml_converter.py @@ -357,6 +357,22 @@ def _normalize_identifier(identifier): return stripped return stripped.upper() +def _split_identifiers(source_str): + """Split a dot-separated identifier string while respecting double quotes.""" + parts = [] + current = [] + in_quotes = False + for char in source_str: + if char == '"': + in_quotes = not in_quotes + current.append(char) + elif char == "." and not in_quotes: + parts.append("".join(current).strip()) + current = [] + else: + current.append(char) + parts.append("".join(current).strip()) + return parts def _parse_source(source): """Parses an Ossie dataset source string into a Snowflake base_table dict. @@ -378,11 +394,7 @@ def _parse_source(source): "WITH ", "WITH\n", "WITH\t")): return {"definition": source_stripped} - # Strict 3-part rule: source must be db.schema.table. This may be relaxed - # in the future to allow 1- or 2-part names. - # TODO: Quoted identifiers (e.g., "my.db"."my schema"."my table") are not - # handled. Basic dot-splitting only. - parts = source_stripped.split(".") + parts = _split_identifiers(source_stripped) if len(parts) == 3: # Only uppercase unquoted identifiers; preserve quoted ones as-is. return { diff --git a/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py b/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py index 0c732e8..075aeeb 100644 --- a/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py +++ b/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py @@ -114,6 +114,14 @@ def test_quoted_identifiers_preserved(self): "table": '"myTable"', } + def test_quoted_identifiers_with_dots_preserved(self): + result = _parse_source('"my.db"."my schema"."my table"') + assert result == { + "database": '"my.db"', + "schema": '"my schema"', + "table": '"my table"', + } + def test_subquery_select(self): result = _parse_source("SELECT * FROM foo") assert result == {"definition": "SELECT * FROM foo"} From 4207719b4e132a115db4b1338772366dfc3d668d Mon Sep 17 00:00:00 2001 From: Yong Zheng Date: Mon, 20 Jul 2026 00:43:02 -0500 Subject: [PATCH 81/89] Fix orionbelt test (#232) --- converters/orionbelt/tests/test_osi_metric_no_silent_loss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/converters/orionbelt/tests/test_osi_metric_no_silent_loss.py b/converters/orionbelt/tests/test_osi_metric_no_silent_loss.py index 3a9d1d3..cb873f1 100644 --- a/converters/orionbelt/tests/test_osi_metric_no_silent_loss.py +++ b/converters/orionbelt/tests/test_osi_metric_no_silent_loss.py @@ -332,7 +332,7 @@ def test_output_passes_osi_validation_after_collision(self) -> None: } osi_again = conv.OBMLtoOSI(obml, "sales").convert() result = conv.validate_osi(osi_again) - assert result.valid, result.errors + assert result.valid, result.summary_lines() class TestIdempotency: From 3db13b4ea02deb293082a1c042b9cd8a12f51965 Mon Sep 17 00:00:00 2001 From: Yong Zheng Date: Mon, 20 Jul 2026 01:46:49 -0500 Subject: [PATCH 82/89] Check in snapshot files for consistent testing (#231) * Checkin snapshot files for consistent testing * Add header to ambr files * Keep syrupy snapshots command --- converters/dbt/README.md | 3 +- .../tests/__snapshots__/test_msi_to_osi.ambr | 200 ++++++++++++++++++ .../tests/__snapshots__/test_osi_to_msi.ambr | 65 ++++++ 3 files changed, 266 insertions(+), 2 deletions(-) create mode 100644 converters/dbt/tests/__snapshots__/test_msi_to_osi.ambr create mode 100644 converters/dbt/tests/__snapshots__/test_osi_to_msi.ambr diff --git a/converters/dbt/README.md b/converters/dbt/README.md index e78663e..c39090f 100644 --- a/converters/dbt/README.md +++ b/converters/dbt/README.md @@ -134,8 +134,7 @@ uv sync uv run pytest ``` -Generate initial syrupy snapshots on first run: - +Generate new Syrupy snapshots: ```bash uv run pytest --snapshot-update ``` diff --git a/converters/dbt/tests/__snapshots__/test_msi_to_osi.ambr b/converters/dbt/tests/__snapshots__/test_msi_to_osi.ambr new file mode 100644 index 0000000..facd135 --- /dev/null +++ b/converters/dbt/tests/__snapshots__/test_msi_to_osi.ambr @@ -0,0 +1,200 @@ +# serializer version: 1 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# name: TestMetricConversion.test_derived_metric_nested + ''' + version: 0.2.0.dev0 + dialects: + - ANSI_SQL + semantic_model: + - name: semantic_model + datasets: + - name: orders + source: schema.table + fields: + - name: revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + - name: cost + expression: + dialects: + - dialect: ANSI_SQL + expression: cost_amount + - name: expenses + expression: + dialects: + - dialect: ANSI_SQL + expression: expense_amount + metrics: + - name: revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + - name: cost + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.cost_amount) + - name: expenses + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.expense_amount) + - name: gross_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) - SUM(orders.cost_amount) + - name: net_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: (SUM(orders.amount) - SUM(orders.cost_amount)) - SUM(orders.expense_amount) + + ''' +# --- +# name: TestMetricConversion.test_ratio_metric_inlines_sub_expressions + ''' + version: 0.2.0.dev0 + dialects: + - ANSI_SQL + semantic_model: + - name: semantic_model + datasets: + - name: orders + source: schema.table + fields: + - name: revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + - name: order_count + expression: + dialects: + - dialect: ANSI_SQL + expression: CASE WHEN order_id IS NOT NULL THEN 1 ELSE 0 END + metrics: + - name: revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + - name: order_count + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(CASE WHEN orders.order_id IS NOT NULL THEN 1 ELSE 0 END) + - name: arpu + expression: + dialects: + - dialect: ANSI_SQL + expression: (SUM(orders.amount)) / (SUM(CASE WHEN orders.order_id IS NOT NULL + THEN 1 ELSE 0 END)) + + ''' +# --- +# name: TestMetricFilterFlattening.test_metric_and_measure_filters_combined_with_and + ''' + version: 0.2.0.dev0 + dialects: + - ANSI_SQL + semantic_model: + - name: semantic_model + datasets: + - name: orders + source: schema.table + fields: + - name: revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + metrics: + - name: paid_intl_revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(CASE WHEN (status = 'paid') AND (region = 'intl') THEN orders.amount + END) + + ''' +# --- +# name: TestRelationshipConversion.test_three_datasets_produce_all_pairs + ''' + version: 0.2.0.dev0 + dialects: + - ANSI_SQL + semantic_model: + - name: semantic_model + datasets: + - name: users_a + source: schema.table + primary_key: + - user_id + fields: + - name: user + expression: + dialects: + - dialect: ANSI_SQL + expression: user_id + - name: users_b + source: schema.table + unique_keys: + - - user_id + fields: + - name: user + expression: + dialects: + - dialect: ANSI_SQL + expression: user_id + - name: orders + source: schema.table + fields: + - name: user + expression: + dialects: + - dialect: ANSI_SQL + expression: user_id + relationships: + - name: users_a__users_b__user + from: users_a + to: users_b + from_columns: + - user_id + to_columns: + - user_id + - name: orders__users_a__user + from: orders + to: users_a + from_columns: + - user_id + to_columns: + - user_id + - name: orders__users_b__user + from: orders + to: users_b + from_columns: + - user_id + to_columns: + - user_id + + ''' +# --- diff --git a/converters/dbt/tests/__snapshots__/test_osi_to_msi.ambr b/converters/dbt/tests/__snapshots__/test_osi_to_msi.ambr new file mode 100644 index 0000000..e886e0b --- /dev/null +++ b/converters/dbt/tests/__snapshots__/test_osi_to_msi.ambr @@ -0,0 +1,65 @@ +# serializer version: 1 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# name: TestOSIToMSIRoundTrip.test_osi_to_msi_to_osi_preserves_structure + ''' + version: 0.2.0.dev0 + dialects: + - ANSI_SQL + semantic_model: + - name: semantic_model + datasets: + - name: orders + source: analytics.orders + primary_key: + - order_id + fields: + - name: order_id + expression: + dialects: + - dialect: ANSI_SQL + expression: order_id + - name: status + expression: + dialects: + - dialect: ANSI_SQL + expression: status + dimension: + is_time: false + - name: created_at + expression: + dialects: + - dialect: ANSI_SQL + expression: created_at + dimension: + is_time: true + - name: amount + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + dimension: + is_time: false + metrics: + - name: revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + + ''' +# --- From be4bbf78d72d1741c43982168b9ab5c0b20b2236 Mon Sep 17 00:00:00 2001 From: Haoran Li Date: Mon, 20 Jul 2026 08:45:56 -0700 Subject: [PATCH 83/89] Add Databricks Unity Catalog Metric View converter (converters/databricks) (#224) Bidirectional, offline converter between Apache Ossie semantic models and Databricks Unity Catalog Metric Views (YAML v1.1), filling the DATABRICKS spoke already listed in converters/README.md. Packaged like the sibling spokes: pyproject.toml (apache-ossie-databricks), an ossie_databricks package under src/, ASF license headers, and a tests/ suite (example-based + Hypothesis property-based round-trip; 74 tests). PyYAML is the only runtime dependency. Co-authored-by: jackstein21 <82542300+jackstein21@users.noreply.github.com> --- converters/databricks/README.md | 124 ++++ converters/databricks/pyproject.toml | 52 ++ .../src/ossie_databricks/__init__.py | 32 + .../src/ossie_databricks/_common.py | 288 ++++++++ .../databricks/src/ossie_databricks/cli.py | 78 ++ .../ossie_databricks/metric_view_to_ossie.py | 366 ++++++++++ .../ossie_databricks/ossie_to_metric_view.py | 660 +++++++++++++++++ .../databricks/tests/_roundtrip_helpers.py | 355 ++++++++++ converters/databricks/tests/_util.py | 76 ++ converters/databricks/tests/conftest.py | 23 + .../tests/fixtures/fixtureA_metric_view.yaml | 53 ++ .../tests/fixtures/fixtureA_ossie.yaml | 79 +++ .../tests/fixtures/fixtureB_metric_view.yaml | 51 ++ .../tests/fixtures/fixtureB_ossie.yaml | 72 ++ .../tests/fixtures/tpcds_metric_view.yaml | 67 ++ .../tests/fixtures/tpcds_ossie.yaml | 89 +++ .../tests/test_metric_view_to_ossie.py | 347 +++++++++ .../tests/test_ossie_to_metric_view.py | 670 ++++++++++++++++++ converters/databricks/tests/test_roundtrip.py | 95 +++ .../tests/test_roundtrip_properties.py | 107 +++ 20 files changed, 3684 insertions(+) create mode 100644 converters/databricks/README.md create mode 100644 converters/databricks/pyproject.toml create mode 100644 converters/databricks/src/ossie_databricks/__init__.py create mode 100644 converters/databricks/src/ossie_databricks/_common.py create mode 100644 converters/databricks/src/ossie_databricks/cli.py create mode 100644 converters/databricks/src/ossie_databricks/metric_view_to_ossie.py create mode 100644 converters/databricks/src/ossie_databricks/ossie_to_metric_view.py create mode 100644 converters/databricks/tests/_roundtrip_helpers.py create mode 100644 converters/databricks/tests/_util.py create mode 100644 converters/databricks/tests/conftest.py create mode 100644 converters/databricks/tests/fixtures/fixtureA_metric_view.yaml create mode 100644 converters/databricks/tests/fixtures/fixtureA_ossie.yaml create mode 100644 converters/databricks/tests/fixtures/fixtureB_metric_view.yaml create mode 100644 converters/databricks/tests/fixtures/fixtureB_ossie.yaml create mode 100644 converters/databricks/tests/fixtures/tpcds_metric_view.yaml create mode 100644 converters/databricks/tests/fixtures/tpcds_ossie.yaml create mode 100644 converters/databricks/tests/test_metric_view_to_ossie.py create mode 100644 converters/databricks/tests/test_ossie_to_metric_view.py create mode 100644 converters/databricks/tests/test_roundtrip.py create mode 100644 converters/databricks/tests/test_roundtrip_properties.py diff --git a/converters/databricks/README.md b/converters/databricks/README.md new file mode 100644 index 0000000..e2fe6cc --- /dev/null +++ b/converters/databricks/README.md @@ -0,0 +1,124 @@ + + +# Apache Ossie Databricks Converter + +Bidirectional, offline conversion between an [Apache Ossie](https://github.com/apache/ossie) +semantic model and a Databricks +[Unity Catalog Metric View](https://docs.databricks.com/aws/en/metric-views/) (YAML +`1.1`). No Databricks connection required. + +- **Export** (`ossie-databricks export`): Apache Ossie -> Metric View (one fact + `source` with a nested `joins` tree and a flat `dimensions` list). +- **Import** (`ossie-databricks import`): Metric View -> Apache Ossie. Metric View features Apache Ossie has + no native field for are preserved in `custom_extensions[DATABRICKS]`, so + `MV -> Apache Ossie -> MV` is lossless. + +On **export** (Apache Ossie -> Metric View), Apache Ossie features with no Metric View slot -- relationship +`ai_context`, `dimension.is_time`, non-`DATABRICKS`/`ANSI_SQL` dialects, foreign-vendor +`custom_extensions` -- are **dropped with a warning**. On **import** (Metric View -> Apache Ossie), +Metric View only features (filter, window, format, rely, ...) are instead **preserved** in +`custom_extensions[DATABRICKS]`, so `MV -> Apache Ossie -> MV` is lossless. Any input that breaks a +[requirement](#requirements) **raises a `ConversionError`** -- the converter never +silently drops a field or produces an invalid result. + +## Installation + +```bash +pip install apache-ossie-databricks # once published to PyPI +# or, from a checkout of this directory: +pip install -e . +``` + +The only runtime dependency is `PyYAML`. Python 3.11+. + +## Usage + +### Command line + +```bash +ossie-databricks export -i model.yaml -o view.yaml [--source orders] # Apache Ossie -> Metric View +ossie-databricks import -i view.yaml -o model.yaml [--name my_model] # Metric View -> Apache Ossie +``` + +With no `-o`, output goes to stdout. `--source` (export) picks the fact/grain (default: +the FK-sink dataset; naming a coarser-grain dataset produces `one_to_many` joins); +`--name` (import) sets the Apache Ossie model name (default: the source's last identifier). + +### Python API + +```python +from ossie_databricks import convert_ossie_to_metric_view, convert_metric_view_to_ossie + +metric_view_yaml = convert_ossie_to_metric_view(ossie_yaml_str) # optionally choose the fact/grain, e.g. (ossie_yaml_str, source="orders") +ossie_yaml = convert_metric_view_to_ossie(metric_view_yaml_str, model_name="sales") +``` + +## Mapping + +Each row maps in both directions; the **Notes** flag where a behavior is specific to +**export** (Apache Ossie -> Metric View) or **import** (Metric View -> Apache Ossie). + +| Apache Ossie | Metric View (v1.1) | Notes | +|---|---|---| +| `semantic_model.description` | `comment` | Model-level description only. | +| root dataset | `source` | The fact/grain. | +| other `datasets` | nested `joins[]` | Export: the relationship graph is reassembled into the join tree; a dataset reached by two paths (a diamond) fans out into one aliased join per path. | +| `relationship` `from_columns`/`to_columns` | join `on` (differing names) / `using` (shared names) | Decomposed into columns on import; rebuilt into `on`/`using` on export. | +| `relationship.from`/`to` direction | join `cardinality` | Export: source on the many (`from`) side -> `many_to_one`; on the one (`to`) side -> `one_to_many`. | +| `dataset.primary_key` / `unique_keys` | join `rely.at_most_one_match` | Both directions: export sets `at_most_one_match` when a key covers the join columns; import recovers a `unique_keys` from it. | +| `dataset.fields[]` | `dimensions[]` | Export: fields flatten into one list and a joined column is qualified by its full join path (`customer.c_name`; `customer.region.r_name` when nested). | +| `field.expression.dialects[]` | `expr` | Export: prefer the `DATABRICKS` dialect, else `ANSI_SQL`. | +| `metrics[]` | `measures[]` | Export: fact columns are referenced bare (`SUM(amount)`). | +| `field.label` | `display_name` | | +| `field` / `metric` `description` | `comment` | | +| `ai_context.synonyms` | `synonyms` | | +| `custom_extensions[DATABRICKS]` | `filter`, `window`, `format`, `rely`, `materialization` | Import stashes Metric View only features here; export restores them -- keeping `MV -> Apache Ossie -> MV` lossless. | + +## Requirements + +Conversion raises a `ConversionError` (rather than guessing or emitting something +invalid) when an input breaks one of these: + +- the Metric View `version` is not `1.1`; +- a `source` is not a 3-part `catalog.schema.table` name or a `SELECT`/`WITH` subquery; +- the relationship graph is not acyclic and resolvable to a single fact -- a cycle, or + multiple candidate facts without `--source`, is rejected (a diamond is allowed and + fanned out); +- a join has no condition (a cross join has no Apache Ossie relationship form); +- a join condition is non-equi or otherwise can't be decomposed into equi-join columns + (Apache Ossie relationships are equi-joins, so the join has no Apache Ossie representation); +- the input YAML is malformed. + +## Development + +```bash +pip install -e ".[dev]" +python3 -m pytest tests/ +``` + +Example-based unit tests plus Hypothesis property-based round-trip tests +(`test_roundtrip_properties.py`, which skip if `hypothesis` is not installed). + +## Future effort + +Both the Apache Ossie specification and the Databricks Unity Catalog Metric View YAML are still +evolving. As either side adds or changes fields, this converter will be updated to track +them -- extending the mapping and coverage in both directions to keep the conversion +current and to support as much as each format allows over time. diff --git a/converters/databricks/pyproject.toml b/converters/databricks/pyproject.toml new file mode 100644 index 0000000..d4ed351 --- /dev/null +++ b/converters/databricks/pyproject.toml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "apache-ossie-databricks" +version = "0.2.0.dev0" +description = "Databricks Unity Catalog Metric View <> Apache Ossie converter" +requires-python = ">=3.11" +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", +] +dependencies = [ + "PyYAML>=6.0", +] + +[project.license] +text = "Apache-2.0" + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "hypothesis>=6.0", +] + +[project.scripts] +ossie-databricks = "ossie_databricks.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_databricks"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/converters/databricks/src/ossie_databricks/__init__.py b/converters/databricks/src/ossie_databricks/__init__.py new file mode 100644 index 0000000..379a2ae --- /dev/null +++ b/converters/databricks/src/ossie_databricks/__init__.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Bidirectional converter between Apache Ossie semantic models and Databricks Unity Catalog +Metric Views (YAML v1.1). Pure offline string-in / string-out transforms. + + from ossie_databricks import convert_ossie_to_metric_view, convert_metric_view_to_ossie +""" + +from ._common import ConversionError +from .metric_view_to_ossie import convert_metric_view_to_ossie +from .ossie_to_metric_view import convert_ossie_to_metric_view + +__all__ = [ + "ConversionError", + "convert_metric_view_to_ossie", + "convert_ossie_to_metric_view", +] diff --git a/converters/databricks/src/ossie_databricks/_common.py b/converters/databricks/src/ossie_databricks/_common.py new file mode 100644 index 0000000..4f17a5b --- /dev/null +++ b/converters/databricks/src/ossie_databricks/_common.py @@ -0,0 +1,288 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared helpers for the Apache Ossie <-> Databricks Metric View converters. + +Both directions are pure offline YAML transforms. The only cross-cutting concerns +live here: version constants, the dialect preference order, the `custom_extensions` +stash protocol, and small SQL-string helpers. +""" + +import json +import re + +import yaml + +# Apache Ossie semantic model spec version this converter targets (see core-spec). +# +# NOTE: this is an exact-match check (see convert_ossie_to_metric_view). Unlike the +# dbt converter, this spoke intentionally has no `apache-ossie` package dependency, so +# nothing updates this automatically -- it MUST be bumped in lockstep with the +# `version` in `core-spec/` whenever the spec version moves, or the converter will +# reject otherwise-valid Apache Ossie files. +OSSIE_VERSION = "0.2.0.dev0" + +# Databricks Unity Catalog Metric View YAML version. Only 1.1 supports joins, +# per-column comments, synonyms, and the format/window/parameters surface. +MV_VERSION = "1.1" + +# Vendor id used for the `custom_extensions` stash and for dialect selection. +VENDOR = "DATABRICKS" + +# Expression dialects this converter understands, in preference order. +DIALECT_DATABRICKS = "DATABRICKS" +DIALECT_ANSI = "ANSI_SQL" + +# Metric Views cap the number of synonyms per column. +SYNONYM_LIMIT = 10 + +# Bump when the shape of a stashed `data` blob changes. +STASH_VERSION = 1 + +# Metric View join cardinalities (the only two values v1.1 defines). Apache Ossie has no +# cardinality field; the value is implied by relationship direction -- `from` is the +# many side, `to` is the one side -- so the converter derives it from / writes it +# into the from/to orientation rather than relying on a dedicated field. +CARD_MANY_TO_ONE = "many_to_one" +CARD_ONE_TO_MANY = "one_to_many" + +# Model-level stash key recording which dataset was the Metric View `source` (its +# grain). Needed only when a one_to_many join puts the source on a relationship's +# `to` side, where the natural FK-sink heuristic would otherwise pick the wrong +# fact on re-export. Absent for plain many-to-one stars, so they stay clean. +STASH_SOURCE_KEY = "source_dataset" + +# A bare SQL identifier (single column reference), e.g. `c_name`. Used to decide +# whether an expression can be safely alias-prefixed on export / de-prefixed on +# import. +_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +class ConversionError(Exception): + """Raised when an input cannot be converted.""" + + +def require(obj, key, what): + """Return `obj[key]`, or raise a clean ConversionError if it's missing/empty -- so + malformed input surfaces as an error message rather than a raw KeyError traceback. + + Presence is tested by key (not truthiness), so a legitimately falsy value such as + `0` or `False` is returned; a missing key, a null, or an empty/whitespace string is + rejected. + """ + if not isinstance(obj, dict) or key not in obj or obj[key] is None: + raise ConversionError(f"{what} is missing required '{key}'") + value = obj[key] + if isinstance(value, str) and not value.strip(): + raise ConversionError(f"{what} has an empty '{key}'") + return value + + +def require_str(obj, key, what): + """Like require(), but also enforce the value is a string -- so a non-string scalar + (e.g. a YAML number for a name or expression) raises a clean ConversionError instead + of crashing later in a string operation.""" + value = require(obj, key, what) + if not isinstance(value, str): + raise ConversionError( + f"{what}: '{key}' must be a string, got {type(value).__name__}") + return value + + +# YAML 1.1 (PyYAML's default) treats bare on/off/yes/no/y/n as booleans, so a metric +# view join's `on:` key would parse as the boolean True and silently lose the join +# condition. Databricks (Jackson) uses YAML 1.2 booleans (only true/false). The Loader +# below uses 1.2 semantics, so it reads DBR's bare `on:` (and any "on"/"off" value) as a +# string. The Dumper additionally force-quotes those tokens on output (see below), so the +# YAML it emits round-trips the same way through a YAML 1.1 reader too (e.g. stock +# yaml.safe_load) rather than turning an "on"/"off" synonym/label into a boolean. +class _Yaml12Loader(yaml.SafeLoader): + """SafeLoader with YAML 1.2 boolean semantics.""" + + +class _Yaml12Dumper(yaml.SafeDumper): + """SafeDumper with YAML 1.2 boolean semantics.""" + + +_YAML12_BOOL = re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$") +for _cls in (_Yaml12Loader, _Yaml12Dumper): + # Drop the YAML 1.1 bool resolver (yes/no/on/off/y/n) and re-add a 1.2 one. + _cls.yaml_implicit_resolvers = { + ch: [(tag, rx) for (tag, rx) in resolvers if tag != "tag:yaml.org,2002:bool"] + for ch, resolvers in _cls.yaml_implicit_resolvers.items() + } + _cls.add_implicit_resolver("tag:yaml.org,2002:bool", _YAML12_BOOL, list("tTfF")) + + +# Force-quote string scalars that a YAML 1.1 reader would otherwise interpret as booleans +# (yes/no/on/off/y/n/true/false, any case). Number- and null-like strings are already +# quoted by PyYAML's surviving resolvers; only these bool tokens need it. Without this, a +# synonym/label/comment like "on" emits bare and a 1.1 consumer reads it back as `True`. +_YAML11_BOOL_STRS = frozenset( + variant + for word in ("y", "n", "yes", "no", "on", "off", "true", "false") + for variant in (word, word.capitalize(), word.upper()) +) + + +def _represent_str(dumper, data): + style = "'" if data in _YAML11_BOOL_STRS else None + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style) + + +_Yaml12Dumper.add_representer(str, _represent_str) + + +def load_yaml(text): + """Parse YAML with 1.2 boolean semantics, so a join `on:` key stays the string + `on` rather than becoming the boolean True. A syntax error is surfaced as a + ConversionError so callers (and the CLI) get a clean message, not a raw traceback.""" + try: + return yaml.load(text, Loader=_Yaml12Loader) + except yaml.YAMLError as e: + raise ConversionError(f"Invalid YAML: {e}") from e + + +def dump_yaml(obj): + """Serialize to YAML with 1.2 boolean semantics. The bool-like token `on` -- whether + a join condition key or an "on"/"off"/"yes"/... string value -- is force-quoted as + `'on'` by the str representer (see `_represent_str`), so a YAML 1.1 reader of this + output reads it as the string, not the boolean True. Databricks' Jackson (1.2) parser + reads the quoted key/value correctly too.""" + return yaml.dump(obj, Dumper=_Yaml12Dumper, sort_keys=False, default_flow_style=False) + + +def is_simple_identifier(expr): + """True if `expr` is a single bare column reference (no operators/functions). + + A non-string input is simply not an identifier (returns False) rather than raising.""" + return isinstance(expr, str) and bool(_IDENTIFIER_RE.match(expr.strip())) + + +def read_stash(obj): + """Return the DATABRICKS stash dict on an Apache Ossie object, or {} if absent. + + The `_v` version marker is stripped from the returned dict. + """ + for ext in (obj or {}).get("custom_extensions") or []: + if ext.get("vendor_name") == VENDOR: + try: + data = json.loads(ext.get("data") or "{}") + except json.JSONDecodeError as e: + raise ConversionError( + f"DATABRICKS custom_extensions data is not valid JSON: {e}") from e + data.pop("_v", None) + return data + return {} + + +def write_stash(obj, data): + """Attach a DATABRICKS `custom_extensions` entry holding `data` (a dict). + + No-op when `data` is empty, so hand-authored Apache Ossie stays clean. Merges into an + existing DATABRICKS entry if one is already present. + """ + if not data: + return + payload = {"_v": STASH_VERSION} + payload.update(data) + blob = json.dumps(payload) + exts = obj.setdefault("custom_extensions", []) + for ext in exts: + if ext.get("vendor_name") == VENDOR: + ext["data"] = blob + return + exts.append({"vendor_name": VENDOR, "data": blob}) + + +def foreign_vendor_extensions(obj): + """Return non-DATABRICKS custom_extensions (dropped on export, with a warning).""" + return [ + ext + for ext in (obj or {}).get("custom_extensions") or [] + if ext.get("vendor_name") != VENDOR + ] + + +def pick_expression(ossie_expression): + """Choose the SQL string for an Apache Ossie expression: DATABRICKS, else ANSI_SQL. + + Returns None if neither dialect is present (the caller warns and skips). Does + not warn about other dialects here -- only the absence of a usable one matters. + """ + dialects = { + d.get("dialect"): d.get("expression") + for d in (ossie_expression or {}).get("dialects") or [] + } + expr = dialects.get(DIALECT_DATABRICKS) or dialects.get(DIALECT_ANSI) + if expr is not None and not isinstance(expr, str): + raise ConversionError( + f"expression must be a string, got {type(expr).__name__}") + return expr + + +def synonyms_of(ai_context): + """Extract the synonyms list from an Apache Ossie ai_context (object form only).""" + if isinstance(ai_context, dict): + return list(ai_context.get("synonyms") or []) + return [] + + +def merge_description(description, ai_context): + """Fold a string-form ai_context into a description. + + The Apache Ossie schema allows ai_context to be either a string or an object. A string + has no Metric View home of its own, so it is appended to the description + (which maps to `comment`). Object-form ai_context is handled separately + (synonyms map natively; instructions/examples are dropped). + """ + if isinstance(ai_context, str) and ai_context.strip(): + return f"{description}\n{ai_context}" if description else ai_context + return description + + +def validate_source(source, dataset_name): + """Validate and normalize a dataset source for a Metric View. + + Accepts a 3-part `catalog.schema.table` identifier or a `SELECT`/`WITH` + subquery. Raises ConversionError otherwise. + """ + if not source or not str(source).strip(): + raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") + s = str(source).strip() + # A SELECT/WITH subquery source. `\b` after the keyword matches `WITH(...)` (no + # space) too, but not an identifier like `WITHHELD`. + if re.match(r"(?i)(select|with)\b", s): + return s + # Exactly 3 parts, each a non-empty token with no whitespace -- so `.sch.tbl`, + # `cat..tbl`, `cat.sch.`, and `cat . sch . tbl` are all rejected (an empty or + # space-laden part is not a valid catalog/schema/table identifier). + parts = s.split(".") + if len(parts) == 3 and all(p and not any(ch.isspace() for ch in p) for p in parts): + return s + raise ConversionError( + f"Dataset '{dataset_name}': source '{source}' must be a 3-part " + f"catalog.schema.table identifier or a SELECT/WITH subquery" + ) + + +def last_identifier(source): + """Last dotted part of a table reference, e.g. `samples.tpch.lineitem` -> `lineitem`. + + Coerces to str so a malformed (non-string) source doesn't crash here -- it gets a + clean error from validate_source instead.""" + return str(source).strip().split(".")[-1].strip("`") if source else source diff --git a/converters/databricks/src/ossie_databricks/cli.py b/converters/databricks/src/ossie_databricks/cli.py new file mode 100644 index 0000000..a3751ba --- /dev/null +++ b/converters/databricks/src/ossie_databricks/cli.py @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command-line interface for the Apache Ossie <-> Databricks Metric View converter. + + ossie-databricks export -i model.yaml [-o view.yaml] [--source orders] + ossie-databricks import -i view.yaml [-o model.yaml] [--name my_model] + +`export` converts an Apache Ossie semantic model to a Databricks Metric View; `import` does the +reverse. With no `-o`, the result is written to stdout. Conversions that drop +information emit warnings to stderr. +""" + +import argparse +import sys + +from ._common import ConversionError +from .metric_view_to_ossie import convert_metric_view_to_ossie +from .ossie_to_metric_view import convert_ossie_to_metric_view + + +def _build_parser(): + parser = argparse.ArgumentParser(prog="ossie-databricks", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command") + sub.required = True # set as attribute (the add_subparsers kwarg is 3.7+) + + exp = sub.add_parser("export", help="Apache Ossie semantic model -> Databricks Metric View YAML") + exp.add_argument("-i", "--input", required=True, help="Apache Ossie YAML file") + exp.add_argument("-o", "--output", help="output Metric View YAML (default: stdout)") + exp.add_argument("-s", "--source", + help="dataset to use as the fact/grain (default: the FK-sink dataset); " + "naming a coarser-grain dataset unlocks one_to_many joins") + + imp = sub.add_parser("import", help="Databricks Metric View YAML -> Apache Ossie semantic model") + imp.add_argument("-i", "--input", required=True, help="Metric View YAML file") + imp.add_argument("-o", "--output", help="output Apache Ossie YAML (default: stdout)") + imp.add_argument("--name", help="Apache Ossie model name (default: derived from the source)") + return parser + + +def main(argv=None): + args = _build_parser().parse_args(argv) + try: + with open(args.input) as fh: + text = fh.read() + if args.command == "export": + out = convert_ossie_to_metric_view(text, source=args.source) + else: + out = convert_metric_view_to_ossie(text, model_name=args.name) + except (ConversionError, OSError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + if args.output: + with open(args.output, "w") as fh: + fh.write(out) + else: + sys.stdout.write(out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/databricks/src/ossie_databricks/metric_view_to_ossie.py b/converters/databricks/src/ossie_databricks/metric_view_to_ossie.py new file mode 100644 index 0000000..75cdb75 --- /dev/null +++ b/converters/databricks/src/ossie_databricks/metric_view_to_ossie.py @@ -0,0 +1,366 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert a Databricks Unity Catalog Metric View (v1.1) to an Apache Ossie semantic model. + +Pure offline conversion. Accepts a Metric View (one `source` with a nested `joins` +tree). Metric View features Apache Ossie has no native field for -- filter, window, format, +rely, cardinality, parameters, materialization -- are preserved in +`custom_extensions[DATABRICKS]` so that converting back reproduces the original view. +A join condition an Apache Ossie relationship cannot represent (a non-equi or cross join) is +rejected, not stashed. See README.md. + +Usage (CLI): + ossie-databricks import -i view.yaml [-o model.yaml] [--name NAME] +""" + +import re +import warnings + +from ._common import ( + CARD_MANY_TO_ONE, + CARD_ONE_TO_MANY, + ConversionError, + DIALECT_DATABRICKS, + MV_VERSION, + OSSIE_VERSION, + STASH_SOURCE_KEY, + dump_yaml, + is_simple_identifier, + last_identifier, + load_yaml, + require, + require_str, + validate_source, + write_stash, +) + +# Metric View fields with no native Apache Ossie home -> stashed verbatim. +_MODEL_STASH_KEYS = ("filter", "parameters", "materialization") +_JOIN_STASH_KEYS = ("rely", "cardinality") +_COLUMN_STASH_KEYS = ("format", "window") + + +def _warn(scope, msg): + warnings.warn(f"[{scope}] {msg}") + + +# Operators that mean a join condition is NOT a simple equi-join (so it cannot be +# expressed as from_columns/to_columns, and the join is rejected on import). +_NON_EQUI_RE = re.compile(r"[<>!]=|<>|[<>]") + + +def _is_wildcard(col): + """A wildcard column (`expr: source.*`) is projected without a `name`; Apache Ossie has + no representation for it. Detected by the absence of a `name` key (a named column, + even one whose name is falsy like `0` or whose expression contains `*` as + multiplication, is not a wildcard).""" + return "name" not in col + + +def convert_metric_view_to_ossie(mv_yaml_str, model_name=None): + """Parse Metric View v1.1 YAML and return Apache Ossie semantic model YAML (string).""" + # load_yaml uses YAML 1.2 booleans, so a join `on:` key stays the string "on" + # (PyYAML's default 1.1 would parse it as the boolean True and drop the condition). + view = load_yaml(mv_yaml_str) + if not isinstance(view, dict): + raise ConversionError("Invalid Metric View YAML: expected a mapping at the root") + + version = str(view.get("version", "")) + if version != MV_VERSION: + raise ConversionError( + f"Unsupported Metric View version '{version}'. This converter targets " + f"v{MV_VERSION} only." + ) + + model = _convert_view(view, model_name) + return dump_yaml({"version": OSSIE_VERSION, "semantic_model": [model]}) + + +def _convert_view(view, model_name): + source = view.get("source") + if not source: + raise ConversionError("Metric View is missing required 'source'") + + # Derive the model/fact name from a table source's last identifier. A SELECT/WITH + # subquery source has no meaningful table name, so use a stable default instead of + # slicing a token out of the SQL text (override with --name). + is_sql = str(source).strip().split(None, 1)[0].upper() in ("SELECT", "WITH") + last_id = last_identifier(source) + fact_name = model_name or ( + last_id if (not is_sql and last_id and is_simple_identifier(last_id)) + else "metric_view" + ) + # Validate the source shape up front (3-part table or SELECT/WITH), mirroring the + # exporter -- so a malformed source fails here with a clean error instead of passing + # silently through to Apache Ossie and only erroring on a later re-export. + validate_source(source, fact_name) + + datasets = [{"name": fact_name, "source": source}] + relationships = [] + alias_to_dataset = {"source": fact_name, fact_name: fact_name} + # Names are compared case-insensitively (DBR identifiers are case-insensitive), so a + # `Fact`/`fact` or `dim`/`Dim` collision is caught here instead of producing two + # datasets DBR would reject on re-export. + seen_names = {fact_name.strip().lower()} + + # Walk the join tree, emitting one dataset + one relationship per join. + def walk(parent_name, parent_alias, joins): + for join in joins or []: + child = require_str(join, "name", "join") + # `source` is the reserved fact qualifier; reject any casing. + if child.strip().lower() == "source": + raise ConversionError( + "Join name 'source' is reserved for the fact source; rename the join." + ) + if child.strip().lower() in seen_names: + raise ConversionError( + f"Duplicate dataset/join name '{child}'; Metric View join names " + f"and the source must be distinct (case-insensitively)." + ) + seen_names.add(child.strip().lower()) + child_ds = {"name": child, "source": require_str(join, "source", f"join '{child}'")} + datasets.append(child_ds) + alias_to_dataset[child] = child + rel = _convert_join(join, parent_name, parent_alias, child) + relationships.append(rel) + # rely.at_most_one_match asserts the join key is unique on the joined + # (one) side, so record those columns as a unique key on the child dataset + # -- recovering key info Apache Ossie would otherwise lack. Only a many_to_one join + # has the child on the `to` side (one_to_many flips it), so this naturally + # skips one_to_many joins. + if (rel["to"] == child and rel.get("to_columns") + and (join.get("rely") or {}).get("at_most_one_match")): + child_ds["unique_keys"] = [list(rel["to_columns"])] + walk(child, child, join.get("joins")) + + walk(fact_name, "source", view.get("joins")) + + # Dimensions -> fields, grouped onto the dataset their alias points at. + # `fields` is a v1.1 alias for `dimensions` (and the form the DBR docs use), + # so accept either key. + if view.get("dimensions") and view.get("fields"): + _warn("view", "both 'dimensions' and 'fields' are set; 'fields' is a v1.1 alias " + "for 'dimensions', so the 'fields' list is ignored") + fields_by_dataset = {d["name"]: [] for d in datasets} + for dim in (view.get("dimensions") or view.get("fields") or []): + if _is_wildcard(dim): + _warn("dimension", f"wildcard column '{dim.get('expr')}' has no Apache Ossie field " + f"representation; skipped") + continue + ds_name, field = _convert_dimension(dim, alias_to_dataset, fact_name) + fields_by_dataset[ds_name].append(field) + for d in datasets: + flds = fields_by_dataset[d["name"]] + if flds: + d["fields"] = flds + + metrics = [] + for m in view.get("measures", []) or []: + if _is_wildcard(m): + _warn("measure", f"wildcard measure '{m.get('expr')}' has no Apache Ossie metric " + f"representation; skipped") + continue + metrics.append(_convert_measure(m, fact_name)) + + model = {"name": fact_name} + if view.get("comment"): + model["description"] = view["comment"] + model["datasets"] = datasets + if relationships: + model["relationships"] = relationships + if metrics: + model["metrics"] = metrics + + # Model-level stash: filter / parameters / materialization, plus the source + # dataset's identity when a one_to_many join is present -- without it the + # exporter's FK-sink heuristic would re-root at the wrong (many-side) dataset. + model_stash = {k: view[k] for k in _MODEL_STASH_KEYS if k in view} + if _has_otm(view.get("joins")): + model_stash[STASH_SOURCE_KEY] = fact_name + write_stash(model, model_stash) + return model + + +def _has_otm(joins): + """True if any join in the (nested) tree is one_to_many.""" + for j in joins or []: + if str(j.get("cardinality") or "").lower() == CARD_ONE_TO_MANY: + return True + if _has_otm(j.get("joins")): + return True + return False + + +def _convert_join(join, parent_name, parent_alias, child): + if not join.get("using") and not join.get("on"): + raise ConversionError( + f"Join '{child}' has no join condition (empty or absent 'on'/'using'); " + f"condition-less (cross) joins have no Apache Ossie relationship representation." + ) + # _decompose_on returns (parent-side columns, child-side columns). + parent_cols, child_cols, raw_on = _decompose_on(join, parent_alias, parent_name, child) + if raw_on is not None: + raise ConversionError( + f"Join '{child}' uses a non-equi or unsupported join condition ('on: {raw_on}') " + f"that an Apache Ossie relationship cannot represent. Apache Ossie joins are equi-joins of simple " + f"`alias.column` pairs (the fact side may be qualified with `source`, the source " + f"table name, or left bare). Cannot import." + ) + if "using" in join and not parent_cols: + # `using: [cols]` -> equal lists on both sides. Two distinct list objects, so the + # emitted YAML doesn't serialize one as an anchor/alias of the other. + parent_cols, child_cols = list(join["using"]), list(join["using"]) + + # Cardinality (default many_to_one) decides the Apache Ossie direction, since `from` is + # always the many side. many_to_one -> parent is many (from=parent); one_to_many + # -> the joined child is many (from=child, to=parent). Compared case-insensitively. + cardinality = join.get("cardinality") or CARD_MANY_TO_ONE + if str(cardinality).lower() == CARD_ONE_TO_MANY: + rel = {"name": f"{child}_to_{parent_name}", "from": child, "to": parent_name, + "from_columns": child_cols, "to_columns": parent_cols} + else: + rel = {"name": f"{parent_name}_to_{child}", "from": parent_name, "to": child, + "from_columns": parent_cols, "to_columns": child_cols} + + stash = {k: join[k] for k in _JOIN_STASH_KEYS if k in join} + write_stash(rel, stash) + return rel + + +def _decompose_on(join, parent_alias, parent_name, child_alias): + """Return (from_columns, to_columns, raw_on). + + raw_on is None when `on` decomposes cleanly into equi-join column pairs; it + holds the original string otherwise (a non-equi/complex condition the caller + rejects). `using` short-circuits to empty columns here and is handled by the caller. + + The child side of a clause is always referenced by its join name. The parent side + may be referenced by its alias (`source` at the top level, else the parent join + name) or by the parent dataset's own name. A bare (unqualified) operand is read as + the fact only at the top level; inside a nested join it is ambiguous (parent vs. + fact) and is rejected rather than guessed. + """ + if "using" in join: + return [], [], None + on = join.get("on") + if not on: + return [], [], None + + parent_aliases = {parent_alias, parent_name} + from_cols, to_cols = [], [] + for clause in re.split(r"\s+AND\s+", on, flags=re.IGNORECASE): + if _NON_EQUI_RE.search(clause): # >=, <=, !=, <>, <, > -> not an equi-join + return [], [], on + m = re.match(r"^\s*(.+?)\s*=\s*(.+?)\s*$", clause) + if not m: + return [], [], on + la, lc = _split_alias(m.group(1)) + ra, rc = _split_alias(m.group(2)) + # Both sides must be `.` (or a bare fact column). If an + # operand is a SQL fragment (e.g. `dim.b + 1`, or the trailing half of an + # OR/`=`-laden clause), `_split_alias` yields a non-identifier "column" -- + # that can't be an FK column pair, so stash the whole condition verbatim. + if not (is_simple_identifier(lc) and is_simple_identifier(rc)): + return [], [], on + # The parent side: its alias or the source table name. A *bare* (unqualified) + # operand is read as the fact only at the top level (`source`); inside a nested + # join an unqualified column is ambiguous (parent vs. fact), so don't guess -- + # leave it for rejection rather than silently attributing it to the parent. + allow_bare = parent_alias == "source" + l_parent = la in parent_aliases or (la is None and allow_bare) + r_parent = ra in parent_aliases or (ra is None and allow_bare) + if la == child_alias and r_parent: + from_cols.append(rc) + to_cols.append(lc) + elif ra == child_alias and l_parent: + from_cols.append(lc) + to_cols.append(rc) + else: + return [], [], on + return from_cols, to_cols, None + + +def _split_alias(operand): + """`customer.c_custkey` -> ('customer', 'c_custkey'); `x` -> (None, 'x').""" + operand = operand.strip() + if "." in operand: + alias, col = operand.split(".", 1) + return alias.strip(), col.strip() + return None, operand + + +def _convert_dimension(dim, alias_to_dataset, fact_name): + name = require_str(dim, "name", "dimension") + expr = require_str(dim, "expr", f"dimension '{name}'") + ds_name, ossie_expr = _resolve_column(expr, alias_to_dataset, fact_name) + + field = { + "name": name, + "expression": {"dialects": [{"dialect": DIALECT_DATABRICKS, "expression": ossie_expr}]}, + } + if dim.get("comment"): + field["description"] = dim["comment"] + if dim.get("display_name"): + field["label"] = dim["display_name"] + if dim.get("synonyms"): + field["ai_context"] = {"synonyms": list(dim["synonyms"])} + write_stash(field, {k: dim[k] for k in _COLUMN_STASH_KEYS if k in dim}) + return ds_name, field + + +def _resolve_column(expr, alias_to_dataset, fact_name): + """Map a dimension expression to (dataset_name, de-aliased_expression). + + A leading join path of known aliases files the field under the **deepest** one and + de-qualifies a bare column -- mirroring the exporter's nested-join qualification: + `partsupp.supplier.nation.n_name` -> `n_name` on `nation`; `customer.c_name` -> + `c_name` on `customer`; `source.x` -> `x` on the fact. A complex expression is filed + under that dataset but kept verbatim. A bare column (no leading alias) is a fact column. + """ + segments = [s.strip() for s in expr.split(".")] + ds = None + i = 0 + # Consume leading segments that are known join/source aliases (but never the last + # segment -- that is the column). The deepest alias is the owning dataset. + while i < len(segments) - 1 and segments[i] in alias_to_dataset: + ds = alias_to_dataset[segments[i]] + i += 1 + if ds is None: + return fact_name, expr + rest = ".".join(segments[i:]) + return (ds, rest) if is_simple_identifier(rest) else (ds, expr) + + +def _convert_measure(measure, fact_name): + name = require_str(measure, "name", "measure") + # Mirror the exporter's word-boundary handling: a `source.` fact qualifier (if + # an author used one) maps back to the fact dataset name. The replacement is a + # lambda so `fact_name` is inserted literally (a `--name` containing backslashes + # is not interpreted as a regex backreference). + raw_expr = require_str(measure, "expr", f"measure '{name}'") + expr = re.sub(r"\bsource\.", lambda _m: f"{fact_name}.", raw_expr) + metric = { + "name": name, + "expression": {"dialects": [{"dialect": DIALECT_DATABRICKS, "expression": expr}]}, + } + if measure.get("comment"): + metric["description"] = measure["comment"] + if measure.get("synonyms"): + metric["ai_context"] = {"synonyms": list(measure["synonyms"])} + write_stash(metric, {k: measure[k] for k in _COLUMN_STASH_KEYS if k in measure}) + return metric diff --git a/converters/databricks/src/ossie_databricks/ossie_to_metric_view.py b/converters/databricks/src/ossie_databricks/ossie_to_metric_view.py new file mode 100644 index 0000000..3c535ed --- /dev/null +++ b/converters/databricks/src/ossie_databricks/ossie_to_metric_view.py @@ -0,0 +1,660 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert an Apache Ossie semantic model to a Databricks Unity Catalog Metric View (v1.1). + +Pure offline conversion -- no Databricks connection required. Produces the +Metric View: one fact `source` with a nested `joins` tree and +all fields flattened into one `dimensions` list. See README.md for the +capability summary and limitations. + +Usage (CLI): + ossie-databricks export -i model.yaml [-o view.yaml] [--source orders] +""" + +import re +import warnings + +from ._common import ( + CARD_ONE_TO_MANY, + ConversionError, + MV_VERSION, + OSSIE_VERSION, + STASH_SOURCE_KEY, + SYNONYM_LIMIT, + dump_yaml, + foreign_vendor_extensions, + is_simple_identifier, + load_yaml, + merge_description, + pick_expression, + read_stash, + require, + require_str, + synonyms_of, + validate_source, +) + + +def _warn(scope, msg): + warnings.warn(f"[{scope}] {msg}") + + +# Fanning a diamond out into per-path joins can expand exponentially on a pathological +# lattice; real snowflakes are tiny, so cap total joins to catch runaway inputs. +_MAX_JOIN_NODES = 200 + + +def convert_ossie_to_metric_view(ossie_yaml_str, source=None): + """Parse Apache Ossie YAML and return Databricks Metric View v1.1 YAML (string). + + `source` names the dataset to use as the view's fact/grain. When omitted, the + fact is the dataset that is never a relationship `to` (the FK sink of a plain + many-to-one star). Naming a coarser-grain dataset as the source is what unlocks + `one_to_many` joins (the joined detail tables sit on the `from`/many side). + """ + root = load_yaml(ossie_yaml_str) + if not isinstance(root, dict): + raise ConversionError("Invalid Apache Ossie YAML: expected a mapping at the root") + + version = str(root.get("version", "")) + if version != OSSIE_VERSION: + raise ConversionError( + f"Unsupported Apache Ossie version '{version}'. Supported: {OSSIE_VERSION}" + ) + + models = root.get("semantic_model") + if not isinstance(models, list) or not models: + raise ConversionError("'semantic_model' must be a non-empty list") + if len(models) > 1: + _warn("model", "multiple semantic models found; converting only the first") + + view = _convert_model(models[0], explicit_source=source) + return dump_yaml(view) + + +def _convert_model(model, explicit_source=None): + name = model.get("name", "") + dataset_list = model.get("datasets", []) or [] + if not dataset_list: + raise ConversionError(f"Model '{name}' has no datasets") + + seen = set() + for d in dataset_list: + ds_name = require_str(d, "name", f"Model '{name}': dataset") + if ds_name.strip().lower() in seen: # case-insensitive: DBR identifiers are + raise ConversionError(f"Model '{name}': duplicate dataset name '{ds_name}'") + seen.add(ds_name.strip().lower()) + datasets = {d["name"]: d for d in dataset_list} + relationships = model.get("relationships", []) or [] + + # Model-level stash: filter / parameters / materialization, plus an optional + # `source_dataset` recording the original grain (written on import only when a + # one_to_many join made the fact ambiguous). An explicit `source` arg wins. + model_stash = read_stash(model) + fact_hint = explicit_source or model_stash.get(STASH_SOURCE_KEY) + root, fact = _build_join_tree(name, datasets, relationships, fact_hint) + counts = _assign_aliases(root, fact) + + # Mark one_to_many nodes (parent on the `to`/one side); their columns can't be + # dimensions. Also validates one_to_many subtree uniformity. + _mark_otm(name, root) + + fact_ds = datasets[fact] + view = {"version": MV_VERSION, "source": validate_source(fact_ds.get("source"), fact)} + + # The view comment is simply the model's top-level description -- the closest + # match. Model ai_context and dataset descriptions are not merged in (dropped; + # see _warn_dropped_model), which keeps model.description round-trippable. + comment = model.get("description") + if comment: + view["comment"] = comment + + if "filter" in model_stash: + view["filter"] = model_stash["filter"] + + joins = [_build_join(child, "source", datasets) for child in root["children"]] + if joins: + view["joins"] = joins + + # Dimensions: every field across every join instance, fact first then join order. + # A dataset joined under more than one alias (fanned out) is emitted once per + # instance with alias-prefixed names. Track dropped names so we can cascade-drop + # anything that references them. + dropped_dims, dropped_measures = set(), set() + dimensions = [] + seen_dims = set() + for node, join_path in _node_order(root): + is_fact = node is root + prefix = node["alias"] if counts[node["dataset"]] > 1 else None + for field in datasets[node["dataset"]].get("fields", []) or []: + fname = require_str(field, "name", f"dataset '{node['dataset']}': field") + if node["is_otm"]: + _warn( + f"field '{fname}'", + "column on a one-to-many-joined table cannot be a dimension " + "(must resolve to one value per source row); dropped", + ) + dropped_dims.add(fname) + continue + dim = _convert_field(field, fname, ".".join(join_path), is_fact, prefix) + if dim is None: + dropped_dims.add(fname) + continue + if dim["name"].lower() in seen_dims: # case-insensitive + raise ConversionError( + f"dataset '{node['dataset']}': dimension name '{dim['name']}' " + f"collides with another dimension/measure; Metric Views require " + f"unique dimension/measure names -- rename before use" + ) + seen_dims.add(dim["name"].lower()) + dimensions.append(dim) + + measures = [] + for metric in model.get("metrics", []) or []: + measure = _convert_metric(metric, fact, seen_dims) + if measure is None: + dropped_measures.add(metric.get("name")) + continue + measures.append(measure) + + # Cascade: drop any dimension/measure whose expression references a dropped + # name (transitively), so we never emit a dangling reference. + _cascade_drop(dimensions, measures, dropped_dims, dropped_measures) + + if dimensions: + view["dimensions"] = dimensions + if measures: + view["measures"] = measures + + if "parameters" in model_stash: + view["parameters"] = model_stash["parameters"] + if "materialization" in model_stash: + view["materialization"] = model_stash["materialization"] + + _warn_dropped_model(model) + return view + + +def _build_join_tree(model_name, datasets, relationships, fact_hint=None): + """Build the Metric View join tree from the Apache Ossie relationship graph; return + (root_node, fact_name). + + Each node is a dict: {alias, dataset, rel, parent_is_from, children, is_otm}. + Edges are oriented away from the fact (the nearer endpoint is the parent), so a + dataset reachable by more than one path -- a diamond, e.g. two facts sharing a + dimension, or a dimension reached via two parents -- is fanned out into one node + per path. Each instance is later given a unique alias, mirroring how a Metric View + joins the same table more than once. Non-tree (cyclic) shapes are rejected. + """ + for rel in relationships: + scope = f"Model '{model_name}': relationship '{rel.get('name', '')}'" + if require(rel, "from", scope) not in datasets or require(rel, "to", scope) not in datasets: + raise ConversionError( + f"Model '{model_name}': relationship '{rel.get('name')}' references " + f"an unknown dataset" + ) + + # Re-orient any relationship whose declared keys show `from`/`to` is mislabeled + # (the `from` columns are a unique key, the `to` columns are not). Done before + # fact selection so cardinality, columns, and fact choice all use the key-derived + # orientation. The join condition is unchanged (it is symmetric). + relationships = [_orient_by_key(model_name, rel, datasets) for rel in relationships] + + fact = _pick_fact(model_name, datasets, relationships, fact_hint) + + # BFS (undirected) measures each dataset's distance from the fact; that distance + # orients every edge away from the fact (parent = the nearer endpoint). + adj = {name: [] for name in datasets} + for rel in relationships: + adj[rel["from"]].append(rel["to"]) + adj[rel["to"]].append(rel["from"]) + dist = {fact: 0} + queue = [fact] + while queue: + cur = queue.pop(0) + for neighbor in adj[cur]: + if neighbor not in dist: + dist[neighbor] = dist[cur] + 1 + queue.append(neighbor) + + unreachable = set(datasets) - set(dist) + if unreachable: + raise ConversionError( + f"Model '{model_name}': datasets {sorted(unreachable)} are not reachable " + f"from fact '{fact}' via relationships." + ) + + # Orient each edge nearer->farther. An edge between two equidistant datasets has + # no fact-ward direction -- that only happens in a cyclic / non-tree graph. + children_of = {name: [] for name in datasets} + for rel in relationships: + a, b = rel["from"], rel["to"] + if dist[a] == dist[b]: + raise ConversionError( + f"Model '{model_name}': relationship '{rel.get('name')}' joins two " + f"datasets equidistant from the fact; the graph is not tree-shaped " + f"(it contains a cycle)." + ) + parent, child = (a, b) if dist[a] < dist[b] else (b, a) + children_of[parent].append((child, rel, parent == rel["from"])) + + counter = [0] + + def build(dataset, rel, parent_is_from): + counter[0] += 1 + if counter[0] > _MAX_JOIN_NODES: + raise ConversionError( + f"Model '{model_name}': join graph fans out to more than " + f"{_MAX_JOIN_NODES} joins; check for an unintended diamond explosion." + ) + node = {"alias": None, "dataset": dataset, "rel": rel, + "parent_is_from": parent_is_from, "children": [], "is_otm": False} + for child, crel, cfrom in children_of[dataset]: + node["children"].append(build(child, crel, cfrom)) + return node + + return build(fact, None, None), fact + + +def _assign_aliases(root, fact): + """Give every node a unique join alias and return per-dataset instance counts. + + A dataset with a single instance keeps its bare name (so non-diamond graphs are + unchanged); a fanned-out dataset's instances are disambiguated by parent alias + (e.g. `customers_regions` / `suppliers_regions`). The fact's alias is `source`. + """ + counts = {} + + def count(node): + counts[node["dataset"]] = counts.get(node["dataset"], 0) + 1 + for c in node["children"]: + count(c) + + count(root) + + used = {"source"} # reserved for the fact, so a dataset named `source` gets renamed + + def assign(node, parent_alias): + if node["dataset"] == fact: + alias = "source" + else: + # Single-instance datasets keep their bare name; fanned-out ones are + # qualified by the parent alias. Either way the result is deduped against + # `used` (which reserves `source`), so no two joins ever share an alias. + if counts[node["dataset"]] == 1: + base = node["dataset"] + else: + base = (f"{parent_alias}_{node['dataset']}" + if parent_alias and parent_alias != "source" else node["dataset"]) + alias, n = base, 2 + while alias in used: + alias, n = f"{base}_{n}", n + 1 + node["alias"] = alias + used.add(alias) + for c in node["children"]: + assign(c, alias) + + assign(root, None) + return counts + + +def _pick_fact(model_name, datasets, relationships, fact_hint): + """Choose the fact/root: an explicit hint if given, else the dataset that is + never a relationship `to` (the FK sink of a plain many-to-one star).""" + if fact_hint is not None: + if fact_hint not in datasets: + raise ConversionError( + f"Model '{model_name}': requested source '{fact_hint}' is not a dataset" + ) + return fact_hint + if len(datasets) > 1 and not relationships: + raise ConversionError( + f"Model '{model_name}': {len(datasets)} datasets but no relationships; " + f"cannot determine the fact table." + ) + incoming = {name: 0 for name in datasets} + for rel in relationships: + incoming[rel["to"]] += 1 + roots = [n for n, c in incoming.items() if c == 0] + if not roots: + raise ConversionError( + f"Model '{model_name}': join graph contains a cycle (no root dataset). " + f"A Metric View requires an acyclic, tree-shaped graph." + ) + if len(roots) > 1: + raise ConversionError( + f"Model '{model_name}': multiple candidate fact datasets {sorted(roots)}. " + f"Name the grain with --source -- e.g. for multiple facts sharing a " + f"dimension, name that dimension so each fact becomes a one_to_many join." + ) + return roots[0] + + +def _mark_otm(model_name, root): + """Mark each node `is_otm` (reached through a one_to_many join -- a parent on the + `to`/one side). Their columns can't be dimensions. Enforces the DBR rule that + every descendant of a one_to_many join is itself one_to_many.""" + + def visit(node, under_otm): + for child in node["children"]: + is_otm = not child["parent_is_from"] # parent on the `to` (one) side + if under_otm and not is_otm: + raise ConversionError( + f"Model '{model_name}': join '{child['alias']}' is many-to-one but " + f"descends from a one-to-many join; all descendants of a one-to-many " + f"join must also be one-to-many (Databricks Metric View rule)." + ) + child["is_otm"] = under_otm or is_otm + visit(child, child["is_otm"]) + + visit(root, False) + + +def _node_order(root): + """Fact first, then a stable depth-first walk of the join tree (one node per join + instance, so a fanned-out dataset appears once per path). Yields (node, join_path): + `join_path` is the tuple of join aliases from the source down to and including the + node (empty for the fact). A joined column is qualified in a dimension/measure by this + full path (`parent.child.col`) -- the Databricks nested-join rule -- which for a + depth-1 join is just the join's own name.""" + order = [] + + def visit(node, path): + order.append((node, path)) + for child in node["children"]: + visit(child, path + (child["alias"],)) + + visit(root, ()) + return order + + +def _build_join(node, parent_alias, datasets): + """Build one Metric View join entry from a tree node (recursively for nested joins). + + `node['parent_is_from']` is True when the parent is the relationship's `from` + (many) side -> a many_to_one join (the default, left implicit). When the parent is + the `to` (one) side the join is one_to_many and the column roles flip. + """ + rel, alias = node["rel"], node["alias"] + join = {"name": alias, + "source": validate_source(datasets[node["dataset"]].get("source"), node["dataset"])} + + stash = read_stash(rel) + from_cols = rel.get("from_columns") or [] + to_cols = rel.get("to_columns") or [] + # Apache Ossie relationships are equi-joins; a relationship without usable equi columns + # (e.g. a non-equi join the importer would have rejected) is rejected here too. + _validate_join_columns(rel, from_cols, to_cols) + # Write parent-side = child-side: the parent uses whichever list belongs to + # it -- from_columns when it is the `from`, to_columns when it is the `to`. + parent_cols, child_cols = ( + (from_cols, to_cols) if node["parent_is_from"] else (to_cols, from_cols)) + if parent_cols == child_cols: + # Equal column lists are an equi-join on shared names -> `using`, which + # round-trips faithfully (the importer maps `using` to equal lists). + join["using"] = list(parent_cols) + else: + join["on"] = " AND ".join( + f"{parent_alias}.{pc} = {alias}.{cc}" for pc, cc in zip(parent_cols, child_cols) + ) + # rely.at_most_one_match: a stashed value round-trips verbatim; otherwise derive it + # for a many_to_one join whose `to_columns` cover a declared primary/unique key of + # the joined dataset (joining on a key matches at most one row -- no fan-out). + if "rely" in stash: + join["rely"] = stash["rely"] + elif node["parent_is_from"] and _covers_unique_key(datasets[node["dataset"]], to_cols): + join["rely"] = {"at_most_one_match": True} + # Cardinality: an explicit stashed value round-trips verbatim; otherwise derive + # from orientation -- parent on the `to` (one) side means one_to_many. The + # many_to_one default is left implicit. + if "cardinality" in stash: + join["cardinality"] = stash["cardinality"] + elif not node["parent_is_from"]: + join["cardinality"] = CARD_ONE_TO_MANY + + nested = [_build_join(c, alias, datasets) for c in node["children"]] + if nested: + join["joins"] = nested + return join + + +def _covers_unique_key(dataset, join_cols): + """True if `join_cols` include a declared `primary_key` or one of `unique_keys` of + `dataset` -- i.e. joining on them matches at most one target row, so a many_to_one + join can assert `rely.at_most_one_match`.""" + cols = set(join_cols) + keys = [dataset.get("primary_key")] if dataset.get("primary_key") else [] + keys += dataset.get("unique_keys") or [] + return any(key and set(key) <= cols for key in keys) + + +def _orient_by_key(model_name, rel, datasets): + """`to` should be the unique 'one' side (per spec `to_columns` are key columns). If + the declared keys say otherwise -- the `from` columns are a unique key while the + `to` side declares keys its `to_columns` don't cover -- `from`/`to` is mislabeled. + Return a copy with `from`/`to` (and their columns) swapped, and warn. The swap is + symmetric, so the join condition is unchanged; only the orientation is corrected. + + When the `from` columns cover a unique key but the `to` side declares no key at all, + the orientation can't be verified either way (the `to` side may or may not be + unique); leave it as-is but warn, since the resulting cardinality may be inverted.""" + from_cols = rel.get("from_columns") or [] + to_cols = rel.get("to_columns") or [] + if not from_cols or not to_cols: + return rel # non-equi / column-less: nothing to deduce from + to_ds = datasets[rel["to"]] + to_has_keys = bool(to_ds.get("primary_key") or to_ds.get("unique_keys")) + from_covers = _covers_unique_key(datasets[rel["from"]], from_cols) + if from_covers and to_has_keys and not _covers_unique_key(to_ds, to_cols): + _warn( + f"relationship '{rel.get('name')}'", + "from/to looks mislabeled (the `from` columns are a declared key, the `to` " + "columns are not); re-orienting so the key side is the `to`/one side", + ) + return {**rel, "from": rel["to"], "to": rel["from"], + "from_columns": to_cols, "to_columns": from_cols} + if from_covers and not to_has_keys: + _warn( + f"relationship '{rel.get('name')}'", + "the `from` columns are a declared key but the `to` side declares none, so " + "from/to orientation can't be verified; using it as-is -- check the join " + "direction if the resulting cardinality looks inverted", + ) + return rel + + +def _validate_join_columns(rel, from_cols, to_cols): + if not from_cols or not to_cols: + raise ConversionError( + f"Relationship '{rel.get('name')}': from_columns and to_columns are required" + ) + if not isinstance(from_cols, list) or not isinstance(to_cols, list): + raise ConversionError( + f"Relationship '{rel.get('name')}': from_columns and to_columns must be lists" + ) + if len(from_cols) != len(to_cols): + raise ConversionError( + f"Relationship '{rel.get('name')}': from_columns ({len(from_cols)}) and " + f"to_columns ({len(to_cols)}) must have the same length" + ) + + +def _convert_field(field, name, qualifier, is_fact, prefix=None): + scope = f"field '{name}'" + expr = pick_expression(field.get("expression")) + if expr is None: + _warn(scope, "no DATABRICKS/ANSI_SQL dialect; dropping field") + return None + + # Requalify a joined-table column with its full join-name path from the source + # (`parent.child.col`); a depth-1 join is just its own name. Only safe for bare + # columns. A complex expression on a single join is emitted as-is (likely resolves; + # warned). On a fanned-out (diamond) dataset it cannot be attributed to one of the + # instances, so it is dropped rather than emitted as an ambiguous dimension. + if not is_fact: + if is_simple_identifier(expr): + expr = f"{qualifier}.{expr}" + elif prefix: + _warn(scope, "complex expression on a fanned-out (diamond) join cannot be " + "unambiguously qualified; dropped") + return None + else: + _warn(scope, "complex expression on a joined table; emitted as-is, verify qualification") + + # A fanned-out dataset (joined under more than one alias) needs unique dimension + # names, so prefix with the instance alias (e.g. customer_region's r_name -> + # customer_region_r_name). Single-instance datasets keep the bare field name. + if prefix: + name = f"{prefix}_{name}" + + dim = {"name": name, "expr": expr} + comment = merge_description(field.get("description"), field.get("ai_context")) + if comment: + dim["comment"] = comment + if field.get("label"): + dim["display_name"] = field["label"] + syns = synonyms_of(field.get("ai_context")) + if syns: + dim["synonyms"] = _truncate_synonyms(syns, scope) + + stash = read_stash(field) + if "format" in stash: + dim["format"] = stash["format"] + _warn_dropped_field(field, scope) + return dim + + +def _convert_metric(metric, fact, seen_names): + name = require_str(metric, "name", "metric") + scope = f"metric '{name}'" + if name.lower() in seen_names: # case-insensitive (shares seen_dims with dimensions) + raise ConversionError( + f"metric '{name}' collides with another dimension/measure; Metric Views " + f"require unique dimension/measure names -- rename before use") + seen_names.add(name.lower()) + expr = pick_expression(metric.get("expression")) + if expr is None: + _warn(scope, "no DATABRICKS/ANSI_SQL dialect; dropping metric") + return None + + # Fact-table columns are referenced by bare name in measure expressions (DBR + # idiom: `SUM(amount)`, not `SUM(source.amount)`); strip a `.` qualifier. + # Joined-table columns keep their alias. Word boundary avoids touching a table + # whose name merely ends with the fact name (e.g. fact 'sales' vs 'store_sales'). + expr = re.sub(r"\b" + re.escape(fact) + r"\.", "", expr) + + measure = {"name": name, "expr": expr} + comment = merge_description(metric.get("description"), metric.get("ai_context")) + if comment: + measure["comment"] = comment + syns = synonyms_of(metric.get("ai_context")) + if syns: + measure["synonyms"] = _truncate_synonyms(syns, scope) + + stash = read_stash(metric) + if "format" in stash: + measure["format"] = stash["format"] + if "window" in stash: + measure["window"] = stash["window"] + return measure + + +def _references_dropped(expr, self_name, dropped_dims, dropped_measures): + """Return a dropped name referenced by `expr`, or None. + + Measures are only referenceable via `measure()` (exact). Dimensions are + referenced by their bare, *unqualified* name: a name that is part of a qualified + path (`alias.name` or `name.col`) is ignored, so a join alias or joined column + that merely shares a dropped dimension's name is not over-dropped. The one + ambiguity the regex can't resolve without a SQL parser is a bare, unqualified + *source column* sharing a dropped dimension's name -- there it errs on dropping. + """ + for m in dropped_measures: + if re.search(r"measure\(\s*" + re.escape(m) + r"\s*\)", expr): + return m + for d in dropped_dims: + # Match only a bare, unqualified token: the negative look-behind/ahead for a + # word char or `.` excludes both substrings of a larger identifier and + # qualified paths (`alias.name` / `name.col`), so a join alias or joined + # column sharing a dropped name is not falsely cascade-dropped. + if d != self_name and re.search( + r"(? SYNONYM_LIMIT: + _warn(scope, f"{len(syns)} synonyms exceeds Metric View limit; keeping first {SYNONYM_LIMIT}") + return syns[:SYNONYM_LIMIT] + return syns + + +def _warn_dropped_model(model): + if foreign_vendor_extensions(model): + _warn("model", "foreign-vendor custom_extensions dropped") + if model.get("ai_context"): # string or object -- only the description maps to comment + _warn("model", "model-level ai_context dropped (only the description maps to the view comment)") + for ds in model.get("datasets", []) or []: + scope = f"dataset '{ds['name']}'" + if ds.get("primary_key") or ds.get("unique_keys"): + _warn(scope, "primary_key/unique_keys not stored as columns; used to set " + "rely.at_most_one_match on a matching many_to_one join where applicable") + if isinstance(ds.get("ai_context"), dict) and ds["ai_context"]: + _warn(scope, "dataset-level ai_context (object) dropped") + # Dataset descriptions are not merged into the view comment (a Metric View + # has no per-source comment); only the model description is used. + if ds.get("description"): + _warn(scope, "dataset-level description dropped (no per-source comment field)") + if foreign_vendor_extensions(ds): + _warn(scope, "foreign-vendor custom_extensions dropped") + for rel in model.get("relationships", []) or []: + if rel.get("ai_context"): + _warn(f"relationship '{rel.get('name', '')}'", "relationship ai_context dropped") + + +def _warn_dropped_field(field, scope): + dim = field.get("dimension") + if isinstance(dim, dict) and "is_time" in dim: + _warn(scope, "dimension.is_time has no Metric View counterpart; dropped") + if foreign_vendor_extensions(field): + _warn(scope, "foreign-vendor custom_extensions dropped") diff --git a/converters/databricks/tests/_roundtrip_helpers.py b/converters/databricks/tests/_roundtrip_helpers.py new file mode 100644 index 0000000..6c4ddfb --- /dev/null +++ b/converters/databricks/tests/_roundtrip_helpers.py @@ -0,0 +1,355 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared model builders and round-trip assertions for property-based tests. + +This module is deliberately free of any third-party test dependency (no hypothesis, +no pytest) so the generation + assertion logic can run two ways: + + - driven by Hypothesis strategies (see test_roundtrip_properties.py), and + - driven by a plain seeded `random.Random` (RandomRnd below), which is how the + logic is exercised in environments where hypothesis is not installed. + +Both drivers implement the small `Rnd` interface (chance/count/pick/text/colname); +the builders below depend only on that interface, so the generated model space is +identical regardless of driver. + +The builders intentionally generate within the *round-trippable subset* of models - +the shapes the converter reproduces exactly. Known normalizations are avoided by +construction (documented inline) rather than asserted around, e.g.: + - join `on` conditions use distinct parent/child column names, so an equi-join on + shared names is never silently rewritten to `using`; + - measure expressions are emitted without a `source.` qualifier, which the exporter + would otherwise strip; + - in the Apache Ossie direction, joined-dataset fields are bare identifiers, so they survive + the alias requalify/de-qualify trip on the same dataset. +Name fuzzing (reserved words, collisions) is left to the targeted unit tests, which +assert the converter *rejects* those inputs. +""" + +import random +import re +import string +import warnings + +from ossie_databricks import metric_view_to_ossie as importer +from ossie_databricks import ossie_to_metric_view as exporter +from ossie_databricks._common import MV_VERSION, OSSIE_VERSION, dump_yaml, load_yaml + +_AGGS = ["SUM", "COUNT", "AVG", "MIN", "MAX"] + + +# --- Rnd backend for offline (no hypothesis) runs -------------------------------- + +class RandomRnd: + """The `Rnd` interface backed by a seeded `random.Random`.""" + + def __init__(self, seed): + self.r = random.Random(seed) + + def chance(self, p=0.5): + return self.r.random() < p + + def count(self, lo, hi): + return self.r.randint(lo, hi) + + def pick(self, seq): + return self.r.choice(list(seq)) + + def text(self): + # Alphanumeric with optional interior spaces; no leading/trailing space and + # no YAML-special characters, so the value is preserved verbatim through a + # dump/load cycle (any failure then reflects the converter, not PyYAML). + alnum = string.ascii_letters + string.digits + n = self.r.randint(0, 10) + body = "".join(self.r.choice(alnum + " ") for _ in range(n)) + return (self.r.choice(alnum) + body).strip() or "x" + + def colname(self): + first = self.r.choice(string.ascii_lowercase + "_") + rest = "".join( + self.r.choice(string.ascii_lowercase + string.digits + "_") + for _ in range(self.r.randint(0, 7)) + ) + return first + rest + + +# --- Small generation helpers ---------------------------------------------------- + +class _Names: + """Hands out globally-unique names with a given prefix.""" + + def __init__(self): + self._n = {} + + def next(self, prefix): + i = self._n.get(prefix, 0) + self._n[prefix] = i + 1 + return f"{prefix}{i}" + + +def _maybe_meta(rnd, target): + """Attach optional comment/display_name/synonyms/format to a dim/measure dict.""" + if rnd.chance(0.4): + target["comment"] = rnd.text() + if rnd.chance(0.3): + target["display_name"] = rnd.text() + if rnd.chance(0.3): + target["synonyms"] = [rnd.text() for _ in range(rnd.count(1, 3))] + if rnd.chance(0.25): + fmt = {"type": rnd.pick(["number", "currency", "date"])} + if fmt["type"] == "currency": + fmt["currency_code"] = "USD" + target["format"] = fmt + + +# --- Metric View builder (for MV -> Apache Ossie -> MV) ----------------------------------- + +def _build_join(rnd, names, parent_alias, depth, ancestor_path): + name = names.next("j") + # Full join-name path from the source -> how dimensions/measures qualify this join's + # columns (DBR nested-join rule). `on:` conditions instead use the immediate names. + qual = ".".join(ancestor_path + [name]) + join = {"name": name, "source": _three_part(rnd)} + if rnd.chance(0.5): + ncols = rnd.count(1, 2) + join["using"] = [f"u{i}_{rnd.colname()}" for i in range(ncols)] + else: + ncols = rnd.count(1, 2) + # distinct parent/child names so the equi-join stays `on`, not `using` + pairs = [(f"fk{i}_{rnd.colname()}", f"pk{i}_{rnd.colname()}") for i in range(ncols)] + join["on"] = " AND ".join(f"{parent_alias}.{pc} = {name}.{cc}" for pc, cc in pairs) + if rnd.chance(0.4): + join["cardinality"] = "many_to_one" # only the lossless cardinality (see module doc) + if rnd.chance(0.3): + join["rely"] = {"at_most_one_match": True} + # dimensions on this join (qualified by the full join path) + dims = [] + for _ in range(rnd.count(0, 2)): + col = rnd.colname() + expr = (f"{qual}.{col}" if rnd.chance(0.7) + else f"{qual}.{col} + {qual}.{rnd.colname()}") + dim = {"name": names.next("c"), "expr": expr} + _maybe_meta(rnd, dim) + dims.append(dim) + if depth < 2 and rnd.chance(0.35): + child, child_dims = _build_join(rnd, names, name, depth + 1, ancestor_path + [name]) + join["joins"] = [child] + dims.extend(child_dims) + return join, dims + + +def build_metric_view(rnd): + """Generate a Metric View YAML dict in the round-trippable subset.""" + names = _Names() + mv = {"version": MV_VERSION, "source": _three_part(rnd)} + if rnd.chance(0.4): + mv["comment"] = rnd.text() + if rnd.chance(0.3): + mv["filter"] = f"{rnd.colname()} > 0" + + fields, joins = [], [] + for _ in range(rnd.count(0, 3)): # source dimensions (bare or function exprs) + col = rnd.colname() + expr = col if rnd.chance(0.7) else f"UPPER({col})" + dim = {"name": names.next("c"), "expr": expr} + _maybe_meta(rnd, dim) + fields.append(dim) + for _ in range(rnd.count(0, 2)): # join subtrees + join, jdims = _build_join(rnd, names, "source", 0, []) + joins.append(join) + fields.extend(jdims) + + measures = [] + for _ in range(rnd.count(0, 2)): + m = {"name": names.next("c"), "expr": f"{rnd.pick(_AGGS)}({rnd.colname()})"} + if rnd.chance(0.4): + m["comment"] = rnd.text() + if rnd.chance(0.3): + m["synonyms"] = [rnd.text() for _ in range(rnd.count(1, 3))] + if rnd.chance(0.3): + m["window"] = [{"order": rnd.colname(), "range": "trailing 7 day"}] + measures.append(m) + + if joins: + mv["joins"] = joins + if fields: + mv["fields"] = fields + if measures: + mv["measures"] = measures + if rnd.chance(0.2): + mv["materialization"] = {"schedule": "every 6 hours", + "mode": rnd.pick(["relaxed", "strict"])} + return mv + + +# --- Apache Ossie builder (for Apache Ossie -> MV -> Apache Ossie) ------------------------------------------ + +def _ossie_field(name, expr): + return {"name": name, + "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": expr}]}} + + +def build_ossie(rnd): + """Generate an Apache Ossie semantic model dict in the round-trippable subset.""" + names = _Names() + fact = "fact" # fact name must equal its source's last identifier to round-trip + datasets = [{"name": fact, "source": f"c.s.{fact}"}] + relationships = [] + + n_dims = rnd.count(0, 3) + dim_names = [names.next("dim") for _ in range(n_dims)] + reachable = [fact] + for i, dname in enumerate(dim_names): + parent = rnd.pick(reachable) # star, or snowflake off an earlier node + ds = {"name": dname, "source": f"c.s.{rnd.colname()}{i}"} + datasets.append(ds) + reachable.append(dname) + if rnd.chance(0.5): # equal column names -> `using`; else distinct -> `on` + cols = [rnd.colname() for _ in range(rnd.count(1, 2))] + relationships.append({"name": names.next("r"), "from": parent, "to": dname, + "from_columns": list(cols), "to_columns": list(cols)}) + else: + n = rnd.count(1, 2) + fcols = [f"fk{j}_{rnd.colname()}" for j in range(n)] + tcols = [f"pk{j}_{rnd.colname()}" for j in range(n)] + relationships.append({"name": names.next("r"), "from": parent, "to": dname, + "from_columns": fcols, "to_columns": tcols}) + + # fields, bare identifiers, filed onto a random dataset (globally unique names) + for ds in datasets: + flds = [_ossie_field(names.next("c"), rnd.colname()) for _ in range(rnd.count(0, 3))] + if flds: + ds["fields"] = flds + + metrics = [{"name": names.next("c"), + "expression": {"dialects": [{"dialect": "DATABRICKS", + "expression": f"{rnd.pick(_AGGS)}({rnd.colname()})"}]}} + for _ in range(rnd.count(0, 2))] + + model = {"name": names.next("m")} + if rnd.chance(0.4): + model["description"] = rnd.text() + model["datasets"] = datasets + if relationships: + model["relationships"] = relationships + if metrics: + model["metrics"] = metrics + return {"version": OSSIE_VERSION, "semantic_model": [model]} + + +def _three_part(rnd): + return f"{rnd.colname()}.{rnd.colname()}.{rnd.colname()}" + + +# --- Round-trip assertions ------------------------------------------------------- + +def _convert(fn, text): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return fn(text) + + +def _cond_canon(join): + if join.get("using"): + return ("using", tuple(sorted(join["using"]))) + on = join.get("on") + if not on: + return (None, None) + pairs = set() + for clause in re.split(r"\s+AND\s+", on, flags=re.IGNORECASE): + left, right = clause.split("=", 1) + pairs.add((left.strip(), right.strip())) + return ("on", frozenset(pairs)) + + +def _flatten_joins(joins, parent="source", acc=None, edges=None): + acc = {} if acc is None else acc + edges = set() if edges is None else edges + for j in joins or []: + acc[j["name"]] = {"source": j["source"], "cond": _cond_canon(j), + "cardinality": j.get("cardinality"), "rely": j.get("rely")} + edges.add((parent, j["name"])) + _flatten_joins(j.get("joins"), j["name"], acc, edges) + return acc, edges + + +def _dims(mv): + # The exporter emits the canonical `dimensions:` key; the importer also accepts the + # `fields:` alias. Read either so the comparison is key-name agnostic. + return mv.get("dimensions") or mv.get("fields") or [] + + +def _dim_norm(d): + return (d["expr"], d.get("comment"), d.get("display_name"), + d.get("synonyms"), d.get("format")) + + +def _meas_norm(m): + return (m["expr"], m.get("comment"), m.get("synonyms"), m.get("format"), m.get("window")) + + +def assert_mv_roundtrip(mv): + """A Metric View dict survives MV -> Apache Ossie -> MV with content preserved.""" + ossie_yaml = _convert(importer.convert_metric_view_to_ossie, dump_yaml(mv)) + mv2 = load_yaml(_convert(exporter.convert_ossie_to_metric_view, ossie_yaml)) + + assert mv2["source"] == mv["source"], "source" + assert mv2.get("comment") == mv.get("comment"), "comment" + assert mv2.get("filter") == mv.get("filter"), "filter" + assert mv2.get("materialization") == mv.get("materialization"), "materialization" + + assert ({d["name"]: _dim_norm(d) for d in _dims(mv)} + == {d["name"]: _dim_norm(d) for d in _dims(mv2)}), "fields" + assert ({m["name"]: _meas_norm(m) for m in mv.get("measures", [])} + == {m["name"]: _meas_norm(m) for m in mv2.get("measures", [])}), "measures" + + a1, e1 = _flatten_joins(mv.get("joins")) + a2, e2 = _flatten_joins(mv2.get("joins")) + assert a1 == a2, "joins" + assert e1 == e2, "join nesting" + + +def _expr_of(obj): + for d in obj["expression"]["dialects"]: + if d["dialect"] == "DATABRICKS": + return d["expression"] + return None + + +def _fields_map(ds): + return {f["name"]: _expr_of(f) for f in ds.get("fields", [])} + + +def _rel_set(model): + return {(r["from"], r["to"], tuple(r.get("from_columns") or []), + tuple(r.get("to_columns") or [])) + for r in model.get("relationships", [])} + + +def assert_ossie_roundtrip(ossie): + """An Apache Ossie model dict survives Apache Ossie -> MV -> Apache Ossie with content preserved.""" + mv_yaml = _convert(exporter.convert_ossie_to_metric_view, dump_yaml(ossie)) + ossie2 = load_yaml(_convert(importer.convert_metric_view_to_ossie, mv_yaml)) + + m1, m2 = ossie["semantic_model"][0], ossie2["semantic_model"][0] + assert ({d["name"]: (d["source"], _fields_map(d)) for d in m1["datasets"]} + == {d["name"]: (d["source"], _fields_map(d)) for d in m2["datasets"]}), "datasets" + assert _rel_set(m1) == _rel_set(m2), "relationships" + assert ({x["name"]: _expr_of(x) for x in m1.get("metrics", [])} + == {x["name"]: _expr_of(x) for x in m2.get("metrics", [])}), "metrics" + assert m1.get("description") == m2.get("description"), "description" diff --git a/converters/databricks/tests/_util.py b/converters/databricks/tests/_util.py new file mode 100644 index 0000000..e82a1d0 --- /dev/null +++ b/converters/databricks/tests/_util.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared test helpers: fixture loading and structural normalization.""" + +import copy +import json +import pathlib + +from ossie_databricks._common import load_yaml # src is on sys.path via conftest.py + +FIXTURES = pathlib.Path(__file__).resolve().parent / "fixtures" + + +def load_fixture(name): + with open(FIXTURES / name) as fh: + return fh.read() + + +def parse(yaml_str): + # YAML 1.2 booleans so a join `on:` key parses as the string "on" (matching the + # converter's own load/dump), not the YAML-1.1 boolean True. + return load_yaml(yaml_str) + + +def canon(obj): + """Deep-copy with every `custom_extensions[].data` JSON string parsed into a + dict, so comparisons are insensitive to JSON key order / whitespace.""" + obj = copy.deepcopy(obj) + + def walk(node): + if isinstance(node, dict): + for ext in node.get("custom_extensions") or []: + if isinstance(ext.get("data"), str): + ext["data"] = json.loads(ext["data"]) + for v in node.values(): + walk(v) + elif isinstance(node, list): + for v in node: + walk(v) + + walk(obj) + return obj + + +def strip_dropped(ossie): + """Normalize away what the Apache Ossie -> MV -> Apache Ossie trip changes, so a round-trip + comparison reflects the documented limitations. Besides outright losses (model + name, descriptions), a declared key transforms across the trip: `primary_key` -> + `rely.at_most_one_match` (MV) -> `unique_keys` + a relationship rely-stash. We drop + both key forms and the relationship stash so the key info is compared as 'gone'.""" + ossie = copy.deepcopy(ossie) + for model in ossie.get("semantic_model", []): + model.pop("name", None) # MV carries no model name + model.pop("description", None) # model + fact descriptions merge into one comment + for ds in model.get("datasets", []): + ds.pop("primary_key", None) + ds.pop("unique_keys", None) + ds.pop("description", None) # no per-source comment in single-source MV + for rel in model.get("relationships", []): + rel.pop("custom_extensions", None) # derived rely-stash from a declared key + return ossie diff --git a/converters/databricks/tests/conftest.py b/converters/databricks/tests/conftest.py new file mode 100644 index 0000000..0bcde53 --- /dev/null +++ b/converters/databricks/tests/conftest.py @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pathlib +import sys + +# Make the converter modules in ../src importable from the tests. +_SRC = pathlib.Path(__file__).resolve().parent.parent / "src" +sys.path.insert(0, str(_SRC)) diff --git a/converters/databricks/tests/fixtures/fixtureA_metric_view.yaml b/converters/databricks/tests/fixtures/fixtureA_metric_view.yaml new file mode 100644 index 0000000..4ffe681 --- /dev/null +++ b/converters/databricks/tests/fixtures/fixtureA_metric_view.yaml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Fixture A -- expected UC Metric View (v1.1, single-source) produced from +# fixtureA_ossie.yaml. Must parse under the v1.1 strict schema. + +version: '1.1' +source: samples.tpch.orders +comment: Sales orders with customer attributes +joins: +- name: customer + source: samples.tpch.customer + on: source.o_custkey = customer.c_custkey + rely: + at_most_one_match: true +dimensions: +- name: o_orderkey + expr: o_orderkey + comment: Order identifier +- name: o_orderdate + expr: o_orderdate + display_name: Order Date + synonyms: + - order date + - date +- name: c_name + expr: customer.c_name + comment: Customer name +measures: +- name: total_revenue + expr: SUM(o_totalprice) + comment: Total order revenue + synonyms: + - revenue + - total revenue + - sales +- name: order_count + expr: COUNT(*) + comment: Number of orders diff --git a/converters/databricks/tests/fixtures/fixtureA_ossie.yaml b/converters/databricks/tests/fixtures/fixtureA_ossie.yaml new file mode 100644 index 0000000..a53942f --- /dev/null +++ b/converters/databricks/tests/fixtures/fixtureA_ossie.yaml @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yaml-language-server: $schema=../../../../core-spec/osi-schema.json +# +# Fixture A -- all-native round-trip (Apache Ossie -> MV -> Apache Ossie). +# Star schema; every field maps to a native MV field. Documented losses on the +# Apache Ossie -> MV -> Apache Ossie trip: the model name (MV carries none) and primary_key. + +version: "0.2.0.dev0" + +semantic_model: + - name: sales + description: Sales orders with customer attributes + datasets: + - name: orders # fact: no incoming relationship -> becomes `source` + source: samples.tpch.orders + primary_key: [o_orderkey] # dropped on export (Apache Ossie-only) + description: One row per order + fields: + - name: o_orderkey + expression: + dialects: + - dialect: DATABRICKS + expression: o_orderkey + description: Order identifier + - name: o_orderdate + expression: + dialects: + - dialect: DATABRICKS + expression: o_orderdate + label: Order Date + ai_context: + synonyms: [order date, date] + - name: customer + source: samples.tpch.customer + primary_key: [c_custkey] + fields: + - name: c_name + expression: + dialects: + - dialect: DATABRICKS + expression: c_name + description: Customer name + relationships: + - name: orders_to_customer + from: orders + to: customer + from_columns: [o_custkey] + to_columns: [c_custkey] + metrics: + - name: total_revenue + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(o_totalprice) # fact columns are bare in measures + description: Total order revenue + ai_context: + synonyms: [revenue, total revenue, sales] + - name: order_count + expression: + dialects: + - dialect: DATABRICKS + expression: COUNT(*) + description: Number of orders diff --git a/converters/databricks/tests/fixtures/fixtureB_metric_view.yaml b/converters/databricks/tests/fixtures/fixtureB_metric_view.yaml new file mode 100644 index 0000000..bbbbabc --- /dev/null +++ b/converters/databricks/tests/fixtures/fixtureB_metric_view.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Fixture B -- stash round-trip (MV -> Apache Ossie -> MV, lossless). +# Exercises the stash at every placement level: model (filter), relationship +# (rely), dimension (format), measure (format). Must parse under v1.1. + +version: '1.1' +source: samples.tpch.lineitem +filter: l_returnflag = 'N' +comment: Line item shipping metrics +joins: +- name: orders + source: samples.tpch.orders + on: source.l_orderkey = orders.o_orderkey + rely: + at_most_one_match: true +dimensions: +- name: line_number + expr: l_linenumber + format: + type: number + decimal_places: + type: exact + places: 0 +measures: +- name: revenue + expr: SUM(l_extendedprice * (1 - l_discount)) + comment: Net revenue + format: + type: currency + currency_code: USD + decimal_places: + type: exact + places: 2 +- name: order_count + expr: COUNT(DISTINCT l_orderkey) diff --git a/converters/databricks/tests/fixtures/fixtureB_ossie.yaml b/converters/databricks/tests/fixtures/fixtureB_ossie.yaml new file mode 100644 index 0000000..f2c11d9 --- /dev/null +++ b/converters/databricks/tests/fixtures/fixtureB_ossie.yaml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yaml-language-server: $schema=../../../../core-spec/osi-schema.json +# +# Fixture B -- expected Apache Ossie produced from fixtureB_metric_view.yaml. MV-only +# features are stashed in custom_extensions[DATABRICKS], keyed by their exact v1.1 +# field name. Exporting this back must reproduce fixtureB_metric_view.yaml. +# Model name is derived from the fact table (`lineitem`). + +version: "0.2.0.dev0" + +semantic_model: + - name: lineitem + description: Line item shipping metrics + datasets: + - name: lineitem + source: samples.tpch.lineitem + fields: + - name: line_number + expression: + dialects: + - dialect: DATABRICKS + expression: l_linenumber + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "format": {"type": "number", "decimal_places": {"type": "exact", "places": 0}}}' + - name: orders + source: samples.tpch.orders + unique_keys: + - [o_orderkey] + relationships: + - name: lineitem_to_orders + from: lineitem + to: orders + from_columns: [l_orderkey] + to_columns: [o_orderkey] + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "rely": {"at_most_one_match": true}}' + metrics: + - name: revenue + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(l_extendedprice * (1 - l_discount)) + description: Net revenue + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "format": {"type": "currency", "currency_code": "USD", "decimal_places": {"type": "exact", "places": 2}}}' + - name: order_count + expression: + dialects: + - dialect: DATABRICKS + expression: COUNT(DISTINCT l_orderkey) + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "filter": "l_returnflag = ''N''"}' diff --git a/converters/databricks/tests/fixtures/tpcds_metric_view.yaml b/converters/databricks/tests/fixtures/tpcds_metric_view.yaml new file mode 100644 index 0000000..b22fc69 --- /dev/null +++ b/converters/databricks/tests/fixtures/tpcds_metric_view.yaml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: '1.1' +source: tpcds.public.store_sales +comment: Store sales enriched with date, item, and customer dimensions +filter: ss_net_profit > 0 +joins: +- name: date_dim + source: tpcds.public.date_dim + on: source.ss_sold_date_sk = date_dim.d_date_sk + rely: + at_most_one_match: true +- name: item + source: tpcds.public.item + on: source.ss_item_sk = item.i_item_sk + rely: + at_most_one_match: true +- name: customer + source: tpcds.public.customer + on: source.ss_customer_sk = customer.c_customer_sk + rely: + at_most_one_match: true +dimensions: +- name: ticket_number + expr: ss_ticket_number +- name: sold_year + expr: date_dim.d_year + display_name: Year + synonyms: + - year + - yr +- name: sold_date + expr: date_dim.d_date +- name: item_category + expr: item.i_category + synonyms: + - category + - product type +- name: item_brand + expr: item.i_brand +- name: birth_country + expr: customer.c_birth_country +measures: +- name: total_sales + expr: SUM(ss_ext_sales_price) + comment: Total sales revenue + format: + type: currency + currency_code: USD +- name: total_quantity + expr: SUM(ss_quantity) + comment: Total units sold diff --git a/converters/databricks/tests/fixtures/tpcds_ossie.yaml b/converters/databricks/tests/fixtures/tpcds_ossie.yaml new file mode 100644 index 0000000..e055eab --- /dev/null +++ b/converters/databricks/tests/fixtures/tpcds_ossie.yaml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: "0.2.0.dev0" +semantic_model: + - name: tpcds_store_sales + description: Store sales enriched with date, item, and customer dimensions + datasets: + - name: store_sales + source: tpcds.public.store_sales + fields: + - name: ticket_number + expression: + dialects: [{dialect: DATABRICKS, expression: ss_ticket_number}] + - name: date_dim + source: tpcds.public.date_dim + primary_key: [d_date_sk] + fields: + - name: sold_year + expression: + dialects: [{dialect: DATABRICKS, expression: d_year}] + label: Year + ai_context: {synonyms: [year, yr]} + - name: sold_date + expression: + dialects: [{dialect: DATABRICKS, expression: d_date}] + - name: item + source: tpcds.public.item + primary_key: [i_item_sk] + fields: + - name: item_category + expression: + dialects: [{dialect: DATABRICKS, expression: i_category}] + ai_context: {synonyms: [category, product type]} + - name: item_brand + expression: + dialects: [{dialect: DATABRICKS, expression: i_brand}] + - name: customer + source: tpcds.public.customer + primary_key: [c_customer_sk] + fields: + - name: birth_country + expression: + dialects: [{dialect: DATABRICKS, expression: c_birth_country}] + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: [ss_sold_date_sk] + to_columns: [d_date_sk] + - name: store_sales_to_item + from: store_sales + to: item + from_columns: [ss_item_sk] + to_columns: [i_item_sk] + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: [ss_customer_sk] + to_columns: [c_customer_sk] + metrics: + - name: total_sales + expression: + dialects: [{dialect: DATABRICKS, expression: SUM(ss_ext_sales_price)}] + description: Total sales revenue + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "format": {"type": "currency", "currency_code": "USD"}}' + - name: total_quantity + expression: + dialects: [{dialect: DATABRICKS, expression: SUM(ss_quantity)}] + description: Total units sold + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "filter": "ss_net_profit > 0"}' diff --git a/converters/databricks/tests/test_metric_view_to_ossie.py b/converters/databricks/tests/test_metric_view_to_ossie.py new file mode 100644 index 0000000..0289003 --- /dev/null +++ b/converters/databricks/tests/test_metric_view_to_ossie.py @@ -0,0 +1,347 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the Databricks Metric View -> Apache Ossie importer.""" + +import pytest + +from ossie_databricks import ConversionError +from ossie_databricks import metric_view_to_ossie as importer +from _util import canon, load_fixture, parse + + +def test_fixtureB_import_matches_expected(): + out = importer.convert_metric_view_to_ossie(load_fixture("fixtureB_metric_view.yaml")) + assert canon(parse(out)) == canon(parse(load_fixture("fixtureB_ossie.yaml"))) + + +def test_fields_is_accepted_as_alias_for_dimensions(): + """`fields:` is a v1.1 alias for `dimensions:` (the form the DBR docs use); the + importer must read it, not silently drop the columns.""" + mv = ( + "version: '1.1'\nsource: c.s.orders\n" + "fields:\n- {name: region, expr: region}\n" + ) + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + fields = ossie["semantic_model"][0]["datasets"][0].get("fields", []) + assert [f["name"] for f in fields] == ["region"] + + +def test_unsupported_version_rejected(): + with pytest.raises(ConversionError): + importer.convert_metric_view_to_ossie("version: '0.1'\nsource: c.s.t\n") + + +def test_both_dimensions_and_fields_present_warns_and_uses_dimensions(): + """`fields` is a v1.1 alias for `dimensions`; if a (malformed) view sets both, the + importer uses `dimensions` and warns that the `fields` list is ignored.""" + import warnings + mv = ( + "version: '1.1'\nsource: c.s.orders\n" + "dimensions:\n- {name: kept, expr: kept}\n" + "fields:\n- {name: ignored, expr: ignored}\n" + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + names = [f["name"] for f in ossie["semantic_model"][0]["datasets"][0].get("fields", [])] + assert names == ["kept"] + assert any("fields" in str(w.message) and "ignored" in str(w.message) for w in caught) + + +def test_stash_written_at_each_level(): + ossie = parse(importer.convert_metric_view_to_ossie(load_fixture("fixtureB_metric_view.yaml"))) + model = ossie["semantic_model"][0] + + # model-level filter + assert any(e["vendor_name"] == "DATABRICKS" and "filter" in e["data"] + for e in model["custom_extensions"]) + # relationship-level rely + rel = model["relationships"][0] + assert any("rely" in e["data"] for e in rel["custom_extensions"]) + # metric-level format + revenue = next(m for m in model["metrics"] if m["name"] == "revenue") + assert any("format" in e["data"] for e in revenue["custom_extensions"]) + + +def test_name_override(): + ossie = parse(importer.convert_metric_view_to_ossie( + load_fixture("fixtureB_metric_view.yaml"), model_name="custom")) + assert ossie["semantic_model"][0]["name"] == "custom" + + +def test_cross_join_rejected(): + mv = "version: '1.1'\nsource: c.s.fact\njoins:\n- name: dim\n source: c.s.dim\n" + with pytest.raises(ConversionError, match="cross"): + importer.convert_metric_view_to_ossie(mv) + + +def test_duplicate_join_name_rejected(): + # a join named like the fact (derived from the source's last identifier) collides + mv = "version: '1.1'\nsource: c.s.fact\njoins:\n- name: fact\n source: c.s.other\n using: [id]\n" + with pytest.raises(ConversionError, match="Duplicate"): + importer.convert_metric_view_to_ossie(mv) + + +def test_complex_joined_dimension_filed_under_join_dataset(): + mv = ( + "version: '1.1'\nsource: c.s.fact\n" + "joins:\n- name: cust\n source: c.s.cust\n on: source.cid = cust.id\n" + "dimensions:\n- name: full\n expr: cust.a || cust.b\n" + ) + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + cust = next(d for d in ossie["semantic_model"][0]["datasets"] if d["name"] == "cust") + assert any(f["name"] == "full" for f in cust.get("fields", [])) + + +def test_non_equi_on_rejected(): + """Apache Ossie relationships are equi-joins (from_columns/to_columns required, minItems 1), so a + non-equi `on` has no Apache Ossie representation and is rejected on import (rather than emitting + a relationship with empty column lists, which is invalid per the Apache Ossie schema).""" + mv = ( + "version: '1.1'\n" + "source: c.s.fact\n" + "joins:\n" + "- name: dim\n" + " source: c.s.dim\n" + " on: source.a >= dim.b\n" + ) + with pytest.raises(ConversionError, match="non-equi"): + importer.convert_metric_view_to_ossie(mv) + + +def test_complex_equi_on_rejected(): + """An equi `on` whose operand is a SQL fragment (OR, computed) can't be decomposed + into from/to columns, so it's rejected rather than producing schema-invalid Apache Ossie with + empty column lists.""" + for cond in ("source.a = dim.b OR source.c = dim.d", "source.a = dim.b + 1"): + mv = ( + "version: '1.1'\nsource: c.s.fact\n" + f"joins:\n- name: dim\n source: c.s.dim\n on: {cond}\n" + ) + with pytest.raises(ConversionError, match="non-equi"): + importer.convert_metric_view_to_ossie(mv) + + +def test_one_to_many_join_flips_from_to_and_stashes_source(): + """A one_to_many MV join becomes an Apache Ossie relationship with the MANY side as `from` + (the joined table), the source/grain on the `to` side, and the grain recorded in + the model-level stash so re-export re-roots correctly.""" + mv = ( + "version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: line_items\n source: c.s.line_items\n" + " on: source.order_id = line_items.l_order_id\n" + " cardinality: one_to_many\n" + "measures:\n- {name: order_count, expr: COUNT(*)}\n" + ) + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + model = ossie["semantic_model"][0] + rel = model["relationships"][0] + assert rel["from"] == "line_items" # many side (holds the FK) + assert rel["to"] == "orders" # one side (holds the PK) + assert rel["from_columns"] == ["l_order_id"] + assert rel["to_columns"] == ["order_id"] + assert any(e["vendor_name"] == "DATABRICKS" and "source_dataset" in e["data"] + for e in model["custom_extensions"]) + + +def test_at_most_one_match_recovers_unique_key(): + """A many_to_one join with rely.at_most_one_match records the join key as a + unique_keys entry on the joined dataset (recovering key info Apache Ossie would lack).""" + mv = ( + "version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: customer\n source: c.s.customer\n" + " on: source.cid = customer.id\n" + " rely: {at_most_one_match: true}\n" + ) + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + cust = next(d for d in ossie["semantic_model"][0]["datasets"] if d["name"] == "customer") + assert cust.get("unique_keys") == [["id"]] + + +def test_join_named_source_rejected(): + """`source` is reserved for the fact; a join named `source` is rejected (DBR + forbids it too) rather than silently overwriting the fact alias.""" + mv = ("version: '1.1'\nsource: c.s.fact\n" + "joins:\n- name: source\n source: c.s.dim\n using: [id]\n") + with pytest.raises(ConversionError, match="reserved"): + importer.convert_metric_view_to_ossie(mv) + + +def test_sql_source_name_defaults_to_metric_view(): + """A SELECT/WITH source has no table name, so the model name defaults to + `metric_view` (not a token sliced out of the SQL).""" + mv = "version: '1.1'\nsource: SELECT a, b FROM main.sales.orders\n" + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + assert ossie["semantic_model"][0]["name"] == "metric_view" + + +def test_join_missing_source_raises(): + """Missing required keys surface as ConversionError, not a raw KeyError.""" + mv = "version: '1.1'\nsource: c.s.fact\njoins:\n- name: dim\n using: [id]\n" + with pytest.raises(ConversionError, match="missing required 'source'"): + importer.convert_metric_view_to_ossie(mv) + + +def test_measure_rewrite_with_regex_special_name(): + r"""The `source.` -> fact-name rewrite in measures inserts the name literally, so a + --name containing regex backreference syntax (e.g. \1) does not raise a re.error.""" + mv = "version: '1.1'\nsource: c.s.fact\nmeasures:\n- {name: rev, expr: SUM(source.amount)}\n" + ossie = parse(importer.convert_metric_view_to_ossie(mv, model_name=r"a\1b")) + expr = ossie["semantic_model"][0]["metrics"][0]["expression"]["dialects"][0]["expression"] + assert expr == r"SUM(a\1b.amount)" + + +def test_invalid_source_rejected(): + """A malformed source (not 3-part / not SELECT) is rejected on import, matching the + exporter -- rather than passing through and only failing on a later re-export.""" + mv = "version: '1.1'\nsource: a.b\n" # 2-part, invalid + with pytest.raises(ConversionError, match="3-part"): + importer.convert_metric_view_to_ossie(mv) + + +def test_malformed_yaml_raises_conversion_error(): + """Invalid YAML surfaces as ConversionError, not a raw yaml.YAMLError traceback.""" + with pytest.raises(ConversionError, match="Invalid YAML"): + importer.convert_metric_view_to_ossie("source: c.s.t\njoins: [oops\n") + + +def test_empty_using_rejected(): + """An empty `using: []` is a condition-less join -- rejected at import rather than + silently producing a relationship with empty key columns (which would then fail on + re-export).""" + mv = "version: '1.1'\nsource: c.s.fact\njoins:\n- name: dim\n source: c.s.dim\n using: []\n" + with pytest.raises(ConversionError, match="cross"): + importer.convert_metric_view_to_ossie(mv) + + +def test_boollike_string_values_stay_strings_for_a_yaml_1_1_reader(): + """Bool-like string scalars (e.g. the synonyms `on`/`off`) must be emitted quoted so a + stock YAML 1.1 reader reads them back as strings, not booleans.""" + import yaml + mv = ("version: '1.1'\nsource: c.s.t\n" + "dimensions:\n- {name: status, expr: status, synonyms: [on, off]}\n") + ossie_out = importer.convert_metric_view_to_ossie(mv) + field = yaml.safe_load(ossie_out)["semantic_model"][0]["datasets"][0]["fields"][0] + assert field["ai_context"]["synonyms"] == ["on", "off"] + + +def test_fact_qualifier_variants_in_on_decompose(): + """The fact side of an `on` is valid MV YAML whether qualified with `source`, the + source table name, or left bare; all decompose to the same equi-join columns rather + than being wrongly rejected as a non-equi condition (bug-bash finding).""" + base = ("version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: customer\n source: c.s.customer\n on: {cond}\n") + for cond in ( + "source.o_custkey = customer.c_custkey", # `source` qualifier + "orders.o_custkey = customer.c_custkey", # source table name + "o_custkey = customer.c_custkey", # bare fact column + "customer.c_custkey = o_custkey", # reversed operand order, bare fact + ): + rel = parse(importer.convert_metric_view_to_ossie( + base.format(cond=cond)))["semantic_model"][0]["relationships"][0] + assert rel["from"] == "orders" and rel["to"] == "customer" + assert rel["from_columns"] == ["o_custkey"] + assert rel["to_columns"] == ["c_custkey"] + + +def test_multi_column_on_with_bare_and_tablename_fact(): + """Composite keys decompose with bare / source-table-name fact qualifiers too.""" + mv = ("version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: customer\n source: c.s.customer\n" + " on: o_a = customer.c_a AND orders.o_b = customer.c_b\n") + rel = parse(importer.convert_metric_view_to_ossie(mv))["semantic_model"][0]["relationships"][0] + assert rel["from_columns"] == ["o_a", "o_b"] + assert rel["to_columns"] == ["c_a", "c_b"] + + +def test_join_named_source_rejected_any_case(): + """`source` is reserved case-insensitively (DBR identifiers are case-insensitive), + so `Source`/`SOURCE` are rejected too (bug-bash finding).""" + for name in ("Source", "SOURCE", "SoUrCe"): + mv = (f"version: '1.1'\nsource: c.s.fact\n" + f"joins:\n- name: {name}\n source: c.s.dim\n using: [id]\n") + with pytest.raises(ConversionError, match="reserved"): + importer.convert_metric_view_to_ossie(mv) + + +def test_empty_source_part_rejected(): + """A 3-dot source with an empty part (`.s.t`, `c..t`, `c.s.`) is not a valid 3-part + identifier and is rejected -- the dot count alone is not enough (bug-bash finding).""" + for src in (".s.t", "c..t", "c.s."): + mv = f"version: '1.1'\nsource: {src}\n" + with pytest.raises(ConversionError, match="3-part"): + importer.convert_metric_view_to_ossie(mv) + + +def test_whitespace_source_part_rejected(): + """A 3-dot source with a whitespace-laden part (`cat . sch . tbl`) is rejected -- + a space is not part of a valid identifier (review finding).""" + with pytest.raises(ConversionError, match="3-part"): + importer.convert_metric_view_to_ossie("version: '1.1'\nsource: cat . sch . tbl\n") + + +def test_with_paren_subquery_source_accepted(): + """A `WITH(...)` subquery with no space after the keyword is recognized as SQL, + not mistaken for a (non-3-part) identifier (review finding).""" + mv = "version: '1.1'\nsource: WITH(t AS (SELECT 1 AS a)) SELECT a FROM t\n" + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + assert ossie["semantic_model"][0]["datasets"][0]["source"].startswith("WITH(") + + +def test_nested_join_bare_column_rejected(): + """A bare (unqualified) operand in a NESTED join's `on` is ambiguous (parent vs. + fact), so it is rejected rather than silently attributed to the immediate parent. + A bare fact column is still accepted at the top level (review finding).""" + mv = ("version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: customer\n source: c.s.customer\n on: source.ckey = customer.c_key\n" + " joins:\n - name: nation\n source: c.s.nation\n on: o_nkey = nation.n_key\n") + with pytest.raises(ConversionError, match="non-equi or unsupported"): + importer.convert_metric_view_to_ossie(mv) + + +def test_case_variant_duplicate_name_rejected(): + """DBR identifiers are case-insensitive, so `dim`/`Dim` collide and must be rejected + (consistent with the case-insensitive reserved-`source` check) (review finding).""" + mv = ("version: '1.1'\nsource: c.s.x\n" + "joins:\n- {name: dim, source: c.s.a, using: [id]}\n- {name: Dim, source: c.s.b, using: [id]}\n") + with pytest.raises(ConversionError, match="[Dd]uplicate"): + importer.convert_metric_view_to_ossie(mv) + + +def test_falsy_dimension_name_is_not_a_wildcard(): + """A present-but-falsy name (`0`) is a malformed column, not a wildcard projection; + it raises a clean error rather than being silently dropped (review finding).""" + with pytest.raises(ConversionError): + importer.convert_metric_view_to_ossie("version: '1.1'\nsource: c.s.o\ndimensions:\n- {name: 0, expr: x}\n") + + +def test_non_string_scalars_raise_clean_error(): + """Non-string scalars where a string is required (join name, measure expr) raise a + ConversionError, not a raw AttributeError/TypeError (review finding).""" + for mv in ("version: '1.1'\nsource: c.s.f\njoins:\n- {name: 5, source: c.s.d, using: [id]}\n", + "version: '1.1'\nsource: c.s.o\nmeasures:\n- {name: rev, expr: 5}\n"): + with pytest.raises(ConversionError): + importer.convert_metric_view_to_ossie(mv) + + +def test_using_join_emits_no_yaml_anchor(): + """`from_columns`/`to_columns` of a `using` join are distinct list objects, so the + output has no YAML anchor/alias (`&id`/`*id`) -- safer for other Apache Ossie consumers.""" + out = importer.convert_metric_view_to_ossie( + "version: '1.1'\nsource: c.s.a\njoins:\n- {name: b, source: c.s.b, using: [id]}\n") + assert "&id" not in out and "*id" not in out diff --git a/converters/databricks/tests/test_ossie_to_metric_view.py b/converters/databricks/tests/test_ossie_to_metric_view.py new file mode 100644 index 0000000..8b8808b --- /dev/null +++ b/converters/databricks/tests/test_ossie_to_metric_view.py @@ -0,0 +1,670 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the Apache Ossie -> Databricks Metric View exporter.""" + +import json + +import pytest + +from ossie_databricks import ConversionError +from ossie_databricks import ossie_to_metric_view as exporter +from _util import canon, load_fixture, parse + + +def test_fixtureA_export_matches_expected(): + out = exporter.convert_ossie_to_metric_view(load_fixture("fixtureA_ossie.yaml")) + assert parse(out) == parse(load_fixture("fixtureA_metric_view.yaml")) + + +def test_tpcds_export_matches_expected(): + """A normalized TPC-DS star (fact + date/item/customer dims) exports to the expected + Metric View: a join tree, primary keys bridged to rely.at_most_one_match, joined + columns alias-qualified, and the filter/format stash carried through.""" + out = exporter.convert_ossie_to_metric_view(load_fixture("tpcds_ossie.yaml")) + assert parse(out) == parse(load_fixture("tpcds_metric_view.yaml")) + + +def test_unsupported_version_rejected(): + ossie = "version: '9.9.9'\nsemantic_model:\n - name: m\n datasets:\n - {name: d, source: c.s.t}\n" + with pytest.raises(ConversionError): + exporter.convert_ossie_to_metric_view(ossie) + + +def _model(rels): + return { + "version": exporter.OSSIE_VERSION, + "semantic_model": [ + { + "name": "m", + "datasets": [ + {"name": "a", "source": "c.s.a"}, + {"name": "b", "source": "c.s.b"}, + {"name": "x", "source": "c.s.x"}, + ], + "relationships": rels, + } + ], + } + + +def _rel(name, frm, to): + return {"name": name, "from": frm, "to": to, + "from_columns": ["k"], "to_columns": ["k"]} + + +def test_multiple_roots_raises(): + import yaml + # a->b leaves x as a second root. + ossie = yaml.safe_dump(_model([_rel("r1", "a", "b")])) + with pytest.raises(ConversionError, match="multiple candidate fact"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_triangle_is_rejected_as_cycle(): + import yaml + # a->b, a->x, b->x : b and x are equidistant from a (a triangle) -> not a tree. + ossie = yaml.safe_dump(_model([_rel("r1", "a", "b"), _rel("r2", "a", "x"), + _rel("r3", "b", "x")])) + with pytest.raises(ConversionError, match="cycle"): + exporter.convert_ossie_to_metric_view(ossie) + + +def _field(name, col): + return {"name": name, "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": col}]}} + + +def test_mto_diamond_fans_out(): + """A shared dimension reached by two parents (orders->customers->regions and + orders->suppliers->regions) is fanned out into two aliased joins, not rejected.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "fields": [_field("amt", "amount")]}, + {"name": "customers", "source": "c.s.customers"}, + {"name": "suppliers", "source": "c.s.suppliers"}, + {"name": "regions", "source": "c.s.regions", "fields": [_field("rname", "r_name")]}, + ], + "relationships": [ + {"name": "r1", "from": "orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r2", "from": "orders", "to": "suppliers", "from_columns": ["sid"], "to_columns": ["id"]}, + {"name": "r3", "from": "customers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + {"name": "r4", "from": "suppliers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + ], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + region_joins = [j for top in out["joins"] for j in top.get("joins", []) if j["source"] == "c.s.regions"] + assert {j["name"] for j in region_joins} == {"customers_regions", "suppliers_regions"} + dims = {d["name"]: d["expr"] for d in out["dimensions"]} + # the fanned `regions` is a depth-2 join, so its column is qualified by the full path + assert dims["customers_regions_rname"] == "customers.customers_regions.r_name" + assert dims["suppliers_regions_rname"] == "suppliers.suppliers_regions.r_name" + + +def test_otm_diamond_fans_out(): + """customers (fact) -> past_orders/future_orders -> line_items: the shared + line_items is fanned out, and every join is one_to_many.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "customers", "source": "c.s.customers"}, + {"name": "past_orders", "source": "c.s.past_orders"}, + {"name": "future_orders", "source": "c.s.future_orders"}, + {"name": "line_items", "source": "c.s.line_items"}, + ], + "relationships": [ + {"name": "r1", "from": "past_orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r2", "from": "future_orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r3", "from": "line_items", "to": "past_orders", "from_columns": ["oid"], "to_columns": ["id"]}, + {"name": "r4", "from": "line_items", "to": "future_orders", "from_columns": ["oid"], "to_columns": ["id"]}, + ], + "metrics": [{"name": "cnt", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie, source="customers")) + leaf_names = {j["name"] for top in out["joins"] for j in top.get("joins", [])} + assert leaf_names == {"past_orders_line_items", "future_orders_line_items"} + + def all_otm(joins): + return all(j.get("cardinality") == "one_to_many" and all_otm(j.get("joins", [])) + for j in joins) + assert all_otm(out["joins"]) + + +def test_cycle_raises(): + import yaml + # a->b->x->a : cycle, and no root. + ossie = yaml.safe_dump(_model([_rel("r1", "a", "b"), _rel("r2", "b", "x"), + _rel("r3", "x", "a")])) + with pytest.raises(ConversionError, match="cycle"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_no_unknown_keys_leak(): + """Exporter output must contain no key outside the v1.1 schema (the strict-parse + guard: no `custom_extensions`, no `sql_on`).""" + out = exporter.convert_ossie_to_metric_view(load_fixture("fixtureA_ossie.yaml")) + assert "custom_extensions" not in out + assert "sql_on" not in out + + +def test_primary_key_is_dropped(): + out = parse(exporter.convert_ossie_to_metric_view(load_fixture("fixtureA_ossie.yaml"))) + assert "primary_key" not in json.dumps(out) + + +def _single_fact_model(metric_expr): + import yaml + return yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [{"name": "orders", "source": "c.s.orders", + "fields": [{"name": "k", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "k"}]}}]}], + "metrics": [{"name": "rev", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": metric_expr}]}}], + }], + }) + + +def test_measure_strips_fact_prefix(): + """Fact columns are bare in measure expressions (DBR idiom), not `source.`-qualified.""" + out = parse(exporter.convert_ossie_to_metric_view(_single_fact_model("SUM(orders.amount)"))) + assert out["measures"][0]["expr"] == "SUM(amount)" + + +def test_measure_keeps_lookalike_table_prefix(): + """A table whose name merely ends with the fact name is not stripped.""" + out = parse(exporter.convert_ossie_to_metric_view(_single_fact_model("SUM(store_orders.amount)"))) + assert out["measures"][0]["expr"] == "SUM(store_orders.amount)" + + +def test_invalid_source_rejected(): + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{"name": "m", "datasets": [{"name": "d", "source": "justatable"}]}], + }) + with pytest.raises(ConversionError, match="source"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_duplicate_dimension_name_rejected(): + # Two datasets each contributing a field named `id` collide in the single flat + # `dimensions` list. Metric Views require unique dimension/measure names, so this + # is rejected (rather than emitting a duplicate the user would hand to Databricks). + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", + "fields": [{"name": "id", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}]}, + {"name": "customer", "source": "c.s.customer", + "fields": [{"name": "id", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}]}, + ], + "relationships": [{"name": "r", "from": "orders", "to": "customer", + "from_columns": ["cid"], "to_columns": ["id"]}], + }], + }) + with pytest.raises(ConversionError, match="collides"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_measure_name_collides_with_dimension_rejected(): + # A measure whose name matches an existing dimension (case-insensitively) collides + # in the shared name space; Metric Views require uniqueness, so this is rejected. + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", + "fields": [{"name": "total", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "total"}]}}]}, + ], + "metrics": [ + {"name": "Total", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "SUM(total)"}]}}, + ], + }], + }) + with pytest.raises(ConversionError, match="collides"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_cascade_drop_downstream_measure_reference(): + """A measure that references a dropped measure via measure() is itself dropped + (transitively), so no dangling reference is emitted.""" + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [{"name": "f", "source": "c.s.f"}], + "metrics": [ + {"name": "base", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "SUM(x)"}]}}, # dropped (no DBX/ANSI) + {"name": "derived", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "measure(base) * 2"}]}}, + {"name": "derived2", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "measure(derived) + 1"}]}}, # transitive + {"name": "ok", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}, + ], + }], + }) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + names = [m["name"] for m in out.get("measures", [])] + assert names == ["ok"] # base dropped; derived + derived2 cascade-dropped; ok survives + + +def test_cascade_drop_downstream_dimension_reference(): + """A field/measure referencing a dropped dimension by name is also dropped.""" + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [{"name": "f", "source": "c.s.f", "fields": [ + {"name": "region", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "r"}]}}, # dropped dim + {"name": "label", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "upper(region)"}]}}, # references region + {"name": "keep", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}, + ]}], + }], + }) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + dims = [d["name"] for d in out.get("dimensions", [])] + assert dims == ["keep"] # region dropped; label cascade-dropped; keep survives + + +def test_orientation_unverifiable_when_to_side_has_no_key_warns(): + """If the `from` columns are a declared key but the `to` side declares no key, the + from/to orientation can't be verified; the converter leaves it as-is (no reorient) + and warns, rather than silently producing a possibly-inverted cardinality.""" + import warnings + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "a", "source": "c.s.a", "primary_key": ["a_id"], "fields": [ + {"name": "a_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "a_name"}]}}]}, + {"name": "b", "source": "c.s.b", "fields": [ + {"name": "b_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "b_name"}]}}]}, + ], + # from columns cover a's PK, but b (the `to` side) declares no key + "relationships": [{"name": "a_to_b", "from": "a", "to": "b", + "from_columns": ["a_id"], "to_columns": ["b_x"]}], + }], + }) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exporter.convert_ossie_to_metric_view(ossie) + assert any("orientation can't be verified" in str(w.message) for w in caught) + + +def test_cascade_drop_skips_qualified_join_alias_collision(): + """A dropped field whose name collides with a join alias must NOT cascade-drop a + *qualified* `alias.col` reference. A genuine dimension reference is unqualified; + `region.r_name` points at the join `region`, a different thing than a dropped bare + `region`, so the cascade must leave it (and the joined column) alone.""" + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "fields": [ + # dropped (no DBX/ANSI dialect); its name collides with the `region` join + {"name": "region", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "r"}]}}, + # references the join alias `region`, not the dropped field -> must survive + {"name": "summary", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "region.r_name"}]}}, + ]}, + {"name": "region", "source": "c.s.region", "primary_key": ["r_key"], "fields": [ + {"name": "r_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "r_name"}]}}, + ]}, + ], + "relationships": [{"name": "orr", "from": "orders", "to": "region", + "from_columns": ["o_rkey"], "to_columns": ["r_key"]}], + }], + }) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + dims = [d["name"] for d in out.get("dimensions", [])] + # `region` field dropped; `summary` (refs alias region.r_name) and the joined + # `r_name` (qualified to region.r_name on export) both survive -- no false cascade. + assert "region" not in dims + assert "summary" in dims and "r_name" in dims + + +def _orders_lineitems_ossie(): + """Apache Ossie: line_items (many, FK l_order_id) -> orders (one, PK order_id).""" + import yaml + return yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "sales", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "primary_key": ["order_id"], + "fields": [{"name": "order_date", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "o_order_date"}]}}]}, + {"name": "line_items", "source": "c.s.line_items", + "fields": [{"name": "product_sk", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "l_product_sk"}]}}]}, + ], + "relationships": [{"name": "li_to_order", "from": "line_items", "to": "orders", + "from_columns": ["l_order_id"], "to_columns": ["order_id"]}], + "metrics": [{"name": "order_count", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}], + }], + }) + + +def test_source_on_to_side_derives_one_to_many(): + """Naming the PK/one-side dataset as the source makes its join one_to_many, and + the many-side table's columns drop (a field must resolve to one value/source row).""" + out = parse(exporter.convert_ossie_to_metric_view(_orders_lineitems_ossie(), source="orders")) + assert out["source"] == "c.s.orders" + join = out["joins"][0] + assert join["name"] == "line_items" + assert join["cardinality"] == "one_to_many" + assert join["on"] == "source.order_id = line_items.l_order_id" + assert [d["name"] for d in out.get("dimensions", [])] == ["order_date"] # product_sk dropped + + +def test_default_fact_is_fk_sink_and_many_to_one(): + """Without an explicit source the fact is the FK-sink (line_items) and the join to + orders is the default many_to_one (no explicit cardinality).""" + out = parse(exporter.convert_ossie_to_metric_view(_orders_lineitems_ossie())) + assert out["source"] == "c.s.line_items" + join = out["joins"][0] + assert join["name"] == "orders" + assert "cardinality" not in join + assert join["on"] == "source.l_order_id = orders.order_id" + + +def test_unknown_source_rejected(): + with pytest.raises(ConversionError, match="not a dataset"): + exporter.convert_ossie_to_metric_view(_orders_lineitems_ossie(), source="nope") + + +def test_one_to_many_subtree_must_stay_one_to_many(): + """A many_to_one join descending from a one_to_many join is rejected (DBR rule).""" + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders"}, + {"name": "line_items", "source": "c.s.line_items"}, + {"name": "product", "source": "c.s.product"}, + ], + "relationships": [ + {"name": "li_to_order", "from": "line_items", "to": "orders", # orders->li : OTM + "from_columns": ["l_order_id"], "to_columns": ["order_id"]}, + {"name": "li_to_product", "from": "line_items", "to": "product", # li->product : MTO + "from_columns": ["l_product_sk"], "to_columns": ["p_sk"]}, + ], + }], + }) + with pytest.raises(ConversionError, match="one-to-many"): + exporter.convert_ossie_to_metric_view(ossie, source="orders") + + +def test_primary_key_deduces_at_most_one_match(): + """A many_to_one join whose to_columns cover the target's declared primary_key + gets rely.at_most_one_match; a join to a key-less dataset does not.""" + import yaml + + def model(dim_extra): + dim = {"name": "customer", "source": "c.s.customer"} + dim.update(dim_extra) + return yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [{"name": "orders", "source": "c.s.orders"}, dim], + "relationships": [{"name": "r", "from": "orders", "to": "customer", + "from_columns": ["cid"], "to_columns": ["id"]}], + }]}) + + join = parse(exporter.convert_ossie_to_metric_view(model({"primary_key": ["id"]})))["joins"][0] + assert join.get("rely") == {"at_most_one_match": True} + join2 = parse(exporter.convert_ossie_to_metric_view(model({})))["joins"][0] + assert "rely" not in join2 + + +def test_mislabeled_from_to_reoriented_by_key(): + """When from/to is swapped but the declared keys show the real one-side, the + converter re-orients to the key side (warns) -- so fact selection and cardinality + come out identical to the well-formed model.""" + import warnings + import yaml + + def model(frm, to, from_cols, to_cols): + return yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "primary_key": ["order_id"], + "fields": [_field("amt", "amount")]}, + {"name": "customer", "source": "c.s.customer", "primary_key": ["c_id"], + "fields": [_field("cname", "c_name")]}, + ], + "relationships": [{"name": "r", "from": frm, "to": to, + "from_columns": from_cols, "to_columns": to_cols}], + }]}) + + well = parse(exporter.convert_ossie_to_metric_view( + model("orders", "customer", ["cust_id"], ["c_id"]))) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + swapped = parse(exporter.convert_ossie_to_metric_view( + model("customer", "orders", ["c_id"], ["cust_id"]))) + assert any("mislabeled" in str(w.message) for w in caught) + assert swapped["source"] == "c.s.orders" # fact selection corrected to the FK holder + assert swapped == well # identical to the well-formed model + + +def test_dataset_named_source_is_renamed(): + """A dataset literally named `source` must not collide with the fact's reserved + `source` alias (would otherwise emit an ambiguous join).""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders"}, + {"name": "source", "source": "c.s.dim", "fields": [_field("x", "xcol")]}, + ], + "relationships": [{"name": "r", "from": "orders", "to": "source", + "from_columns": ["sid"], "to_columns": ["id"]}], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + join = out["joins"][0] + assert join["name"] != "source" + assert join["on"] == f"source.sid = {join['name']}.id" + assert out["dimensions"][0]["expr"] == f"{join['name']}.xcol" + + +def test_fanout_alias_collision_deduped(): + """A real dataset whose name equals a synthesized fan-out alias still gets a + distinct alias -- no two joins share a name.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders"}, + {"name": "customers", "source": "c.s.customers"}, + {"name": "suppliers", "source": "c.s.suppliers"}, + {"name": "regions", "source": "c.s.regions"}, + {"name": "customers_regions", "source": "c.s.cr"}, # collides with fan-out alias + ], + "relationships": [ + {"name": "r1", "from": "orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r2", "from": "orders", "to": "suppliers", "from_columns": ["sid"], "to_columns": ["id"]}, + {"name": "r3", "from": "customers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + {"name": "r4", "from": "suppliers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + {"name": "r5", "from": "orders", "to": "customers_regions", "from_columns": ["xid"], "to_columns": ["id"]}, + ], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + names = [] + + def collect(joins): + for j in joins: + names.append(j["name"]) + collect(j.get("joins", [])) + + collect(out["joins"]) + assert len(names) == len(set(names)), names # all join names unique + + +def test_malformed_input_raises_conversion_error(): + """Missing required keys surface as ConversionError, not a raw KeyError traceback.""" + import yaml + bad = yaml.safe_dump({"version": exporter.OSSIE_VERSION, + "semantic_model": [{"name": "m", "datasets": [{"source": "c.s.t"}]}]}) + with pytest.raises(ConversionError, match="missing required 'name'"): + exporter.convert_ossie_to_metric_view(bad) + + +def test_nameless_relationship_with_ai_context_does_not_crash(): + """A relationship may omit `name`; the dropped-ai_context warning must not raise a + raw KeyError when it has ai_context but no name.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "fields": [_field("amt", "amt")]}, + {"name": "customers", "source": "c.s.customers"}, + ], + "relationships": [ + {"from": "orders", "to": "customers", "from_columns": ["cid"], + "to_columns": ["id"], "ai_context": "joins orders to customers"}, + ], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) # must not raise + assert out["joins"][0]["name"] == "customers" + + +def _cfield(name, expr): + return {"name": name, "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": expr}]}} + + +def test_fanout_complex_expr_dropped_not_emitted_ambiguously(): + """On a fanned-out (diamond) dataset, a simple column fans out into one aliased + dimension per instance, but a complex expression -- which cannot be attributed to a + single instance -- is dropped rather than emitted ambiguously.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders"}, + {"name": "customers", "source": "c.s.customers"}, + {"name": "suppliers", "source": "c.s.suppliers"}, + {"name": "regions", "source": "c.s.regions", + "fields": [_cfield("r_name", "r_name"), _cfield("rfull", "r_a || r_b")]}, + ], + "relationships": [ + {"name": "r1", "from": "orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r2", "from": "orders", "to": "suppliers", "from_columns": ["sid"], "to_columns": ["id"]}, + {"name": "r3", "from": "customers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + {"name": "r4", "from": "suppliers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + ], + }]}) + dims = parse(exporter.convert_ossie_to_metric_view(ossie)).get("dimensions", []) + # the simple column fans out into two unambiguous, alias-qualified dimensions ... + assert sum(1 for d in dims if d["name"].endswith("_r_name")) == 2 + # ... while the ambiguous complex expression is dropped (never emitted unqualified) + assert not any("||" in d["expr"] for d in dims) + + +def test_malformed_yaml_raises_conversion_error(): + """Invalid YAML surfaces as ConversionError, not a raw yaml.YAMLError traceback.""" + with pytest.raises(ConversionError, match="Invalid YAML"): + exporter.convert_ossie_to_metric_view("semantic_model: [oops\n") + + +def test_nested_join_uses_full_path_qualification(): + """A snowflake (orders -> customer -> nation) qualifies a column from the nested + `nation` join by its full join path from the source (`customer.nation.n_name`) -- the + Databricks nested-join rule -- not the single-level `nation.n_name`. A depth-1 join + stays single-name.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "fields": [_field("amt", "amount")]}, + {"name": "customer", "source": "c.s.customer", "fields": [_field("cname", "c_name")]}, + {"name": "nation", "source": "c.s.nation", "fields": [_field("nname", "n_name")]}, + ], + "relationships": [ + {"name": "r1", "from": "orders", "to": "customer", "from_columns": ["ckey"], "to_columns": ["c_key"]}, + {"name": "r2", "from": "customer", "to": "nation", "from_columns": ["nkey"], "to_columns": ["n_key"]}, + ], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + exprs = {d["name"]: d["expr"] for d in out["dimensions"]} + assert exprs["cname"] == "customer.c_name" # depth-1: the join's own name + assert exprs["nname"] == "customer.nation.n_name" # depth-2: full path from source + # the nested join's `on:` still uses immediate names (single-level) + nation_join = out["joins"][0]["joins"][0] + assert nation_join["on"] == "customer.nkey = nation.n_key" + + +def test_case_variant_dataset_name_rejected(): + """DBR identifiers are case-insensitive, so two datasets differing only in case + (`customer`/`Customer`) collide and are rejected (review finding).""" + ossie = ("version: 0.2.0.dev0\nsemantic_model:\n- name: m\n datasets:\n" + " - {name: customer, source: c.s.c}\n - {name: Customer, source: c.s.c2}\n") + with pytest.raises(ConversionError, match="duplicate"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_non_string_field_expression_raises_clean_error(): + """A non-string dialect expression raises a ConversionError, not a raw crash + (review finding).""" + ossie = ("version: 0.2.0.dev0\nsemantic_model:\n- name: m\n datasets:\n" + " - name: o\n source: c.s.o\n fields:\n - name: d\n expression:\n" + " dialects:\n - {dialect: DATABRICKS, expression: 123}\n") + with pytest.raises(ConversionError, match="must be a string"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_scalar_join_columns_rejected(): + """`from_columns`/`to_columns` given as a scalar string (not a list) raise a clear + 'must be lists' error rather than a misleading character-count length error.""" + ossie = ("version: 0.2.0.dev0\nsemantic_model:\n- name: m\n datasets:\n" + " - {name: a, source: c.s.a}\n - {name: b, source: c.s.b}\n relationships:\n" + " - {name: ab, from: a, to: b, from_columns: cid, to_columns: id}\n") + with pytest.raises(ConversionError, match="must be lists"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_malformed_stash_json_raises_conversion_error(): + # A hand-edited DATABRICKS custom_extensions with invalid JSON in `data` must + # surface as a clean ConversionError, not a raw json.JSONDecodeError traceback. + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "custom_extensions": [{"vendor_name": "DATABRICKS", "data": "{not valid json"}], + "datasets": [{"name": "f", "source": "c.s.f", + "fields": [{"name": "x", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "x"}]}}]}], + "metrics": [{"name": "n", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}], + }], + }) + with pytest.raises(ConversionError, match="not valid JSON"): + exporter.convert_ossie_to_metric_view(ossie) diff --git a/converters/databricks/tests/test_roundtrip.py b/converters/databricks/tests/test_roundtrip.py new file mode 100644 index 0000000..c9d5629 --- /dev/null +++ b/converters/databricks/tests/test_roundtrip.py @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Round-trip tests in both directions.""" + +from ossie_databricks import metric_view_to_ossie as importer +from ossie_databricks import ossie_to_metric_view as exporter +from _util import canon, load_fixture, parse, strip_dropped + + +def test_ossie_to_mv_to_ossie(): + """Apache Ossie -> MV -> Apache Ossie preserves everything except the documented drops + (model name, primary_key/unique_keys).""" + ossie_in = load_fixture("fixtureA_ossie.yaml") + mv = exporter.convert_ossie_to_metric_view(ossie_in) + ossie_out = importer.convert_metric_view_to_ossie(mv) + assert strip_dropped(parse(ossie_out)) == strip_dropped(parse(ossie_in)) + + +def test_using_clause_round_trips(): + """A `using` join survives MV -> Apache Ossie -> MV (equal column lists re-emit as `using`).""" + mv_in = ( + "version: '1.1'\nsource: c.s.fact\n" + "joins:\n- name: dim\n source: c.s.dim\n using: [id]\n" + "measures:\n- name: n\n expr: count(*)\n" + ) + ossie = importer.convert_metric_view_to_ossie(mv_in) + join = parse(exporter.convert_ossie_to_metric_view(ossie))["joins"][0] + assert join.get("using") == ["id"] + assert "on" not in join + + +def test_mv_to_ossie_to_mv_is_lossless(): + """MV -> Apache Ossie -> MV is byte-faithful (structurally): the stash carries every + MV-only feature through Apache Ossie and back.""" + mv_in = load_fixture("fixtureB_metric_view.yaml") + ossie = importer.convert_metric_view_to_ossie(mv_in) + mv_out = exporter.convert_ossie_to_metric_view(ossie) + assert parse(mv_out) == parse(mv_in) + + +def test_tpcds_mv_round_trips(): + """The TPC-DS Metric View (multi-join star with rely/filter/format) survives + MV -> Apache Ossie -> MV unchanged.""" + mv_in = load_fixture("tpcds_metric_view.yaml") + ossie = importer.convert_metric_view_to_ossie(mv_in) + mv_out = exporter.convert_ossie_to_metric_view(ossie) + assert parse(mv_out) == parse(mv_in) + + +def test_one_to_many_round_trips_mv_ossie_mv(): + """A one_to_many Metric View survives MV -> Apache Ossie -> MV: cardinality rides the + relationship direction (+ stash), and the source/grain rides the model stash, so + the exporter re-roots at `orders` rather than the FK-sink `line_items`.""" + mv_in = ( + "version: '1.1'\nsource: c.s.orders\ncomment: Orders\n" + "joins:\n- name: line_items\n source: c.s.line_items\n" + " on: source.order_id = line_items.l_order_id\n" + " cardinality: one_to_many\n" + "dimensions:\n- {name: order_date, expr: o_order_date}\n" + "measures:\n- {name: order_count, expr: COUNT(*)}\n" + ) + ossie = importer.convert_metric_view_to_ossie(mv_in) + mv_out = exporter.convert_ossie_to_metric_view(ossie) + assert parse(mv_out) == parse(mv_in) + + +# Property-based round-trip coverage. The Hypothesis driver lives in +# test_roundtrip_properties.py; these run the same generators/assertions under a plain +# seeded RNG so the property coverage also holds where Hypothesis is not installed. + +def test_property_mv_to_ossie_to_mv_seeded(): + from _roundtrip_helpers import RandomRnd, assert_mv_roundtrip, build_metric_view + for seed in range(250): + assert_mv_roundtrip(build_metric_view(RandomRnd(seed))) + + +def test_property_ossie_to_mv_to_ossie_seeded(): + from _roundtrip_helpers import RandomRnd, assert_ossie_roundtrip, build_ossie + for seed in range(250): + assert_ossie_roundtrip(build_ossie(RandomRnd(1_000_000 + seed))) diff --git a/converters/databricks/tests/test_roundtrip_properties.py b/converters/databricks/tests/test_roundtrip_properties.py new file mode 100644 index 0000000..d022ebb --- /dev/null +++ b/converters/databricks/tests/test_roundtrip_properties.py @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Property-based round-trip tests (Hypothesis). + +For any generated model in the round-trippable subset, converting one direction and +back preserves content: + + - MV -> Apache Ossie -> MV : source, every dimension/measure name+expression+metadata, every + join (name, source, condition, cardinality, rely) and the join nesting, and the + model-level filter/comment/materialization. + - Apache Ossie -> MV -> Apache Ossie : dataset names+sources+fields, relationship from/to/columns, + metric name+expression, and the model description. + +The model generation and the assertions live in `_roundtrip_helpers` (no test-framework +dependency) so the exact same logic also runs under a plain seeded RNG where Hypothesis +is unavailable (see the `*_seeded` tests in test_roundtrip.py). This file is the thin +Hypothesis driver: it maps draws into the shared `Rnd` interface and runs the properties. + +Run: `pytest test_roundtrip_properties.py` (needs `hypothesis`). +""" + +import pytest + +pytest.importorskip("hypothesis") # skip cleanly if hypothesis is not installed + +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from _roundtrip_helpers import ( + assert_mv_roundtrip, + assert_ossie_roundtrip, + build_metric_view, + build_ossie, +) + +# Alphanumeric metadata text with optional interior spaces (no leading/trailing space, +# no YAML-special characters), so values survive a dump/load cycle verbatim. +_safe_text = st.from_regex(r"[A-Za-z0-9]([A-Za-z0-9 ]{0,18}[A-Za-z0-9])?", fullmatch=True) +# A SQL column-style identifier. +_colident = st.from_regex(r"[a-z_][a-z0-9_]{0,7}", fullmatch=True) + +_SETTINGS = settings( + max_examples=300, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) + + +class _HypothesisRnd: + """The `Rnd` interface backed by a Hypothesis `draw`. `chance(p)` ignores `p` + (Hypothesis explores both branches regardless).""" + + def __init__(self, draw): + self._draw = draw + + def chance(self, p=0.5): + return self._draw(st.booleans()) + + def count(self, lo, hi): + return self._draw(st.integers(min_value=lo, max_value=hi)) + + def pick(self, seq): + return self._draw(st.sampled_from(list(seq))) + + def text(self): + return self._draw(_safe_text) + + def colname(self): + return self._draw(_colident) + + +@st.composite +def metric_views(draw): + return build_metric_view(_HypothesisRnd(draw)) + + +@st.composite +def ossie_models(draw): + return build_ossie(_HypothesisRnd(draw)) + + +class TestMetricViewRoundTrip: + @given(mv=metric_views()) + @_SETTINGS + def test_mv_to_ossie_to_mv(self, mv): + assert_mv_roundtrip(mv) + + +class TestOSIRoundTrip: + @given(ossie=ossie_models()) + @_SETTINGS + def test_ossie_to_mv_to_ossie(self, ossie): + assert_ossie_roundtrip(ossie) From 225ca4ba6f9b6293413bca7c80ee7cb515f9c4ed Mon Sep 17 00:00:00 2001 From: Emil Sadek Date: Mon, 20 Jul 2026 12:59:14 -0700 Subject: [PATCH 84/89] chore(cli): add gofmt format target to Makefile (#226) Co-authored-by: Emil Sadek --- cli/Makefile | 5 ++++- cli/internal/ossiedir/ossiedir.go | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cli/Makefile b/cli/Makefile index 9ecbcb7..637d8f1 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -1,7 +1,7 @@ BINARY_NAME := ossie BUILD_DIR := dist -.PHONY: build test lint release-dry-run clean +.PHONY: build test lint format release-dry-run clean build: go build -o $(BUILD_DIR)/$(BINARY_NAME) . @@ -12,6 +12,9 @@ test: lint: go vet ./... +format: + gofmt -w . + release-dry-run: goreleaser release --snapshot --clean --config .goreleaser.yaml diff --git a/cli/internal/ossiedir/ossiedir.go b/cli/internal/ossiedir/ossiedir.go index 051a8bd..323c41a 100644 --- a/cli/internal/ossiedir/ossiedir.go +++ b/cli/internal/ossiedir/ossiedir.go @@ -8,8 +8,8 @@ import ( const ( defaultOssieDir = ".ossie" - pluginsSubdir = "plugins" - envVar = "OSSIE_PLUGIN_DIR" + pluginsSubdir = "plugins" + envVar = "OSSIE_PLUGIN_DIR" ) // PluginDir returns the resolved plugin directory path. From 8a73a38fafbf3c78ef9f7570d07ba8ace6fafa91 Mon Sep 17 00:00:00 2001 From: Yong Zheng Date: Tue, 21 Jul 2026 09:39:18 -0500 Subject: [PATCH 85/89] Switch to UV (#228) * Switch to UV * Switch to UV * Switch to UV * Restored ASF header --- converters/dbt/pyproject.toml | 38 +- converters/dbt/uv.lock | 756 ++++++++++++++++++ converters/honeydew/README.md | 10 +- converters/honeydew/pyproject.toml | 43 +- .../honeydew/src/honeydew_osi/__init__.py | 16 + .../converter.py} | 6 +- .../tests/test_honeydew_osi_converter.py | 9 +- converters/honeydew/uv.lock | 129 +++ converters/omni/pyproject.toml | 39 +- converters/omni/uv.lock | 306 +++++++ converters/orionbelt/pyproject.toml | 56 +- converters/orionbelt/uv.lock | 26 +- converters/snowflake/README.md | 6 +- converters/snowflake/pyproject.toml | 62 ++ .../snowflake/src/ossie_snowflake/__init__.py | 16 + .../converter.py} | 2 +- .../test_osi_to_snowflake_yaml_converter.py | 6 +- converters/snowflake/uv.lock | 138 ++++ python/README.md | 38 + python/pyproject.toml | 19 +- python/uv.lock | 220 +++++ 21 files changed, 1844 insertions(+), 97 deletions(-) create mode 100644 converters/dbt/uv.lock create mode 100644 converters/honeydew/src/honeydew_osi/__init__.py rename converters/honeydew/src/{honeydew_osi_converter.py => honeydew_osi/converter.py} (99%) create mode 100644 converters/honeydew/uv.lock create mode 100644 converters/omni/uv.lock create mode 100644 converters/snowflake/pyproject.toml create mode 100644 converters/snowflake/src/ossie_snowflake/__init__.py rename converters/snowflake/src/{osi_to_snowflake_yaml_converter.py => ossie_snowflake/converter.py} (99%) create mode 100644 converters/snowflake/uv.lock create mode 100644 python/README.md create mode 100644 python/uv.lock diff --git a/converters/dbt/pyproject.toml b/converters/dbt/pyproject.toml index 3666f02..5dea258 100644 --- a/converters/dbt/pyproject.toml +++ b/converters/dbt/pyproject.toml @@ -19,14 +19,25 @@ requires = ["hatchling"] build-backend = "hatchling.build" +[dependency-groups] +dev = [ + "pytest>=8.0", + "syrupy>=4.0", +] + [project] name = "apache-ossie-dbt" version = "0.2.0.dev0" description = "dbt (MetricFlow Semantic Interface) <> Apache Ossie converter" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] requires-python = ">=3.11" -classifiers = [ - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Open Semantic Interchange", + "dbt" ] dependencies = [ "apache-ossie>=0.2.0.dev0", @@ -36,25 +47,26 @@ dependencies = [ "PyYAML>=6.0", ] -[project.license] -text = "Apache-2.0" - [project.scripts] ossie-dbt = "ossie_dbt.cli:main" +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + [tool.hatch.build.targets.wheel] packages = ["src/ossie_dbt"] +[tool.pytest.ini_options] +testpaths = ["tests"] + [tool.uv] -dev-dependencies = [ - "pytest>=8.0", - "syrupy>=4.0", +required-version = ">=0.9.0" +default-groups = [ + "dev" ] # apache-ossie is not yet published to PyPI; resolve it from the in-repo # package for now. Remove this block once apache-ossie published to PyPI. [tool.uv.sources] -apache-ossie = { path = "../../python", editable = true} - -[tool.pytest.ini_options] -testpaths = ["tests"] +apache-ossie = { path = "../../python", editable = true} \ No newline at end of file diff --git a/converters/dbt/uv.lock b/converters/dbt/uv.lock new file mode 100644 index 0000000..37eac1d --- /dev/null +++ b/converters/dbt/uv.lock @@ -0,0 +1,756 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "apache-ossie" +version = "0.2.0.dev0" +source = { editable = "../../python" } +dependencies = [ + { name = "pydantic" }, + { name = "pyyaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "pydantic", specifier = ">=2.0" }, + { name = "pyyaml", specifier = ">=6.0" }, +] + +[[package]] +name = "apache-ossie-dbt" +version = "0.2.0.dev0" +source = { editable = "." } +dependencies = [ + { name = "apache-ossie" }, + { name = "jinja2" }, + { name = "metricflow" }, + { name = "pyyaml" }, + { name = "sqlglot" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "syrupy" }, +] + +[package.metadata] +requires-dist = [ + { name = "apache-ossie", editable = "../../python" }, + { name = "jinja2", specifier = ">=3.0" }, + { name = "metricflow", specifier = ">=0.209.0" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "sqlglot", specifier = ">=20.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.0" }, + { name = "syrupy", specifier = ">=4.0" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "importlib-metadata" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7", size = 27789, upload-time = "2026-03-20T06:42:55.665Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "metricflow" +version = "0.211.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "more-itertools" }, + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "rapidfuzz" }, + { name = "referencing" }, + { name = "sqlglot" }, + { name = "tabulate" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/1c/2303b109a1e02216615d5c01cccf61a12d133f326388b9c881d32d00d182/metricflow-0.211.0.tar.gz", hash = "sha256:e15d0c24647f4498fa0926e87c61dd23b15205c9577566bcf848460d58f48bec", size = 1549648, upload-time = "2026-05-11T21:53:51.895Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/b7/de51d5d04ac43aeecbfea7741bf29da84fd5a04995e25a2b85abaa6ee807/metricflow-0.211.0-py3-none-any.whl", hash = "sha256:9cb83ccaa0c47363482fcc5cb57944a3f1af17ca948d90805524161f4e03d600", size = 1914356, upload-time = "2026-05-11T21:53:49.225Z" }, +] + +[[package]] +name = "more-itertools" +version = "10.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/5d/38b681d3fce7a266dd9ab73c66959406d565b3e85f21d5e66e1181d93721/more_itertools-10.8.0.tar.gz", hash = "sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd", size = 137431, upload-time = "2025-09-02T15:23:11.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/8e/469e5a4a2f5855992e425f3cb33804cc07bf18d48f2db061aec61ce50270/more_itertools-10.8.0-py3-none-any.whl", hash = "sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b", size = 69667, upload-time = "2025-09-02T15:23:09.635Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "rapidfuzz" +version = "3.14.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/21/ef6157213316e85790041254259907eb722e00b03480256c0545d98acd33/rapidfuzz-3.14.5.tar.gz", hash = "sha256:ba10ac57884ce82112f7ed910b67e7fb6072d8ef2c06e30dc63c0f604a112e0e", size = 57901753, upload-time = "2026-04-07T11:16:31.931Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/f9/3c41a7be8855803f4f6c713b472226a98d31d41869d98f64f4ca790510d6/rapidfuzz-3.14.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e251126d48615e1f02b4a178f2cd0cd4f0332b8a019c01a2e10480f7552554b4", size = 1952372, upload-time = "2026-04-07T11:13:58.32Z" }, + { url = "https://files.pythonhosted.org/packages/9e/89/c2557e37531d03465193bff0ab9de70b468420a807d71a26a65100635459/rapidfuzz-3.14.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5ab449c9abd0d4e1f8145dce0798a4c822a1a1933d613c764a641bea88b8bdab", size = 1159782, upload-time = "2026-04-07T11:14:00.127Z" }, + { url = "https://files.pythonhosted.org/packages/1a/b2/ffeeb7eca1a897d51b998f4c0ef0281696c3b06abcca4f88f9def708ffe1/rapidfuzz-3.14.5-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb2829fedd672dd7107267189dabe2bbe07972801d636014417c6861eb89e358", size = 1383677, upload-time = "2026-04-07T11:14:01.696Z" }, + { url = "https://files.pythonhosted.org/packages/6b/d0/4539e42a2d596e068f7738f279638a4a74edd1fbb6f8594e2458058979c6/rapidfuzz-3.14.5-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d50e5861872935fece391351cbb5ba21d1bced277cf5e1143d207a0a35f1925", size = 3168906, upload-time = "2026-04-07T11:14:03.29Z" }, + { url = "https://files.pythonhosted.org/packages/5e/1c/3ec897eb9d8b05308aa8ef6ae4ed64b088ad521a3f9d8ff469e7e97bc2b0/rapidfuzz-3.14.5-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:7092a216728f80c960bd6b3807275d1ee318b168986bd5dc523349581d4890b8", size = 1478176, upload-time = "2026-04-07T11:14:04.94Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ba/970c03a12ce20a5399e22afe9f8932fd4cd1265b8a8461d0e63b00eb4eae/rapidfuzz-3.14.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9669753caef7fdc6529f6adcc5883ed98d65976445d9322e7dbdb6b697feee13", size = 2402441, upload-time = "2026-04-07T11:14:07.228Z" }, + { url = "https://files.pythonhosted.org/packages/81/93/61d351cae60c1d0e21ba5ff1a1015ad045539ed215da9d6e302204ed887a/rapidfuzz-3.14.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:823b1b9d9230809d8edcc18872770764bfe8ef4357995e16744047c8ccf0e489", size = 2511628, upload-time = "2026-04-07T11:14:09.234Z" }, + { url = "https://files.pythonhosted.org/packages/87/52/374d2d4f60fd98155142a869323aa221e30868cfa1f15171a0f64070c247/rapidfuzz-3.14.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f0b2af76b7e7060c09e1a0dfa9410eb19369cbe6164509bff2ef94094b54d2b6", size = 4275480, upload-time = "2026-04-07T11:14:11.332Z" }, + { url = "https://files.pythonhosted.org/packages/d8/04/82e7989bc9ec20a15b720a335c5cb6b0724bf6582013898f90a3280cfccd/rapidfuzz-3.14.5-cp311-cp311-win32.whl", hash = "sha256:c5801a89604c65ab4cc9e91b23bc4076d0ca80efd8c976fb63843d7879a85d7f", size = 1725627, upload-time = "2026-04-07T11:14:13.217Z" }, + { url = "https://files.pythonhosted.org/packages/b9/b5/eca8ac5609bc9bcb02bb6ff87fa5983cc92b8772d66a431556ab8a8c178f/rapidfuzz-3.14.5-cp311-cp311-win_amd64.whl", hash = "sha256:d7ca16637c0ede8243f84074044bd0b2335a0341421f8227c85756de2d18c819", size = 1545977, upload-time = "2026-04-07T11:14:14.766Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e1/dbf318de28f65fa2cdd0a9dfbdee380f8199eb83b19259bc4f8592551b4e/rapidfuzz-3.14.5-cp311-cp311-win_arm64.whl", hash = "sha256:8c90cdf8516d9057e502aa6003cea71cf5ec27cc44699ca52412b502a04761bb", size = 816827, upload-time = "2026-04-07T11:14:16.788Z" }, + { url = "https://files.pythonhosted.org/packages/d3/e3/574435c6aafb80254c191ef40d7aca2cb2bb97a095ec9395e9fa59ac307a/rapidfuzz-3.14.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0d3378f471ef440473a396ce2f8e97ee12f89a78b495540e0a5617bbfe895638", size = 1944601, upload-time = "2026-04-07T11:14:18.771Z" }, + { url = "https://files.pythonhosted.org/packages/d0/1f/fbad3102a255ecc112ce9a7e779bacab7fd14398217be8868dc9082ba363/rapidfuzz-3.14.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e910eebca9fd0eba245c0555e764597e8a0cccb673a92da2dc2397050725f48", size = 1164293, upload-time = "2026-04-07T11:14:20.534Z" }, + { url = "https://files.pythonhosted.org/packages/88/37/a3eb7ff6121ed3a5f199a8c38cc86c8e481816f879cb0e0b738b078c9a7e/rapidfuzz-3.14.5-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:01550fe5f60fd176aa66b7611289d46dc4aa4b1b904874c7b6d1d54e581c5ec1", size = 1371999, upload-time = "2026-04-07T11:14:22.63Z" }, + { url = "https://files.pythonhosted.org/packages/79/72/97a9728c711c7c1b06e107d3f0623880fb4ef90e147ed13c551a1730e7cc/rapidfuzz-3.14.5-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48bee0b91bebfaec41e1081e351000659ab7570cc4598d617aa04d5bf827f9e6", size = 3145715, upload-time = "2026-04-07T11:14:24.508Z" }, + { url = "https://files.pythonhosted.org/packages/ed/54/d5caabbea233ac90c286c87c260e49d7641467e87438a18d858e41c82e91/rapidfuzz-3.14.5-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:7e580cb04ad849ae9b786fa21383c6b994b6e6c1444ad1cb9f22392759d72741", size = 1456304, upload-time = "2026-04-07T11:14:26.515Z" }, + { url = "https://files.pythonhosted.org/packages/fc/a7/2d1a81250ac8c01a0100c026018e76f0e7a097ff63e4c553e02a6938c6fb/rapidfuzz-3.14.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:09d6c9ba091854f07817055d795d604179c12a8f308ba4c7d56f3719dfea1646", size = 2389089, upload-time = "2026-04-07T11:14:28.635Z" }, + { url = "https://files.pythonhosted.org/packages/65/0d/c47c3872203ae88e6506997c0b576ad731f5261daa25d559be09c9756658/rapidfuzz-3.14.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:1e989f86113be66574113b9c7bdf4793f3f863d248e47d911b355e05ca6b6b10", size = 2493404, upload-time = "2026-04-07T11:14:30.577Z" }, + { url = "https://files.pythonhosted.org/packages/8f/2f/71e0a5a3130792146c8a200a2dd1e52aa16f7c1074012e17f2601eea9a90/rapidfuzz-3.14.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ebd1a18e2e47bc0b292a07e6ed9c3642f8aaa672d12253885f599b50807a4f9", size = 4251709, upload-time = "2026-04-07T11:14:32.451Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/d39874901abacef325adb5b34ae416817c8486dfb4fb87c7a9b74ec5b072/rapidfuzz-3.14.5-cp312-cp312-win32.whl", hash = "sha256:9981d38a703b86f0e315a3cd229fd1906fe1d91c989ed121fb975b3c849f89f5", size = 1710069, upload-time = "2026-04-07T11:14:34.37Z" }, + { url = "https://files.pythonhosted.org/packages/85/0b/f65572c53de8a1c704bda707f63a447b67bdbe95d7cdc70d18885e191df5/rapidfuzz-3.14.5-cp312-cp312-win_amd64.whl", hash = "sha256:d8375e3da319593389727c3187ccaf3e0e84199accc530866b8e0f2b79af05e9", size = 1540630, upload-time = "2026-04-07T11:14:36.287Z" }, + { url = "https://files.pythonhosted.org/packages/5e/c3/143be3a578f989758cae516f3270d5cbb49783a7bfdf57cc27a670e00456/rapidfuzz-3.14.5-cp312-cp312-win_arm64.whl", hash = "sha256:478b59bb018a6780d73f33e38d0b3ec5e968a6c1ed42876b993dd456b7aa20e8", size = 813137, upload-time = "2026-04-07T11:14:38.289Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/252803f2010ba699618cdc048b6e1f7cc1f433c08b4a9a17579b92ab0142/rapidfuzz-3.14.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ebd8fd343bf8492a1e60bcb6dc99f90f74f65d98d8241a6b3e1fed225b76ecd6", size = 1940205, upload-time = "2026-04-07T11:14:40.319Z" }, + { url = "https://files.pythonhosted.org/packages/ea/59/b2afd98e41af9cd54554a4c1c423d84cdd60e6b1c0a09496f033b55f60ec/rapidfuzz-3.14.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6737b35d5af7479c5bf9710f7b17edd9d2c43128d974d25fb4ea653e42c64609", size = 1159639, upload-time = "2026-04-07T11:14:42.52Z" }, + { url = "https://files.pythonhosted.org/packages/a3/31/7aa7e62c4c516a7af322ed0c4f0774208b72d457d0cfec808bad0df12f4a/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b002c7994cc9f2bc9d9856f0fbaee6e8072c983873846c92f25cefba5b2a925f", size = 1367194, upload-time = "2026-04-07T11:14:44.25Z" }, + { url = "https://files.pythonhosted.org/packages/90/79/2fc252a63bc91d3c3b234d0a3a6ad4ebc460037a23cdcdaf9285f986e6c9/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:17a34330cd2a538c1ce5d400b61ba358c5b72c654b928ff87b362e88f8b864c7", size = 3151805, upload-time = "2026-04-07T11:14:46.21Z" }, + { url = "https://files.pythonhosted.org/packages/17/54/0c83508f2683ea70e2d05f8527eb07328acf7bb1e9d97a3bece5702378e7/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:95d937e74c1a7a1287dfb03b62a827be08ede10a155cf1af73bbf47f2b73ee6e", size = 1455667, upload-time = "2026-04-07T11:14:47.991Z" }, + { url = "https://files.pythonhosted.org/packages/71/1b/070175e873177814d58850a01ebe80e20ae11e93eb4da894d563988660fa/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:46b92a9970dcc34f0096901c792644094cab49554ac3547f35e3aebbdf0a3610", size = 2388246, upload-time = "2026-04-07T11:14:50.098Z" }, + { url = "https://files.pythonhosted.org/packages/c9/dd/77caf7aaf9c2be050ad1f128d7c24ff0f59079aa62c5f62f9df41c0af45e/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e012177c8e8a8a0754ae0d6027d63042aa5ff036d9f40f07cb3466a6082e21b8", size = 2494333, upload-time = "2026-04-07T11:14:52.303Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/dd7e1f2aa31a8fbbfc16b0610af1d770ffaf1287490f3c8c5b1c52da264f/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a2ae6f53f99c9a0eca7a0afc5b4e45fc73bc1dd4ac74c00509031d76df80ed98", size = 4258579, upload-time = "2026-04-07T11:14:54.538Z" }, + { url = "https://files.pythonhosted.org/packages/9c/0a/ac99e1ba347ba0e85e0bb60b74231d55fb93c0eff43f2920ccb413d0be08/rapidfuzz-3.14.5-cp313-cp313-win32.whl", hash = "sha256:4a60f0057231188e3bd30216f7b4e0f279b11fa4ec818bb6c1d9f014d1562fbc", size = 1709231, upload-time = "2026-04-07T11:14:56.524Z" }, + { url = "https://files.pythonhosted.org/packages/cf/cb/0e251d731b3166378644238e8f0cf9e89858c024e19f75ca9f7e3ae83fd5/rapidfuzz-3.14.5-cp313-cp313-win_amd64.whl", hash = "sha256:11bfc2ed8fbe4ab86bd516fadefab126f90e6dcadffa761739fcb304707dfd35", size = 1538519, upload-time = "2026-04-07T11:14:58.635Z" }, + { url = "https://files.pythonhosted.org/packages/30/6f/4548132acc947db6d5346a248e44a8b3a22d608ef30e770fb578caaf2d00/rapidfuzz-3.14.5-cp313-cp313-win_arm64.whl", hash = "sha256:b486b5218808f6f4dc471b114b1054e63553db69705c97da0271f47bd706aedd", size = 812628, upload-time = "2026-04-07T11:15:00.552Z" }, + { url = "https://files.pythonhosted.org/packages/00/60/69b177577290c5eab892c6f75fe89c3aff3f9ae80298a78d9372b1cecb9a/rapidfuzz-3.14.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:39ef8658aaf67d51667e7bdaf7096f432333377d8302ac43c70b5df8a4cf89b8", size = 1970231, upload-time = "2026-04-07T11:15:02.603Z" }, + { url = "https://files.pythonhosted.org/packages/48/38/2fd790052659cc4e2907b63c25433f0987864b445c1aeec1a302ef5ad948/rapidfuzz-3.14.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9ad37a0be705b544af6296da8edddc260d10a8ae5462530fc9991f66498bb1f9", size = 1194394, upload-time = "2026-04-07T11:15:04.572Z" }, + { url = "https://files.pythonhosted.org/packages/80/f4/28430ad8472fc3536e8ebd51a864a226e979cfe924c6e3f83d111373aa74/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d45e06f60729e07d9b20c205f7e5cff90b6ef2584e852eecf46e045aea69627d", size = 1377051, upload-time = "2026-04-07T11:15:06.728Z" }, + { url = "https://files.pythonhosted.org/packages/77/7e/9aeacabcfd1e77397968362e5b98fe14248b8307011136b17daf99752a8e/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e52da10236aa6212de71b9e170bace65b64b129c0dea7fc243d6c9ce976f5074", size = 3160565, upload-time = "2026-04-07T11:15:08.667Z" }, + { url = "https://files.pythonhosted.org/packages/56/f4/db4dd7be0cd2f2022117ac5407d905f435d60e48baaea313a567ad27e865/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_39_riscv64.whl", hash = "sha256:440d30faaf682ca496170a7f0cc5453ec942e3e079f0fd802c9a7f938dfb50a3", size = 1442113, upload-time = "2026-04-07T11:15:11.138Z" }, + { url = "https://files.pythonhosted.org/packages/a4/99/0e9f6aa57f3e32a767216f797e56dc96b720fcecfb9d8ee907ecc82f8d66/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:56227a61fd3d17b0cd9793132431f3a3d07c8654be96794ba9f89fe0fc8b2d09", size = 2396618, upload-time = "2026-04-07T11:15:13.154Z" }, + { url = "https://files.pythonhosted.org/packages/60/94/44a78e39ffce17cbdd3e2b53b696acc751d5d153be0f499d052b07a4d904/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2e83cd2e25bb4edd97b689d9979d9c3acccdaaf26ceac08212ceece202febcfa", size = 2478220, upload-time = "2026-04-07T11:15:15.193Z" }, + { url = "https://files.pythonhosted.org/packages/dd/df/454311469a09a507e9d784a35796742bec22e4cebe75551e2da4e0e290fd/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:af3b859726cd3374287e405e14b9634563c078c5531a4f62375508addebddad1", size = 4265027, upload-time = "2026-04-07T11:15:17.28Z" }, + { url = "https://files.pythonhosted.org/packages/fc/01/175465a9ab3e3b70ba669058372f009d1d49c1746e2dcd56b69df188d3a5/rapidfuzz-3.14.5-cp313-cp313t-win32.whl", hash = "sha256:8ce1d850b3c0178440efde9e884d98421b5e87ff925f364d6d79e23910d7593f", size = 1766814, upload-time = "2026-04-07T11:15:19.687Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a0/a9b84a47af06ebed94a1439eb2f02adebfb8628bcd30af1fe3e02f5ef56c/rapidfuzz-3.14.5-cp313-cp313t-win_amd64.whl", hash = "sha256:c84af70bcf34e99aee894e46a0f1ac77f17d0ef828179c387407642e2466d28a", size = 1582448, upload-time = "2026-04-07T11:15:21.98Z" }, + { url = "https://files.pythonhosted.org/packages/1e/f1/5937800238b3f8248e70860d79f69ba8f73e764fff47e36bc9e2f26dbcc6/rapidfuzz-3.14.5-cp313-cp313t-win_arm64.whl", hash = "sha256:aac0ad28c686a5e72b81668b906c030ee28050b244544b8af68e12fb32543895", size = 832932, upload-time = "2026-04-07T11:15:24.358Z" }, + { url = "https://files.pythonhosted.org/packages/81/41/aa3ffb3355e62e1bf91f6599b3092e866bc88487a07c524004943c7676df/rapidfuzz-3.14.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1a31cc6d7d03e7318a0974c038959c59e19c752b81115f2e9138b3331cd64d45", size = 1943327, upload-time = "2026-04-07T11:15:26.266Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e1/c2141f1840a41e07ad2db6f724945f8f8ff3065463899a22939152dd6e09/rapidfuzz-3.14.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0298d357e2bc59d572da4db0bc631009b6f8f6c9bc8c11e99a12b833f16b6575", size = 1161755, upload-time = "2026-04-07T11:15:28.659Z" }, + { url = "https://files.pythonhosted.org/packages/ca/07/66e753eeaa353161d1d331b7dd517bb349b0bacfebe8496d7b26be26f81f/rapidfuzz-3.14.5-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:59b3dba758661a318995655435c6ab20a04ade79fa51e75bc8dc107cac8df280", size = 1376571, upload-time = "2026-04-07T11:15:31.225Z" }, + { url = "https://files.pythonhosted.org/packages/c8/85/9535df0b78ba51f478c9ce7eb6d1f85535cc31fe356773b48fd9d3e563ca/rapidfuzz-3.14.5-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4900143d82071bdda533b00300c40b14b963ff826b3642cc463b6dd0f036585e", size = 3156468, upload-time = "2026-04-07T11:15:33.428Z" }, + { url = "https://files.pythonhosted.org/packages/81/ee/b667eb93bba6dc4e0de658edd778e1619dc4d6aab68fa5e5c7f075152735/rapidfuzz-3.14.5-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:feedf219672eef83ea6be6f3bb093bba396a8560fc75be85ba225f082903df0a", size = 1458311, upload-time = "2026-04-07T11:15:35.557Z" }, + { url = "https://files.pythonhosted.org/packages/7d/ce/479074f5624364a48df3403c538797ef22d3ac49c19dc76c3f79fcdcc70c/rapidfuzz-3.14.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:419e4397a36e2665ec992d8d64c20ba4b2a42500c76ecadeca78a4f19cb9cc32", size = 2398228, upload-time = "2026-04-07T11:15:37.669Z" }, + { url = "https://files.pythonhosted.org/packages/0b/15/a8982f649150fffbdcd6f17565974501f6ab33b2795267bffbd4a7ba905b/rapidfuzz-3.14.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:97131ab2be39043054ee28d99e09efe316e6d53449b7e962dfcf3c2de8b2b246", size = 2497226, upload-time = "2026-04-07T11:15:39.857Z" }, + { url = "https://files.pythonhosted.org/packages/19/52/5267c03ef6759831b7d4625a0c9c06e87baa2fae084b61ac9c388858317b/rapidfuzz-3.14.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:593c00dac4e30231c35bf3b4f1da8ec0998762e9e94425586a5d636fcd57f9d0", size = 4262283, upload-time = "2026-04-07T11:15:42.279Z" }, + { url = "https://files.pythonhosted.org/packages/71/c0/2579f343a97f5254c43bb5853baccc01488357dcb64a27bcb869b7888a4a/rapidfuzz-3.14.5-cp314-cp314-win32.whl", hash = "sha256:0084b687b02b4e569b46d8d6d4ad25659528e6081cd6d067ca453a69035f07e4", size = 1744614, upload-time = "2026-04-07T11:15:44.498Z" }, + { url = "https://files.pythonhosted.org/packages/17/eb/8edfed1e80119dc9c35b11df4bc701eea85622ad681fff0263b6961d3224/rapidfuzz-3.14.5-cp314-cp314-win_amd64.whl", hash = "sha256:5dfa89d78f22cd773054caff44827b846161a29f2dcf7e78b8f90d086621e502", size = 1588971, upload-time = "2026-04-07T11:15:46.86Z" }, + { url = "https://files.pythonhosted.org/packages/f6/04/5676df93c85cfa57a3045d8047318df9f3cd58c7b8a99340dd95f874795e/rapidfuzz-3.14.5-cp314-cp314-win_arm64.whl", hash = "sha256:67f3f9d2b444268ab53e47d31bab89954888d23c04c6789f2c727e51fe4b1d13", size = 834985, upload-time = "2026-04-07T11:15:49.411Z" }, + { url = "https://files.pythonhosted.org/packages/f7/0d/4a8988cea658fe335048ddef8c876addff1b6daa3c9ca8ad65a5a2196e69/rapidfuzz-3.14.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:77eac0526899b3c3ad1454bb2b03cdb491d67358ec8ef0c9c48bd61b632b431d", size = 1972517, upload-time = "2026-04-07T11:15:51.819Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a3/f5cfd9965a9d9a9e32249159797c47b5d6299ea6d1629f9126b25f1c10a3/rapidfuzz-3.14.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b9c6bd754d11f6e78ac54e3d86b4b11dc1ba2f13e5fc958899574532897f5a99", size = 1196056, upload-time = "2026-04-07T11:15:54.292Z" }, + { url = "https://files.pythonhosted.org/packages/64/07/561c2e40cfd10e6630a7b0ac5a2a813aef50d944bcd1f3d260319d659d5b/rapidfuzz-3.14.5-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:738c96944d076deeaff70e92b65696ab4f7ecb8081d7791c5403a3257dfaf8ff", size = 1374732, upload-time = "2026-04-07T11:15:56.584Z" }, + { url = "https://files.pythonhosted.org/packages/c2/39/123bb94fee40e2fb3b7c49b80827c7ef42d838e18def3fc2fef5a3cf817a/rapidfuzz-3.14.5-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4c1bca487a17fe4226b4ffb2d30e799d2b274d692cffa76bd0746f56235fca3", size = 3166902, upload-time = "2026-04-07T11:15:58.768Z" }, + { url = "https://files.pythonhosted.org/packages/75/0a/45716fafc9fd2e028cf20b5ac5bc704887081cd312f84edb0e325599414b/rapidfuzz-3.14.5-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:af6a90a4ed2a48fa1a2d17e9d824e6c7c950bea5bad0b707c77fd55751e6bfef", size = 1452130, upload-time = "2026-04-07T11:16:01.453Z" }, + { url = "https://files.pythonhosted.org/packages/ca/49/4e96c413114398481c0a5b0086af32c364a18613c9a2ea578d17c4bea4ee/rapidfuzz-3.14.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bf5018938208d4597b2e679a4f8cff9fd252f1df53583130ae56281a21801b64", size = 2396308, upload-time = "2026-04-07T11:16:03.588Z" }, + { url = "https://files.pythonhosted.org/packages/89/b7/49fea9fc6878d59bd259d01dd1972d9b86117992b1c66d9b16f0a65273c3/rapidfuzz-3.14.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c0919d1f89ddf91129906705723118ea09754171e4116f5a5dbc667c7bc9b261", size = 2488210, upload-time = "2026-04-07T11:16:05.871Z" }, + { url = "https://files.pythonhosted.org/packages/0c/44/a1f732b93ffacbdad077b7c801149549b2938e1bece6addb5ad85ed74df8/rapidfuzz-3.14.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:93d8da883a35116d6813432177f35e570db5b0a5e30ecb0cbd7cb39c815735df", size = 4270621, upload-time = "2026-04-07T11:16:08.483Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ce/ff942d19fce5385054650bb71a58495ddda299d94661ccc4e6e7fa44868b/rapidfuzz-3.14.5-cp314-cp314t-win32.whl", hash = "sha256:0f23e37019ec07712d58976b1ab2b889f8649a7f7c2f626a2f34ea9139e79279", size = 1803950, upload-time = "2026-04-07T11:16:10.873Z" }, + { url = "https://files.pythonhosted.org/packages/5c/0f/9aafc63f9661222b819b391c187eed29fc90ad5935f9690e5ecc2d2047a4/rapidfuzz-3.14.5-cp314-cp314t-win_amd64.whl", hash = "sha256:7d5ca9c7832e6879a707296d1463685f7c243a27846227044504741640caec66", size = 1632357, upload-time = "2026-04-07T11:16:13.1Z" }, + { url = "https://files.pythonhosted.org/packages/70/a6/51fc1b0e61e3326e1c68a61cfd0c6b3c34c843681c4b1eefbf0596f59162/rapidfuzz-3.14.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3e91dcd2549b8f8d843f98ba03a17e01f3d8b72ce942adbbb6761bc58ffce813", size = 855409, upload-time = "2026-04-07T11:16:15.787Z" }, + { url = "https://files.pythonhosted.org/packages/d9/ee/e71853bf82846c5c2174b924b71d8e8099fb05ff87c958a720380b434ba3/rapidfuzz-3.14.5-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:578e6051f6d5e6200c259b47a103cf06bb875ab5814d17333fc0b5c290b22f4c", size = 1888603, upload-time = "2026-04-07T11:16:18.223Z" }, + { url = "https://files.pythonhosted.org/packages/36/82/40f67b730f32be2ebad9f62add1571c754f52249254b2e88af094b907eee/rapidfuzz-3.14.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fbf1b8bb2695415b347f3727da1addca2acb82c9b97ac86bebf8b1bead1eb12d", size = 1120599, upload-time = "2026-04-07T11:16:20.682Z" }, + { url = "https://files.pythonhosted.org/packages/ef/9f/a3635cc4ec8fc6e14b46e7db1f7f8763d8c4bef33dcc124eea2e6cb2c8f3/rapidfuzz-3.14.5-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f4a8f5cc84c7ad6bffa0e9947b33eb343ad66e6b53e94fe54378a5508c5ed53", size = 1348524, upload-time = "2026-04-07T11:16:23.451Z" }, + { url = "https://files.pythonhosted.org/packages/cc/1b/2b229520f0b48464cfcd7aa758f74551d12c9bc4ab544022a60210aab064/rapidfuzz-3.14.5-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:97c6d85283629646fa87acc22c66b30ea9d4de7f6fdf887daa2e30fa041829b5", size = 3099302, upload-time = "2026-04-07T11:16:25.858Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b5/363906b1064fc6fe611783a61764927bbd91919aaaabe8cba82151ca93ef/rapidfuzz-3.14.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:dfef96543ced67d9513a422755db422ae1dc34dade0a1485e0b43e7342ed3ebf", size = 1509889, upload-time = "2026-04-07T11:16:28.487Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/1f/a2dca5ffdbf1d475ffc4e80e4d5d720ff3a00f691795910116960ee12511/rpds_py-2026.6.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7", size = 342174, upload-time = "2026-06-30T07:14:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/4d/dc/323d08583c0832911768663d1944f0107fcd4088704858d84b5e06d105a0/rpds_py-2026.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911", size = 345513, upload-time = "2026-06-30T07:14:56.515Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2a/e31989834d18d2f26ec1d2774c5b1eb3331df4ea8ada525175294c94b48a/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4", size = 373783, upload-time = "2026-06-30T07:14:57.736Z" }, + { url = "https://files.pythonhosted.org/packages/87/fe/e80107ee3639585c9941c17d6a42cd65325022f656c023191fce78c324c8/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261", size = 378316, upload-time = "2026-06-30T07:14:59.077Z" }, + { url = "https://files.pythonhosted.org/packages/22/6f/81e3adf81acfb6fa694de2a6e4e7d8863121e3e0799e0a7725e6cf5679c4/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278", size = 499423, upload-time = "2026-06-30T07:15:00.488Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9a/41263969df0ce3d9af2a96d5005a288200af1989aed3354bfceb5fc0b21f/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9", size = 386077, upload-time = "2026-06-30T07:15:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/7e98f468bd50346faff5b10e5297374b443bfdddacc8e9fbc65984539597/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7", size = 371315, upload-time = "2026-06-30T07:15:03.317Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/2b973b4d371906a134b03decfea7f5d9835a2c6d263454392e15b64b5b18/rpds_py-2026.6.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3", size = 383502, upload-time = "2026-06-30T07:15:04.627Z" }, + { url = "https://files.pythonhosted.org/packages/98/2a/12e2799500af0a307bca76b63361c51f9fe479223561489c29eea1f2ee41/rpds_py-2026.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da", size = 402673, upload-time = "2026-06-30T07:15:05.856Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e3/21e5872d165fe08be4f229e3d5ee9d90019c0bf0e5538de60dbd54009450/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4", size = 549964, upload-time = "2026-06-30T07:15:07.159Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d0/5ee0fe36844297de8123bee27bc12078c1a7416ad9f1b8a8ca18d6b0c0ac/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6", size = 615446, upload-time = "2026-06-30T07:15:08.531Z" }, + { url = "https://files.pythonhosted.org/packages/b1/80/1ea5873cb683f2fbe5f21b23ea1f6d179ead19f3c5b249b7eb5dca568ef2/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93", size = 576975, upload-time = "2026-06-30T07:15:09.97Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e1/90ef639217a5ddb15b7f4f61b1c33911fd044ad03c311bafdd2bcab85582/rpds_py-2026.6.3-cp311-cp311-win32.whl", hash = "sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a", size = 204453, upload-time = "2026-06-30T07:15:11.324Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b7/b7a1695d7af36f521fb11e80d6d3adbd744f73b921859bd3c2a2c0dc706f/rpds_py-2026.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127", size = 223219, upload-time = "2026-06-30T07:15:12.476Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a2/145afacf796e4506062825941176ad9445c2dcf2b3b6a1f13d3030a15e19/rpds_py-2026.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804", size = 219137, upload-time = "2026-06-30T07:15:13.631Z" }, + { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, + { url = "https://files.pythonhosted.org/packages/b4/9c/f0d19ac587fd0e4ab6b72cda355e9c5a6166b01ef7e064e437aef8eb9fef/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f", size = 349791, upload-time = "2026-06-30T07:17:33.315Z" }, + { url = "https://files.pythonhosted.org/packages/38/c7/1d49d204c9fd2ee6c537601dc4c1ba921e03363ca576bfab94a00254ac9a/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171", size = 352842, upload-time = "2026-06-30T07:17:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e5/c0b5dc93cd0d4c06ce1f438907649514e2ea077bcd911e3154a51e96c38e/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90", size = 382094, upload-time = "2026-06-30T07:17:36.514Z" }, + { url = "https://files.pythonhosted.org/packages/0d/54/ec0e907b4ca8d541112db352409bd15f871c9b243e0c92c9b5a46ae96f01/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca", size = 388662, upload-time = "2026-06-30T07:17:38.235Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f4/921c22a4fd0f1c1ac13a3996ffbf0aa67951e2c8ad0d1d9574938a2932e8/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9", size = 504896, upload-time = "2026-06-30T07:17:39.689Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1b/a114b972cefa1ab1cdb3c7bb177cd3844a12826c507c722d3a73516dbbaf/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c", size = 391545, upload-time = "2026-06-30T07:17:41.336Z" }, + { url = "https://files.pythonhosted.org/packages/4e/98/af9b3db77d47fcbe6c8c1f36e2c2147ec70292819e99c325f871584a1c11/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9", size = 380059, upload-time = "2026-06-30T07:17:42.857Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ba/0efd8668b97c1d26a61566386c636a7a7a09829e474fdf807caa15a2c844/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41", size = 393235, upload-time = "2026-06-30T07:17:44.637Z" }, + { url = "https://files.pythonhosted.org/packages/62/90/8c139ee9690f73b0829f32647de6f40d826f8f443af6fa72644f96351aac/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c", size = 413008, upload-time = "2026-06-30T07:17:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/0043896fdd7828ce09a1d9a8b06433714d0960fc4ff3fc4aa72b666b764e/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9", size = 558118, upload-time = "2026-06-30T07:17:47.759Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/02355f0e134f783a8f9814c4680a1bd311d37671577a5964ea838573ff37/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76", size = 623138, upload-time = "2026-06-30T07:17:49.355Z" }, + { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "sqlglot" +version = "30.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/ed/a6c45aec29353b6392ea34548c40af3ac6ffd6bc5572cf23b2ce250876fc/sqlglot-30.12.0.tar.gz", hash = "sha256:6b8369704662d4f654bc934cea4dd31c916c2a571b389210cb9e951a275e5fd9", size = 5905110, upload-time = "2026-06-26T14:09:40.408Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/9e/82a390ecc85f066ff80affa01d195f744e3de60ad4d695b8de31c9a66da3/sqlglot-30.12.0-py3-none-any.whl", hash = "sha256:86cccc610073c645c03e72b55b60ae0518aa3253a7fc3bd56551370d003c6554", size = 707583, upload-time = "2026-06-26T14:09:38.525Z" }, +] + +[[package]] +name = "syrupy" +version = "5.5.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/39/17dde9f0c76cc5abcdeef1b2243791bb850d498f784147b8e460dd23abe8/syrupy-5.5.3.tar.gz", hash = "sha256:fa21e4ae77c89ec5abfca513338d8a7eb916da6618ca0f8db301476e0768e57a", size = 91614, upload-time = "2026-07-11T15:54:31.46Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/f9/617a194c1a4203279998e1859426cf2635042533a99a93d63d3ee1fb967e/syrupy-5.5.3-py3-none-any.whl", hash = "sha256:0b260a0c9dad55e1fb83818973dc36fbc1aea3fd5381592f442a986686c4171a", size = 54959, upload-time = "2026-07-11T15:54:29.836Z" }, +] + +[[package]] +name = "tabulate" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "zipp" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, +] diff --git a/converters/honeydew/README.md b/converters/honeydew/README.md index 5ee6453..03f1bab 100644 --- a/converters/honeydew/README.md +++ b/converters/honeydew/README.md @@ -44,25 +44,23 @@ Bidirectional converter between [OSI](../../core-spec/spec.md) semantic models a ## Setup ```bash -pip install . -# or in editable mode for development: -pip install -e . +uv sync ``` ## Usage ```bash # OSI YAML → Honeydew workspace directory -python src/honeydew_osi_converter.py osi-to-honeydew -i input.yaml -o output_dir/ +uv run honeydew-osi osi-to-honeydew -i input.yaml -o output_dir/ # Honeydew workspace directory → OSI YAML -python src/honeydew_osi_converter.py honeydew-to-osi -i workspace_dir/ -o output.yaml +uv run honeydew-osi honeydew-to-osi -i workspace_dir/ -o output.yaml ``` ## Tests ```bash -python -m pytest tests/ +uv run pytest ``` ## Limitations diff --git a/converters/honeydew/pyproject.toml b/converters/honeydew/pyproject.toml index 34447ae..7c94c33 100644 --- a/converters/honeydew/pyproject.toml +++ b/converters/honeydew/pyproject.toml @@ -15,28 +15,49 @@ # specific language governing permissions and limitations # under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.0", +] + [project] name = "honeydew-osi" version = "0.2.0.dev0" description = "Bidirectional converter between Honeydew workspace YAML and OSI semantic model" -readme = "README.md" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] requires-python = ">=3.12" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Open Semantic Interchange", + "Honeydew" +] dependencies = [ "pyyaml>=6.0", ] -[dependency-groups] -dev = [ - "pytest>=8.0", -] +[project.scripts] +honeydew-osi = "honeydew_osi.converter:main" -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" -[tool.hatch.build] -include = ["src/**/*"] +[tool.hatch.build.targets.wheel] +packages = ["src/honeydew_osi"] [tool.pytest.ini_options] testpaths = ["tests"] -pythonpath = ["src"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev" +] \ No newline at end of file diff --git a/converters/honeydew/src/honeydew_osi/__init__.py b/converters/honeydew/src/honeydew_osi/__init__.py new file mode 100644 index 0000000..d216be4 --- /dev/null +++ b/converters/honeydew/src/honeydew_osi/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. \ No newline at end of file diff --git a/converters/honeydew/src/honeydew_osi_converter.py b/converters/honeydew/src/honeydew_osi/converter.py similarity index 99% rename from converters/honeydew/src/honeydew_osi_converter.py rename to converters/honeydew/src/honeydew_osi/converter.py index 5bfe200..e3f78f4 100644 --- a/converters/honeydew/src/honeydew_osi_converter.py +++ b/converters/honeydew/src/honeydew_osi/converter.py @@ -24,8 +24,8 @@ Honeydew → OSI: Reads a Honeydew workspace directory and produces an OSI YAML. Usage: - python honeydew_osi_converter.py osi-to-honeydew -i input.yaml -o output_dir/ - python honeydew_osi_converter.py honeydew-to-osi -i workspace_dir/ -o output.yaml + python converter.py osi-to-honeydew -i input.yaml -o output_dir/ + python converter.py honeydew-to-osi -i workspace_dir/ -o output.yaml """ import argparse @@ -1065,4 +1065,4 @@ def main() -> None: if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/converters/honeydew/tests/test_honeydew_osi_converter.py b/converters/honeydew/tests/test_honeydew_osi_converter.py index 4d09ae6..c3aaa75 100644 --- a/converters/honeydew/tests/test_honeydew_osi_converter.py +++ b/converters/honeydew/tests/test_honeydew_osi_converter.py @@ -26,8 +26,7 @@ import pytest import yaml -sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) -from honeydew_osi_converter import ( +from honeydew_osi.converter import ( HoneydewConversionError, _assign_metrics_to_entities, _build_osi_metadata, @@ -1445,7 +1444,7 @@ def test_main_osi_to_honeydew(tmp_path): })) output_dir = tmp_path / "out" result = subprocess.run( - [sys.executable, str(Path(__file__).resolve().parent.parent / "src" / "honeydew_osi_converter.py"), + [sys.executable, "-m", "honeydew_osi.converter", "osi-to-honeydew", "-i", str(input_file), "-o", str(output_dir)], capture_output=True, text=True, ) @@ -1463,7 +1462,7 @@ def test_main_honeydew_to_osi(tmp_path): }]) output_file = tmp_path / "output.yaml" result = subprocess.run( - [sys.executable, str(Path(__file__).resolve().parent.parent / "src" / "honeydew_osi_converter.py"), + [sys.executable, "-m", "honeydew_osi.converter", "honeydew-to-osi", "-i", str(tmp_path), "-o", str(output_file)], capture_output=True, text=True, ) @@ -1488,7 +1487,7 @@ def test_main_path_traversal_rejected(tmp_path): ) output_dir = tmp_path / "out" result = subprocess.run( - [sys.executable, str(Path(__file__).resolve().parent.parent / "src" / "honeydew_osi_converter.py"), + [sys.executable, "-m", "honeydew_osi.converter", "osi-to-honeydew", "-i", str(input_file), "-o", str(output_dir)], capture_output=True, text=True, ) diff --git a/converters/honeydew/uv.lock b/converters/honeydew/uv.lock new file mode 100644 index 0000000..1220f53 --- /dev/null +++ b/converters/honeydew/uv.lock @@ -0,0 +1,129 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "honeydew-osi" +version = "0.2.0.dev0" +source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.0" }] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] diff --git a/converters/omni/pyproject.toml b/converters/omni/pyproject.toml index e8bc633..69bea10 100644 --- a/converters/omni/pyproject.toml +++ b/converters/omni/pyproject.toml @@ -19,30 +19,45 @@ requires = ["hatchling"] build-backend = "hatchling.build" +[dependency-groups] +dev = [ + "pytest>=8.0", + "hypothesis>=6.0", +] + [project] name = "osi-omni" version = "0.2.0.dev0" -description = "Bidirectional converter between OSI semantic models and Omni semantic model files" -requires-python = ">=3.9" +description = "Bidirectional converter between Apache Ossie semantic models and Omni semantic model files" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.10" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Open Semantic Interchange", + "Omni" +] dependencies = [ "PyYAML>=6.0", ] -[project.license] -text = "Apache-2.0" - -[project.optional-dependencies] -dev = [ - "pytest>=8.0", - "hypothesis>=6.0", -] - [project.scripts] osi-omni = "osi_omni.cli:main" +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + [tool.hatch.build.targets.wheel] packages = ["src/osi_omni"] [tool.pytest.ini_options] testpaths = ["tests"] -pythonpath = ["src"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev" +] \ No newline at end of file diff --git a/converters/omni/uv.lock b/converters/omni/uv.lock new file mode 100644 index 0000000..5a3af71 --- /dev/null +++ b/converters/omni/uv.lock @@ -0,0 +1,306 @@ +version = 1 +revision = 3 +requires-python = ">=3.10" + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + +[[package]] +name = "hypothesis" +version = "6.156.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/20/83/8dbe89bdb8c6f25a7a52e7898af6d82fe35dfef08e5c702f6e33231ce6c6/hypothesis-6.156.6.tar.gz", hash = "sha256:96de02faefa3ce079873541da96f42595583bb001e8e4219294ed7d4501cc4cc", size = 476304, upload-time = "2026-07-10T20:56:49.96Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/dc/0c2a851f06c91d5ac9ef0f3b9615efc1ed650411d2eee23b6334f491c85e/hypothesis-6.156.6-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:caf6a93d011c10972da111c38ceb34ced20feaa8581e2b350c0655b022e27875", size = 747998, upload-time = "2026-07-10T20:56:16.311Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f8/59203ca978ab51595d12d6bc7e7a63300d7373431ab42ca3f1742e45db68/hypothesis-6.156.6-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:07f2bc9df1aeba80e12029c1618e2ee54abc440068c305d7075ffd6b85251843", size = 743073, upload-time = "2026-07-10T20:55:36.825Z" }, + { url = "https://files.pythonhosted.org/packages/68/d8/86a0023740434098d1b187a62bd5f99b198f098fb43e7fc58342283a8270/hypothesis-6.156.6-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7baca17f4803ad4aa151732326f3990baf54c3127df44aa872ac5bdf8a98a9a6", size = 1070169, upload-time = "2026-07-10T20:55:49.47Z" }, + { url = "https://files.pythonhosted.org/packages/9b/82/673453915fd0c67673f35a4876ba88f48c621335f293f3537d77b27d4286/hypothesis-6.156.6-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8083806645f84243aade727f4978185caaa0b7190af4318673999ee15fdbf424", size = 1121760, upload-time = "2026-07-10T20:55:53.502Z" }, + { url = "https://files.pythonhosted.org/packages/8a/c3/3a5557f52912f2fecc6ed59642dcf80dd8e89d0d9664502b68e23d66bf3d/hypothesis-6.156.6-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a922eedcd8618f9c2e17b79fa7b3f3f0b2df34e201958611cc3f0f46cca33c10", size = 1111440, upload-time = "2026-07-10T20:55:43.054Z" }, + { url = "https://files.pythonhosted.org/packages/38/a6/ae636d4ca7f996a1ccb4b3d5997d949f1718fba52b01559b3ab53b237b3f/hypothesis-6.156.6-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5291bd33c4704d274d7c214d5c200e77f372a06644f5cbbe96dcbe53cb2fbf10", size = 1244944, upload-time = "2026-07-10T20:55:56.109Z" }, + { url = "https://files.pythonhosted.org/packages/1e/79/c425d22d734be0268ca60d120c6296299e4220a1783cb1a4cc76232807bb/hypothesis-6.156.6-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55f3ec50161b4a95bae63bff2b5166e45935b493013d3be30ede279bf6192318", size = 1288808, upload-time = "2026-07-10T20:56:06.249Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3a/cc9f479d22cbdd36ddfc55a978378eddadd183b09339ebdb81be33bb18e7/hypothesis-6.156.6-cp310-abi3-win32.whl", hash = "sha256:e96570ca5cdd9a5f2ff9e80a6fb2fd5420ebf33b833d7de5b09b6ebb26a3eb6c", size = 634868, upload-time = "2026-07-10T20:55:37.959Z" }, + { url = "https://files.pythonhosted.org/packages/d6/89/2008d287289841a936456cb13443ca89d88da6e4527d611d482e9544164d/hypothesis-6.156.6-cp310-abi3-win_amd64.whl", hash = "sha256:32710718c22fe8c5571464e898bb87d282837b02617d6ad68130abf7cb4843cb", size = 640382, upload-time = "2026-07-10T20:55:30.634Z" }, + { url = "https://files.pythonhosted.org/packages/0b/dc/b502504a972af7368b62e712268d310ffbc81d0ebfb31d5a9c60332a5063/hypothesis-6.156.6-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:afec0631a7a557acb50f665108b5d1d1123c21bc702d156a740db1cc33be4a7d", size = 749319, upload-time = "2026-07-10T20:55:35.708Z" }, + { url = "https://files.pythonhosted.org/packages/8f/fe/a4867bc2b8b81d9b1648992fb7e4a732b3db480ff2d02df2c7b59189c812/hypothesis-6.156.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:583a658162ee7e1a82155e3f146932a3a2269030294f3c83f5cf52fcaa0562c3", size = 743969, upload-time = "2026-07-10T20:55:31.983Z" }, + { url = "https://files.pythonhosted.org/packages/2e/22/6ece4337e01c634594bb177009c049aa3133c151d9e397edccb6c3938567/hypothesis-6.156.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9600277defbfa769d8a6af264cbdff16294682c210c2d058ef39348fec16c0c", size = 1070874, upload-time = "2026-07-10T20:55:44.277Z" }, + { url = "https://files.pythonhosted.org/packages/05/00/5820a3b1fe264b23770f77dbb3ac0f6fdadd21b640624791a05950f934da/hypothesis-6.156.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7c3f067166bfc28b67f0042fc5fdcc87b3b58664da0c2f563fe544448b7ab3b", size = 1122191, upload-time = "2026-07-10T20:56:20.721Z" }, + { url = "https://files.pythonhosted.org/packages/90/f8/0861ed15c96302577229334655e038e8c46e4b5b2a8cae971409a7e176e5/hypothesis-6.156.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f57842c1c5314839bdf4be5cff108589ff435cd7192c035dc48e6f14032915a5", size = 1246060, upload-time = "2026-07-10T20:55:41.834Z" }, + { url = "https://files.pythonhosted.org/packages/64/5c/63b60c6566eafac0b150f6acae5d1cbe0ed64dc294c863888eb87088a542/hypothesis-6.156.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c4a787c3af0035846034461792f2a62f1c43af850feb970a5b53a629d64b52fb", size = 1289510, upload-time = "2026-07-10T20:55:33.133Z" }, + { url = "https://files.pythonhosted.org/packages/96/4e/bfe57f182f51784549210903241057ae94784858117b965613089f5f89ad/hypothesis-6.156.6-cp310-cp310-win_amd64.whl", hash = "sha256:02accb187617ebaebb120da931f799a3cb0df7c38706f97f9d022441d4faf533", size = 640384, upload-time = "2026-07-10T20:56:26.939Z" }, + { url = "https://files.pythonhosted.org/packages/13/64/e4a0796190d8089e85f06731e21fdddd7e8edd3a4e562101527a048e21c4/hypothesis-6.156.6-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:5baec7943a14d106e982121dd4f74cfc5ef45e37c17f94fe49338d3d1377f38c", size = 748988, upload-time = "2026-07-10T20:56:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a2/4a789b286cd2cced31992e1f683036b51dd6909b934ea007ffb43aa3a32f/hypothesis-6.156.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b5f519905ddeb10e23b8ba2c254541a5b1a8f146fe0551be94d972f4a77226f4", size = 743754, upload-time = "2026-07-10T20:56:09.113Z" }, + { url = "https://files.pythonhosted.org/packages/d2/f7/3dd36c1c03d24ae3ffc3c5b0eca8cc4ae90c07abc320f76509eceb37019a/hypothesis-6.156.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c108655960b58ded3ca71b2dc5c69fb2ba7e9c723aeb6106facec3892d09087", size = 1070732, upload-time = "2026-07-10T20:56:28.738Z" }, + { url = "https://files.pythonhosted.org/packages/57/51/befc4b816b471078034a875eb1ef69e0411ab84bcce582b4be173258785a/hypothesis-6.156.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c8d37bebb6924729bc0bbf5852689df568842948abe4d93dd0ad3377adf76fa", size = 1121988, upload-time = "2026-07-10T20:56:07.676Z" }, + { url = "https://files.pythonhosted.org/packages/05/b7/a796f5e3e4b7cb911ff346008d49720296d1f4073490b8bc1cce6b3fbb07/hypothesis-6.156.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:74137bef6d502305c3648b2ed1a9bb4bc05fb1025e96b30a2c092204c40fe097", size = 1245596, upload-time = "2026-07-10T20:55:50.662Z" }, + { url = "https://files.pythonhosted.org/packages/37/65/849c4cba44a6f6cc888fd931124429b24180234ccc4883abab8cad5fcfcb/hypothesis-6.156.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5e0afdf79cceed20fcf0a9fb80d4064a9b2b53d4d4eecbac0e21208a13f5a31b", size = 1289172, upload-time = "2026-07-10T20:55:58.915Z" }, + { url = "https://files.pythonhosted.org/packages/13/03/7106a110df29eb631d66776e8aa8128f82f04a9dd2b6b22b612e6025e3a2/hypothesis-6.156.6-cp311-cp311-win_amd64.whl", hash = "sha256:84dc89caaf741a02f904ca7bd02b1af99650c75552868162290208aeecb70858", size = 640222, upload-time = "2026-07-10T20:56:10.396Z" }, + { url = "https://files.pythonhosted.org/packages/8c/45/9f009005b9c796f4a40424484ac7e70847bc088456fd940a937f96bb4b6d/hypothesis-6.156.6-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a2a728b514fceb81e3f0464508911d5220fd74dadc3270f859427a686b60c4cf", size = 748844, upload-time = "2026-07-10T20:56:38.036Z" }, + { url = "https://files.pythonhosted.org/packages/02/2f/4d852bb8a9c73a68b18eca9b5b085285282122166e158f4d2a477639bfee/hypothesis-6.156.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7489b9a8f9df8227edd6c7cd8b9ccfab2483bab24da6a474c175973ca2294f58", size = 741936, upload-time = "2026-07-10T20:55:27.539Z" }, + { url = "https://files.pythonhosted.org/packages/74/89/b9968070ae042f9bf3149bb6ba6399d5f28f452e0fb7f638cafc69ff0b9a/hypothesis-6.156.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42760873d6db1069d6edbaa355a61b9078a9950259efcfc72fc695741d7db7cd", size = 1069749, upload-time = "2026-07-10T20:56:43.017Z" }, + { url = "https://files.pythonhosted.org/packages/00/a9/753806f5292b40aeab1d269e408e3a7e85be3c0d88828fb78ab4a34d6626/hypothesis-6.156.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4e66aaa7385538a5d617174d47c198ee807f06de99e282a67c6cb724c69340d", size = 1120983, upload-time = "2026-07-10T20:56:25.424Z" }, + { url = "https://files.pythonhosted.org/packages/85/88/8386d064d680be27e936eba94f1448bc93ef6fa05473ee5034139f1c4284/hypothesis-6.156.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:08796b674c0b31a5dd4119b2173823390055921588d13eb77324e861b00fd7f8", size = 1243911, upload-time = "2026-07-10T20:55:54.799Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8c/7524c1e5279e7728eb47c99f2357cbc5f08ae92e9bce49bf50118b53f9c9/hypothesis-6.156.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4ca8cc26ea2d31d22cf7710e92951cfaa921f0f8aa1b6db33a5176335f583a4f", size = 1287806, upload-time = "2026-07-10T20:56:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b3/c347ad913e1c5f2988956fe17826c0400b4ce470b973e6c248e97b6a0acf/hypothesis-6.156.6-cp312-cp312-win_amd64.whl", hash = "sha256:c3363d3fb8015594636689572510bb6090602d8e8e838a5693c2d52d3b5b09d8", size = 637679, upload-time = "2026-07-10T20:55:39.056Z" }, + { url = "https://files.pythonhosted.org/packages/70/5d/9583fe153573523dac27226c89e041a86ad4aeeae08c868160cbb93d39d2/hypothesis-6.156.6-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:59a8def90d9a5a9b67e1ac529e903a2363ceb6cf873c209da6b4284c5daab671", size = 749264, upload-time = "2026-07-10T20:56:46.118Z" }, + { url = "https://files.pythonhosted.org/packages/86/35/e4113d06769b544f0fb77ffea9195b598b4c56a298905c21fd47c4eed388/hypothesis-6.156.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c574c3224563d730848bc5d1ef1683c4f83993400c0167899fe328f4bfcd4725", size = 742095, upload-time = "2026-07-10T20:56:41.412Z" }, + { url = "https://files.pythonhosted.org/packages/d8/5c/a47666ede10384e8978722cade7ab96a42df71d2ab577317092d0fed7c8a/hypothesis-6.156.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01bb8270c46b3ef53b0c2d23ff613ea506d609d06f936d823ea57c58b66b05f7", size = 1069917, upload-time = "2026-07-10T20:56:04.845Z" }, + { url = "https://files.pythonhosted.org/packages/79/93/75f6057dadd9dc0134f37c08d5d14d04d3cd7374debbcb0cc4569c6712f1/hypothesis-6.156.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4ea6559c13606e13b645927f2e0906e52b5ac5d99b40d3abaaeb2e8c7ceeb75", size = 1121204, upload-time = "2026-07-10T20:55:52.008Z" }, + { url = "https://files.pythonhosted.org/packages/62/87/308efef08bc60d1e673d035e8ca8e9663f4b6b3ba519c3cdebf6583c2b76/hypothesis-6.156.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2d47054d0230f0dd9b6868fc030126c7a6c25527144272ff376cc4e9c39f7540", size = 1244168, upload-time = "2026-07-10T20:55:40.288Z" }, + { url = "https://files.pythonhosted.org/packages/3b/66/de8fff5bd9a40a4056dafbe7f904887ef12632282bbbac90f1977c30dd3b/hypothesis-6.156.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:050c8c0815f88d47dd0875a92698d20d61639b7b721ee043a6d687c7f14ff7d8", size = 1288127, upload-time = "2026-07-10T20:56:00.541Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8d/794fb26e1fd3ff004978f8f18b7aa7e1c2270ba72e1f977b987a812064f8/hypothesis-6.156.6-cp313-cp313-win_amd64.whl", hash = "sha256:f0d73edab7b8a0051b3634f2d04d62b7e7282f8f274963b11188ee4957d672ef", size = 637954, upload-time = "2026-07-10T20:56:33.35Z" }, + { url = "https://files.pythonhosted.org/packages/a5/be/5b4b27984cb43c60e95f570b069660335dad34cb67f7d226017c5d35d31e/hypothesis-6.156.6-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:34a70a7b8226e34d658072d8fb81d03f97f0a75ceb536329a321b94ce2232fd6", size = 749312, upload-time = "2026-07-10T20:55:46.902Z" }, + { url = "https://files.pythonhosted.org/packages/31/11/709cceffc28666c9d4cb75ffc6df5ce30db8c7dd5cc2c8b38a2fd837427f/hypothesis-6.156.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f1969646beead7d8cf6a2537d2765af89d73056e2cb218e7fae92b83802250a3", size = 742332, upload-time = "2026-07-10T20:56:30.254Z" }, + { url = "https://files.pythonhosted.org/packages/84/52/cfc79b13d8dd3cd6de6b9df921c557efe8528a9c90a3a7cd93b37188d57e/hypothesis-6.156.6-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbc2ec7b7d905e6b6ec1635f6340bfa52aaab718101c59f052bc012a6b486cd8", size = 1070109, upload-time = "2026-07-10T20:55:48.244Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ac/1da4def1f006b5ad01187ff96379e24c37439d659ec10c3e944c03436c0f/hypothesis-6.156.6-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9367ae25dfa6dc1af37904785e43c4f8fe1c4118cafdc2f06514154fbdd90992", size = 1121528, upload-time = "2026-07-10T20:56:39.665Z" }, + { url = "https://files.pythonhosted.org/packages/68/47/744e4f5e3d635dea20dbedf3fa486e2a6fa5210e0a52a0d5c4da56babd84/hypothesis-6.156.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:455f09107ec07c78f2a83cb8fc19e23879c9d51cdc831de6f9cb6ec4059cb9af", size = 1244690, upload-time = "2026-07-10T20:56:31.854Z" }, + { url = "https://files.pythonhosted.org/packages/25/8a/42252fcd5e521d140dac532f29c2a13ca8f22908cb545ffdd64b5e225680/hypothesis-6.156.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c76634c45a3ceee4c4fdfed39aebd08b8b822ec8b0c556877ef82846fd777730", size = 1288519, upload-time = "2026-07-10T20:56:03.429Z" }, + { url = "https://files.pythonhosted.org/packages/44/e7/176df9e47cf583d2b8d234b78c0aac3a47075ad5d147e60b2c21a1338bb1/hypothesis-6.156.6-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:eb7e9f8343bc6b948937e6ec12e6879ed25a17b53ceccbd2b84adadd3d511698", size = 586452, upload-time = "2026-07-10T20:56:22.285Z" }, + { url = "https://files.pythonhosted.org/packages/8c/75/2c8a0411bbe76429f3ae738ef9a00107201bf6146d9534350014ce369d98/hypothesis-6.156.6-cp314-cp314-win_amd64.whl", hash = "sha256:f9631cd604ae6032c3edf99160dc1b9e33873f2e52762246b24f07fb758652ae", size = 637774, upload-time = "2026-07-10T20:56:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/2a/22/8115005e9aa72c8d63d90e9db5e0b8425fd8950fbc5d6e332805d4d32c9e/hypothesis-6.156.6-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:1f81163d36d3763b09ffaef7c3a71e88174ca3e6816201fca9d1d159f448fdb5", size = 747428, upload-time = "2026-07-10T20:56:44.611Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c2/66bfe9337f4a4b1f7754ee6d01d950280152a81d0d797e6c1d9eb0909750/hypothesis-6.156.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:556b905767e36147918634a64356aa05d8c956576f00aee01eb351678f193908", size = 740889, upload-time = "2026-07-10T20:55:57.656Z" }, + { url = "https://files.pythonhosted.org/packages/95/3b/69f45af2d4f0950b7d1af3cdbdd800b88a6c2370331481eda79d6171fbe3/hypothesis-6.156.6-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3f2604b28d16d696aaaf4954d20f907b27e54034df98e64746a20c74c319f03", size = 1069270, upload-time = "2026-07-10T20:56:12.024Z" }, + { url = "https://files.pythonhosted.org/packages/f8/43/6b2549885da08f5e50ba34fb8e0d0a60b2f190ffd516fac220f8db5b5869/hypothesis-6.156.6-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffe012ad66dbe7b8e8ddef6f6992ab1b36719ea64430c2bf1ff7135521052a15", size = 1120409, upload-time = "2026-07-10T20:55:34.551Z" }, + { url = "https://files.pythonhosted.org/packages/70/97/745c778c3eb29befa2367b1ded8437eecfbbe6932359d0f831275bc32170/hypothesis-6.156.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5bfa3c7b758f7278081c6bfec5f89b43c4eb075c0c9eb095323f7a9eb019b513", size = 1243111, upload-time = "2026-07-10T20:56:17.83Z" }, + { url = "https://files.pythonhosted.org/packages/ab/d7/c5ec6a442dc9b822f47064bda4b6d3e739dccdd1c5bf44c9a57fb6136830/hypothesis-6.156.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d0db1f4573800c618773622f03cb6533bb3377430ef938c9476ba10c39d22591", size = 1287262, upload-time = "2026-07-10T20:56:23.749Z" }, + { url = "https://files.pythonhosted.org/packages/11/0c/c134d61710e14b68b010215dcf6bd57d2ec05cd169dff8bfab8fefc2d410/hypothesis-6.156.6-cp314-cp314t-win_amd64.whl", hash = "sha256:38cd0c4a7b9f809f1e23a4d15adfa9c5d99869b9afc327350a5e563350b78e48", size = 637862, upload-time = "2026-07-10T20:56:13.347Z" }, + { url = "https://files.pythonhosted.org/packages/59/9c/b94f3a31665527b6181616b72990fcf8d6d5fa82b4187aab104ab5f548f0/hypothesis-6.156.6-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:8893b4da90e06828846c1b100c3414a7729d047a020d854c0899ae9339df0e70", size = 749575, upload-time = "2026-07-10T20:55:29.371Z" }, + { url = "https://files.pythonhosted.org/packages/21/74/dcf695f79f526543ae5d0f8c1325508e9fe990a996c0e0853129a9a5d81d/hypothesis-6.156.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b7fc0b7df9b28d028e4cc295b2ac8fbbbc22e090a23382c92fff5e37696be74f", size = 744351, upload-time = "2026-07-10T20:56:36.46Z" }, + { url = "https://files.pythonhosted.org/packages/11/d3/5bff4c55c6995a6c43f66ec8e5866b56e34f03837fd0be0e4922f3bab168/hypothesis-6.156.6-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38ed3178526382d392d04ad699ad7a2e53845e521a09d40f1cbbc1e1ff63ba48", size = 1070916, upload-time = "2026-07-10T20:56:19.256Z" }, + { url = "https://files.pythonhosted.org/packages/ba/af/5ed42117a69221ea118caaff933d8212039a0ac0bc15afa915635f13984c/hypothesis-6.156.6-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ad94e28aabf4db0d479297d43b8a2a01e7caaa9bdfccfdac7a4a3717e05b993", size = 1122625, upload-time = "2026-07-10T20:56:14.758Z" }, + { url = "https://files.pythonhosted.org/packages/3b/d3/02499badc6e3f3e980941021edf5fd780c895d8d08c9015e78516340ed83/hypothesis-6.156.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:983a5cfd955994bffc7eb02976241f7a1f3c2d94dbc3389430c45858fa5c1ae0", size = 640823, upload-time = "2026-07-10T20:55:45.461Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "osi-omni" +version = "0.2.0.dev0" +source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "hypothesis" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] + +[package.metadata.requires-dev] +dev = [ + { name = "hypothesis", specifier = ">=6.0" }, + { name = "pytest", specifier = ">=8.0" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, + { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, + { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, + { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] diff --git a/converters/orionbelt/pyproject.toml b/converters/orionbelt/pyproject.toml index 56a675e..7305fac 100644 --- a/converters/orionbelt/pyproject.toml +++ b/converters/orionbelt/pyproject.toml @@ -19,15 +19,29 @@ requires = ["hatchling"] build-backend = "hatchling.build" +[dependency-groups] +dev = [ + "pytest>=8.0", + "mypy>=1.10", + "ruff>=0.4", + "types-jsonschema", + "types-PyYAML", +] + [project] name = "apache-ossie-orionbelt" -version = "0.1.0" +version = "0.2.0.dev0" description = "Bidirectional OBML <-> OSI (Open Semantic Interchange) converter for OrionBelt semantic models" -license = { text = "Apache-2.0" } -readme = "README.md" -authors = [{ name = "Ralf Becher", email = "info@orionbelt.ai" }] -keywords = ["osi", "open-semantic-interchange", "obml", "semantic-layer", "orionbelt", "converter"] +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] requires-python = ">=3.12" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Open Semantic Interchange", + "orionbelt" +] dependencies = [ "pyyaml>=6.0", "jsonschema>=4.18", @@ -39,17 +53,14 @@ dependencies = [ # full OrionBelt engine. Optional: validate_obml degrades to JSON-schema-only # checks with a warning when this is not installed. obml-validation = ["orionbelt-semantic-layer"] -dev = [ - "pytest>=8.0", - "mypy>=1.10", - "ruff>=0.4", - "types-jsonschema", - "types-PyYAML", -] [project.scripts] ossie-orionbelt = "ossie_orionbelt.cli:main" +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + [tool.hatch.build.targets.wheel] packages = ["src/ossie_orionbelt"] @@ -66,13 +77,6 @@ packages = ["src/ossie_orionbelt"] # validation._osi_schema_path (no duplicate copy). The OSI ontology export is # validated semantically only (ossie ships no ontology schema). -[tool.ruff] -line-length = 100 -target-version = "py312" - -[tool.ruff.lint] -select = ["E", "F", "I", "N", "UP", "B", "A", "SIM"] - [tool.mypy] python_version = "3.12" files = ["src/ossie_orionbelt"] @@ -86,4 +90,16 @@ ignore_missing_imports = true [tool.pytest.ini_options] testpaths = ["tests"] -pythonpath = ["src"] + +[tool.ruff] +line-length = 100 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "UP", "B", "A", "SIM"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev" +] \ No newline at end of file diff --git a/converters/orionbelt/uv.lock b/converters/orionbelt/uv.lock index d493c99..4d2ef6f 100644 --- a/converters/orionbelt/uv.lock +++ b/converters/orionbelt/uv.lock @@ -39,7 +39,7 @@ wheels = [ [[package]] name = "apache-ossie-orionbelt" -version = "0.1.0" +version = "0.2.0.dev0" source = { editable = "." } dependencies = [ { name = "jsonschema" }, @@ -48,6 +48,11 @@ dependencies = [ ] [package.optional-dependencies] +obml-validation = [ + { name = "orionbelt-semantic-layer" }, +] + +[package.dev-dependencies] dev = [ { name = "mypy" }, { name = "pytest" }, @@ -55,23 +60,24 @@ dev = [ { name = "types-jsonschema" }, { name = "types-pyyaml" }, ] -obml-validation = [ - { name = "orionbelt-semantic-layer" }, -] [package.metadata] requires-dist = [ { name = "jsonschema", specifier = ">=4.18" }, - { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10" }, { name = "orionbelt-semantic-layer", marker = "extra == 'obml-validation'" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "referencing", specifier = ">=0.30" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.4" }, - { name = "types-jsonschema", marker = "extra == 'dev'" }, - { name = "types-pyyaml", marker = "extra == 'dev'" }, ] -provides-extras = ["obml-validation", "dev"] +provides-extras = ["obml-validation"] + +[package.metadata.requires-dev] +dev = [ + { name = "mypy", specifier = ">=1.10" }, + { name = "pytest", specifier = ">=8.0" }, + { name = "ruff", specifier = ">=0.4" }, + { name = "types-jsonschema" }, + { name = "types-pyyaml" }, +] [[package]] name = "ast-serialize" diff --git a/converters/snowflake/README.md b/converters/snowflake/README.md index 249624e..0026e0a 100644 --- a/converters/snowflake/README.md +++ b/converters/snowflake/README.md @@ -26,19 +26,19 @@ Converts Ossie YAML semantic models to [Snowflake Cortex Analyst](https://docs.s ## Setup ```bash -pip3 install -r requirements.txt +uv sync ``` ## Usage ```bash -python3 src/osi_to_snowflake_yaml_converter.py -i input.yaml -o output.yaml +uv run ossie-snowflake -i input.yaml -o output.yaml ``` ## Tests ```bash -python3 -m pytest tests/ +uv run pytest ``` ## Limitations diff --git a/converters/snowflake/pyproject.toml b/converters/snowflake/pyproject.toml new file mode 100644 index 0000000..0523f39 --- /dev/null +++ b/converters/snowflake/pyproject.toml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.0", +] + +[project] +name = "apache-ossie-snowflake" +version = "0.2.0.dev0" +description = "Snowflake Cortex Analyst <> Apache Ossie converter" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.11" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Open Semantic Interchange", + "Snowflake" +] +dependencies = [ + "PyYAML>=5.0", +] + +[project.scripts] +ossie-snowflake = "ossie_snowflake.converter:main" + +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_snowflake"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev" +] \ No newline at end of file diff --git a/converters/snowflake/src/ossie_snowflake/__init__.py b/converters/snowflake/src/ossie_snowflake/__init__.py new file mode 100644 index 0000000..d216be4 --- /dev/null +++ b/converters/snowflake/src/ossie_snowflake/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. \ No newline at end of file diff --git a/converters/snowflake/src/osi_to_snowflake_yaml_converter.py b/converters/snowflake/src/ossie_snowflake/converter.py similarity index 99% rename from converters/snowflake/src/osi_to_snowflake_yaml_converter.py rename to converters/snowflake/src/ossie_snowflake/converter.py index 196a1d3..447827d 100644 --- a/converters/snowflake/src/osi_to_snowflake_yaml_converter.py +++ b/converters/snowflake/src/ossie_snowflake/converter.py @@ -21,7 +21,7 @@ connection required. Usage: - python3 osi_to_snowflake_yaml_converter.py -i input.yaml -o output.yaml + python3 converter.py -i input.yaml -o output.yaml """ import argparse diff --git a/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py b/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py index 075aeeb..c1b5469 100644 --- a/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py +++ b/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py @@ -17,16 +17,12 @@ """Tests for the Ossie to Snowflake YAML converter.""" -import sys import warnings -from pathlib import Path import pytest import yaml -# Make src/ importable -sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) -from osi_to_snowflake_yaml_converter import ( +from ossie_snowflake.converter import ( OsiConversionError, convert_osi_to_snowflake, _classify_field, diff --git a/converters/snowflake/uv.lock b/converters/snowflake/uv.lock new file mode 100644 index 0000000..eaa7da9 --- /dev/null +++ b/converters/snowflake/uv.lock @@ -0,0 +1,138 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "apache-ossie-snowflake" +version = "0.2.0.dev0" +source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [{ name = "pyyaml", specifier = ">=5.0" }] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.0" }] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..38af97d --- /dev/null +++ b/python/README.md @@ -0,0 +1,38 @@ + + +# Apache Ossie Python Package + +The Apache Ossie Python package provides Pydantic v2 models for the Apache Ossie semantic model specification. It is the shared fundation used by Apache Ossie converters to parse, construct, validate, and serialize OSI documents from Python application. + +## Development + +### Prerequisites +- Python 3.11 or later +- uv >= 0.9.0 + +### Installation +```bash +uv sync +``` + +### Generating package distributions +```bash +uv build +``` \ No newline at end of file diff --git a/python/pyproject.toml b/python/pyproject.toml index d209859..7dab9b6 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -23,23 +23,26 @@ build-backend = "hatchling.build" name = "apache-ossie" version = "0.2.0.dev0" description = "Python types for the Apache Ossie semantic model specification" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] requires-python = ">=3.11" -classifiers = [ - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Open Semantic Interchange", ] dependencies = [ "pydantic>=2.0", "PyYAML>=6.0", ] -[project.license] -text = "Apache-2.0" +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" [tool.hatch.build.targets.wheel] packages = ["src/ossie"] [tool.uv] -dev-dependencies = [ - "pytest>=8.0", -] +required-version = ">=0.9.0" \ No newline at end of file diff --git a/python/uv.lock b/python/uv.lock new file mode 100644 index 0000000..17d1769 --- /dev/null +++ b/python/uv.lock @@ -0,0 +1,220 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "apache-ossie" +version = "0.2.0.dev0" +source = { editable = "." } +dependencies = [ + { name = "pydantic" }, + { name = "pyyaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "pydantic", specifier = ">=2.0" }, + { name = "pyyaml", specifier = ">=6.0" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] From 11f0f2b27557658917a7130580d41027d6039d6c Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Fri, 10 Jul 2026 15:47:58 -0700 Subject: [PATCH 86/89] Add WisdomAI domain export converter Adds converters/wisdom, a one-way converter from a WisdomAI domain export JSON (format 1.0) to an Ossie semantic model YAML, following the layout of the dbt converter. Mapping: domain -> semantic model; domain knowledge and system instructions -> model-level ai_context; tables/columns/formulas -> datasets/fields; relationship graph -> relationships (cardinality folded into edge direction, compound AND-joins flattened to composite keys); per-table measures -> model-level metrics. Expressions are emitted verbatim under the dialect mapped from the table's connection (snowflake, databricks), falling back to ANSI_SQL with a warning for unsupported dialects. Information loss is reported through the same ConverterResult/ConverterIssue pattern as ossie-dbt. Co-Authored-By: Claude Fable 5 --- converters/wisdom/README.md | 89 +++++ converters/wisdom/pyproject.toml | 51 +++ .../wisdom/src/ossie_wisdom/__init__.py | 26 ++ converters/wisdom/src/ossie_wisdom/cli.py | 80 ++++ .../src/ossie_wisdom/converter_issues.py | 50 +++ .../wisdom/src/ossie_wisdom/wisdom_to_osi.py | 358 ++++++++++++++++++ converters/wisdom/tests/__init__.py | 16 + .../wisdom/tests/fixtures/sample_export.json | 227 +++++++++++ converters/wisdom/tests/test_wisdom_to_osi.py | 167 ++++++++ 9 files changed, 1064 insertions(+) create mode 100644 converters/wisdom/README.md create mode 100644 converters/wisdom/pyproject.toml create mode 100644 converters/wisdom/src/ossie_wisdom/__init__.py create mode 100644 converters/wisdom/src/ossie_wisdom/cli.py create mode 100644 converters/wisdom/src/ossie_wisdom/converter_issues.py create mode 100644 converters/wisdom/src/ossie_wisdom/wisdom_to_osi.py create mode 100644 converters/wisdom/tests/__init__.py create mode 100644 converters/wisdom/tests/fixtures/sample_export.json create mode 100644 converters/wisdom/tests/test_wisdom_to_osi.py diff --git a/converters/wisdom/README.md b/converters/wisdom/README.md new file mode 100644 index 0000000..166c356 --- /dev/null +++ b/converters/wisdom/README.md @@ -0,0 +1,89 @@ + + +# WisdomAI → Apache Ossie Converter + +Converts a [WisdomAI](https://wisdom.ai) domain export (format `1.0`, produced by wisdom's +`exportDomain` API) into an Ossie semantic model YAML document. + +## Usage + +```bash +pip install -e ../../python -e . +ossie-wisdom wisdom-to-osi -i domain-export.json -o semantic_model.yaml +``` + +Conversion warnings (information loss) are printed to stderr; the output YAML validates +against the [Ossie JSON Schema](../../core-spec/osi-schema.json): + +```bash +python ../../validation/validate.py semantic_model.yaml --schema ../../core-spec/osi-schema.json +``` + +## Field mapping + +| Ossie | Wisdom | +|-------|--------| +| `semantic_model[].name` | domain `ref.name` | +| `semantic_model[].description` | domain `description` | +| `semantic_model[].ai_context` | `domainSystemInstructions` + each domain `knowledge[].content` as a bulleted list | +| `datasets[].name` | table `ref.name` | +| `datasets[].source` | table `location.database.schema.dbTable` | +| `datasets[].description` | table `description` | +| `datasets[].primary_key` | table `primaryKey.columns`, else columns flagged `isPrimaryKey` | +| `datasets[].fields[]` | table `columns[]` (expression = bare column name) and `formulas[]` (expression verbatim) | +| `fields[].label` | column/formula `properties.displayName` | +| `fields[].dimension.is_time` | set when `properties.dataType` is `DATE`, `DATETIME`, or `TIMESTAMP` | +| `relationships[]` | domain `relationshipGraph.relationships[]` (see cardinality below) | +| `metrics[]` | every table's `measures[]`, hoisted to the model level | + +Expressions are emitted verbatim under the Ossie dialect mapped from the table's connection +(`snowflake → SNOWFLAKE`, `databricks → DATABRICKS`). Connections with any other dialect fall +back to `ANSI_SQL` verbatim with an `UNSUPPORTED_DIALECT` warning. + +### Relationship cardinality + +Ossie encodes cardinality by direction (`from` = many side, `to` = one side), so wisdom's +`relationshipType` is folded into the edge direction: + +| Wisdom `relationshipType` | Ossie | +|---------------------------|-------| +| `MANY_TO_ONE` | `from` = left, `to` = right | +| `ONE_TO_MANY` | `from` = right, `to` = left | +| `ONE_TO_ONE` | `from` = left, plus an `ai_context` note (direction is arbitrary) | +| `MANY_TO_MANY` | `from` = left, plus an `ai_context` note and a `CARDINALITY_LOSS` warning | + +Compound join conditions that are an `AND` of equality conditions are flattened into +positional `from_columns`/`to_columns` arrays; any other compound condition (e.g. `OR`) +drops the relationship with a `RELATIONSHIP_DROPPED` warning. + +## Known limitations + +- One-way only (wisdom → Ossie). No Ossie → wisdom export yet. +- Hidden columns and stale measures are converted anyway (with a `STALE_MEASURE` warning + for the latter), so the output may reference columns wisdom itself hides from querying. +- Out of scope for now: reviewed queries, recommended questions, synonym sets, row-level + security config, per-knowledge schema annotations, column enum values, and LLM prompts. + +## Development + +```bash +pip install -e ../../python -e . pytest +pytest tests/ +``` diff --git a/converters/wisdom/pyproject.toml b/converters/wisdom/pyproject.toml new file mode 100644 index 0000000..7cb7701 --- /dev/null +++ b/converters/wisdom/pyproject.toml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "apache-ossie-wisdom" +version = "0.2.0.dev0" +description = "WisdomAI domain export -> Apache Ossie converter" +requires-python = ">=3.11" +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", +] +dependencies = [ + "apache-ossie>=0.2.0.dev0", + "PyYAML>=6.0", +] + +[project.license] +text = "Apache-2.0" + +[project.scripts] +ossie-wisdom = "ossie_wisdom.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_wisdom"] + +[tool.uv] +dev-dependencies = [ + "pytest>=8.0", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/converters/wisdom/src/ossie_wisdom/__init__.py b/converters/wisdom/src/ossie_wisdom/__init__.py new file mode 100644 index 0000000..94153f4 --- /dev/null +++ b/converters/wisdom/src/ossie_wisdom/__init__.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from ossie_wisdom.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult +from ossie_wisdom.wisdom_to_osi import WisdomToOSIConverter + +__all__ = [ + "ConverterIssue", + "ConverterIssueType", + "ConverterResult", + "WisdomToOSIConverter", +] diff --git a/converters/wisdom/src/ossie_wisdom/cli.py b/converters/wisdom/src/ossie_wisdom/cli.py new file mode 100644 index 0000000..0274e62 --- /dev/null +++ b/converters/wisdom/src/ossie_wisdom/cli.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""CLI entry point for the ossie-wisdom converter. + +Usage: + ossie-wisdom wisdom-to-osi -i domain-export.json -o output.yaml +""" + +import argparse +import json +import sys +from pathlib import Path + +from ossie_wisdom.converter_issues import ConverterIssueType +from ossie_wisdom.wisdom_to_osi import WisdomToOSIConverter + +_ISSUE_REASON: dict[ConverterIssueType, str] = { + ConverterIssueType.UNSUPPORTED_DIALECT: "the connection dialect has no Ossie equivalent; expressions were emitted verbatim as ANSI_SQL", + ConverterIssueType.CARDINALITY_LOSS: "Ossie relationships encode cardinality by direction and cannot represent many-to-many", + ConverterIssueType.RELATIONSHIP_DROPPED: "the relationship uses a join Ossie cannot represent (non-table source, OR/non-equi condition, or unknown dataset)", + ConverterIssueType.METRIC_NAME_COLLISION: "another table defines a measure with the same name; this one was prefixed with its table name", + ConverterIssueType.STALE_MEASURE: "wisdom marked this measure stale; it was converted anyway", + ConverterIssueType.DUPLICATE_FIELD_DROPPED: "the dataset already has a field with this name", +} + +_DROPPED_ISSUE_TYPES = { + ConverterIssueType.RELATIONSHIP_DROPPED, + ConverterIssueType.DUPLICATE_FIELD_DROPPED, +} + + +def _cmd_wisdom_to_osi(args: argparse.Namespace) -> None: + input_path = Path(args.input) + output_path = Path(args.output) + + export = json.loads(input_path.read_text()) + result = WisdomToOSIConverter().convert(export) + + for issue in result.issues: + verb = "was dropped" if issue.issue_type in _DROPPED_ISSUE_TYPES else "was converted with loss" + reason = _ISSUE_REASON[issue.issue_type] + print(f"[WARNING] {issue.issue_type.value}: {issue.element_name} {verb} during conversion because {reason}", file=sys.stderr) + + output_path.write_text(result.output.to_osi_yaml()) + print(f"Written to {output_path}", file=sys.stderr) + + +def main() -> None: + parser = argparse.ArgumentParser( + prog="ossie-wisdom", + description="Convert a WisdomAI domain export JSON to Ossie YAML.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + wisdom_to_osi = subparsers.add_parser("wisdom-to-osi", help="Convert domain-export.json → Ossie YAML") + wisdom_to_osi.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to wisdom domain export JSON") + wisdom_to_osi.add_argument("-o", "--output", required=True, metavar="FILE", help="Path for output Ossie YAML") + + args = parser.parse_args() + if args.command == "wisdom-to-osi": + _cmd_wisdom_to_osi(args) + + +if __name__ == "__main__": + main() diff --git a/converters/wisdom/src/ossie_wisdom/converter_issues.py b/converters/wisdom/src/ossie_wisdom/converter_issues.py new file mode 100644 index 0000000..f8c8fb2 --- /dev/null +++ b/converters/wisdom/src/ossie_wisdom/converter_issues.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass +from enum import Enum +from typing import Generic, List, TypeVar + + +class ConverterIssueType(Enum): + """Identifies the kind of information loss that occurred during conversion.""" + + UNSUPPORTED_DIALECT = "UNSUPPORTED_DIALECT" + CARDINALITY_LOSS = "CARDINALITY_LOSS" + RELATIONSHIP_DROPPED = "RELATIONSHIP_DROPPED" + METRIC_NAME_COLLISION = "METRIC_NAME_COLLISION" + STALE_MEASURE = "STALE_MEASURE" + DUPLICATE_FIELD_DROPPED = "DUPLICATE_FIELD_DROPPED" + + +@dataclass(frozen=True) +class ConverterIssue: + """Records a single instance of information loss during conversion.""" + + issue_type: ConverterIssueType + element_name: str + + +T = TypeVar("T") + + +@dataclass(frozen=True) +class ConverterResult(Generic[T]): + """Return value of a converter's convert() method, pairing the output with any conversion issues.""" + + output: T + issues: List[ConverterIssue] diff --git a/converters/wisdom/src/ossie_wisdom/wisdom_to_osi.py b/converters/wisdom/src/ossie_wisdom/wisdom_to_osi.py new file mode 100644 index 0000000..855641d --- /dev/null +++ b/converters/wisdom/src/ossie_wisdom/wisdom_to_osi.py @@ -0,0 +1,358 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Converts a WisdomAI domain export (format 1.0) into an Ossie Document. + +The input is the JSON produced by wisdom's ``exportDomain`` RPC: a wrapper +holding the domain ZSheet, one ZSheet per table, and connection metadata, +all serialized as protobuf-JSON (camelCase keys). +""" + +import re +from typing import Dict, List, Optional, Set, Tuple + +from ossie import ( + OSIDataset, + OSIDialect, + OSIDialectExpression, + OSIDimension, + OSIDocument, + OSIExpression, + OSIField, + OSIMetric, + OSIRelationship, + OSISemanticModel, +) +from ossie_wisdom.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult + +_DIALECT_MAP: Dict[str, OSIDialect] = { + "snowflake": OSIDialect.SNOWFLAKE, + "databricks": OSIDialect.DATABRICKS, + "ansi": OSIDialect.ANSI_SQL, + "ansi_sql": OSIDialect.ANSI_SQL, +} + +_TIME_DATA_TYPES = {"DATE", "DATETIME", "TIMESTAMP"} + +_SIMPLE_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +class WisdomToOSIConverter: + """Converts a wisdom domain export dict into an Ossie Document.""" + + def convert(self, export: dict) -> ConverterResult[OSIDocument]: + issues: List[ConverterIssue] = [] + + domain = export.get("domain", {}).get("zsheet_json", {}) + dialect_by_connection = self._build_dialect_index(export, issues) + + datasets = [] + metrics: List[OSIMetric] = [] + for table in export.get("tables", []): + zsheet = table.get("zsheet_json", {}) + dialect = self._resolve_dialect(zsheet, dialect_by_connection) + datasets.append(self._convert_table(zsheet, dialect, issues)) + metrics.extend(self._convert_measures(zsheet, dialect, issues)) + if not datasets: + raise ValueError("Export contains no tables; an Ossie semantic model requires at least one dataset.") + metrics = self._dedupe_metrics(metrics, issues) + + dataset_names = {d.name for d in datasets} + relationships = self._convert_relationships(domain, dataset_names, issues) + + model = OSISemanticModel( + name=domain.get("ref", {}).get("name") or export.get("export_metadata", {}).get("domain_name", "domain"), + description=domain.get("description") or None, + ai_context=self._build_ai_context(domain), + datasets=datasets, + relationships=relationships or None, + metrics=[metric for _, metric in metrics] or None, + ) + return ConverterResult(output=OSIDocument(semantic_model=[model]), issues=issues) + + def _build_dialect_index(self, export: dict, issues: List[ConverterIssue]) -> Dict[str, OSIDialect]: + index: Dict[str, OSIDialect] = {} + for connection in export.get("connections", []): + dialect_name = connection.get("dialect", "").lower() + dialect = _DIALECT_MAP.get(dialect_name) + if dialect is None: + dialect = OSIDialect.ANSI_SQL + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.UNSUPPORTED_DIALECT, + element_name=f"{connection.get('name', connection.get('connection_id', '?'))} ({dialect_name})", + ) + ) + index[connection.get("connection_id", "")] = dialect + return index + + def _resolve_dialect(self, zsheet: dict, dialect_by_connection: Dict[str, OSIDialect]) -> OSIDialect: + connection_id = zsheet.get("location", {}).get("connectionId", "") + return dialect_by_connection.get(connection_id, OSIDialect.ANSI_SQL) + + def _build_ai_context(self, domain: dict) -> Optional[str]: + parts: List[str] = [] + system_instructions = domain.get("domainSystemInstructions", "").strip() + if system_instructions: + parts.append(system_instructions) + for knowledge in domain.get("knowledge", []): + content = knowledge.get("content", "").strip() + if content: + parts.append(f"- {content}") + return "\n".join(parts) or None + + def _convert_table(self, zsheet: dict, dialect: OSIDialect, issues: List[ConverterIssue]) -> OSIDataset: + name = zsheet.get("ref", {}).get("name", "") + location = zsheet.get("location", {}) + source = ".".join( + part for part in (location.get("database"), location.get("schema"), location.get("dbTable")) if part + ) + + fields: List[OSIField] = [] + seen: Set[str] = set() + for column in zsheet.get("columns", []): + field = self._convert_column(column, dialect) + if field.name in seen: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.DUPLICATE_FIELD_DROPPED, element_name=f"{name}.{field.name}" + ) + ) + continue + seen.add(field.name) + fields.append(field) + for formula in zsheet.get("formulas", []): + field = self._convert_formula(formula, dialect) + if field is None: + continue + if field.name in seen: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.DUPLICATE_FIELD_DROPPED, element_name=f"{name}.{field.name}" + ) + ) + continue + seen.add(field.name) + fields.append(field) + + return OSIDataset( + name=name, + source=source, + primary_key=self._extract_primary_key(zsheet), + description=zsheet.get("description") or None, + fields=fields or None, + ) + + def _extract_primary_key(self, zsheet: dict) -> Optional[List[str]]: + primary_key = zsheet.get("primaryKey", {}).get("columns") + if primary_key: + return list(primary_key) + flagged = [ + column["name"] + for column in zsheet.get("columns", []) + if column.get("properties", {}).get("isPrimaryKey") + ] + return flagged or None + + def _convert_column(self, column: dict, dialect: OSIDialect) -> OSIField: + properties = column.get("properties", {}) + return OSIField( + name=column.get("name", ""), + expression=self._make_expression(self._quote_identifier(column.get("name", ""), dialect), dialect), + dimension=OSIDimension(is_time=True) if properties.get("dataType") in _TIME_DATA_TYPES else None, + label=properties.get("displayName") or None, + description=column.get("description") or None, + ) + + def _convert_formula(self, formula: dict, dialect: OSIDialect) -> Optional[OSIField]: + name = formula.get("name", "") + expression = formula.get("expression", "") + if not name or not expression: + return None + properties = formula.get("properties", {}) + return OSIField( + name=name, + expression=self._make_expression(expression, dialect), + dimension=OSIDimension(is_time=True) if properties.get("dataType") in _TIME_DATA_TYPES else None, + label=properties.get("displayName") or None, + description=formula.get("description") or None, + ) + + def _convert_measures( + self, zsheet: dict, dialect: OSIDialect, issues: List[ConverterIssue] + ) -> List[Tuple[str, OSIMetric]]: + table_name = zsheet.get("ref", {}).get("name", "") + metrics: List[Tuple[str, OSIMetric]] = [] + for measure in zsheet.get("measures", []): + name = measure.get("name", "") + expression = measure.get("expression", "") + if not name or not expression: + continue + if measure.get("staleReason"): + issues.append( + ConverterIssue(issue_type=ConverterIssueType.STALE_MEASURE, element_name=f"{table_name}.{name}") + ) + metrics.append( + ( + table_name, + OSIMetric( + name=name, + expression=self._make_expression(expression, dialect), + description=measure.get("description") or None, + ), + ) + ) + return metrics + + def _dedupe_metrics( + self, metrics: List[Tuple[str, OSIMetric]], issues: List[ConverterIssue] + ) -> List[Tuple[str, OSIMetric]]: + result: List[Tuple[str, OSIMetric]] = [] + seen: Set[str] = set() + for table_name, metric in metrics: + name = metric.name + if name in seen: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.METRIC_NAME_COLLISION, element_name=f"{table_name}.{name}" + ) + ) + name = f"{table_name}_{name}" + suffix = 2 + while name in seen: + name = f"{table_name}_{metric.name}_{suffix}" + suffix += 1 + metric = metric.model_copy(update={"name": name}) + seen.add(name) + result.append((table_name, metric)) + return result + + def _convert_relationships( + self, domain: dict, dataset_names: Set[str], issues: List[ConverterIssue] + ) -> List[OSIRelationship]: + relationships: List[OSIRelationship] = [] + seen_names: Set[str] = set() + edges = domain.get("relationshipGraph", {}).get("relationships", []) + for edge in edges: + left = edge.get("leftDataSource", {}).get("zsheet", {}).get("name") + right = edge.get("rightDataSource", {}).get("zsheet", {}).get("name") + properties = edge.get("properties", {}) + column_pairs = self._extract_column_pairs(properties) + if not left or not right or column_pairs is None: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.RELATIONSHIP_DROPPED, + element_name=f"{left or '?'} <-> {right or '?'}", + ) + ) + continue + if left not in dataset_names or right not in dataset_names: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.RELATIONSHIP_DROPPED, element_name=f"{left} <-> {right}" + ) + ) + continue + + relationship_type = properties.get("relationshipType", "") + ai_context: Optional[str] = None + # Ossie encodes cardinality by direction: `from` is the many side, `to` the one side. + if relationship_type == "ONE_TO_MANY": + from_dataset, to_dataset = right, left + from_columns = [pair[1] for pair in column_pairs] + to_columns = [pair[0] for pair in column_pairs] + else: + from_dataset, to_dataset = left, right + from_columns = [pair[0] for pair in column_pairs] + to_columns = [pair[1] for pair in column_pairs] + if relationship_type == "ONE_TO_ONE": + ai_context = "one-to-one relationship" + elif relationship_type == "MANY_TO_MANY": + ai_context = "many-to-many relationship; cardinality is not representable in Ossie" + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.CARDINALITY_LOSS, element_name=f"{left} <-> {right}" + ) + ) + + name = f"{from_dataset}_to_{to_dataset}" + suffix = 2 + while name in seen_names: + name = f"{from_dataset}_to_{to_dataset}_{suffix}" + suffix += 1 + seen_names.add(name) + + relationships.append( + OSIRelationship( + name=name, + from_dataset=from_dataset, + to=to_dataset, + from_columns=from_columns, + to_columns=to_columns, + ai_context=ai_context, + ) + ) + return relationships + + def _extract_column_pairs(self, properties: dict) -> Optional[List[Tuple[str, str]]]: + """Returns (left_column, right_column) pairs, or None when the condition cannot be represented.""" + condition = properties.get("joinCondition") + if condition is not None: + pair = self._simple_condition_pair(condition) + return [pair] if pair else None + compound = properties.get("compoundJoinCondition") + if compound is not None: + return self._flatten_compound_condition(compound) + return None + + def _simple_condition_pair(self, condition: dict) -> Optional[Tuple[str, str]]: + left = condition.get("leftColumn", {}).get("name") + right = condition.get("rightColumn", {}).get("name") + # The only join operator wisdom emits is EQUAL (proto default, omitted in JSON). + if condition.get("operator") not in (None, "EQUAL") or not left or not right: + return None + return (left, right) + + def _flatten_compound_condition(self, compound: dict) -> Optional[List[Tuple[str, str]]]: + """Flattens an AND-of-equals compound condition into column pairs; None for anything else.""" + simple = compound.get("simpleCondition") + if simple is not None: + pair = self._simple_condition_pair(simple) + return [pair] if pair else None + nested = compound.get("nestedCondition") + if nested is not None: + if nested.get("logicalOperator") != "AND": + return None + pairs: List[Tuple[str, str]] = [] + for child in nested.get("conditions", []): + child_pairs = self._flatten_compound_condition(child) + if child_pairs is None: + return None + pairs.extend(child_pairs) + return pairs or None + return None + + def _quote_identifier(self, name: str, dialect: OSIDialect) -> str: + """Quotes a column name for use as a SQL expression when it is not a plain identifier.""" + if _SIMPLE_IDENTIFIER.match(name): + return name + if dialect is OSIDialect.DATABRICKS: + return "`" + name.replace("`", "``") + "`" + return '"' + name.replace('"', '""') + '"' + + def _make_expression(self, expression: str, dialect: OSIDialect) -> OSIExpression: + return OSIExpression(dialects=[OSIDialectExpression(dialect=dialect, expression=expression)]) diff --git a/converters/wisdom/tests/__init__.py b/converters/wisdom/tests/__init__.py new file mode 100644 index 0000000..13a8339 --- /dev/null +++ b/converters/wisdom/tests/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/converters/wisdom/tests/fixtures/sample_export.json b/converters/wisdom/tests/fixtures/sample_export.json new file mode 100644 index 0000000..ebe256c --- /dev/null +++ b/converters/wisdom/tests/fixtures/sample_export.json @@ -0,0 +1,227 @@ +{ + "version": "1.0", + "export_metadata": { + "exported_at": "2026-07-10T00:00:00+00:00", + "source_domain_id": "ET_DOMAIN_sample", + "domain_name": "Sample Sales" + }, + "domain": { + "zsheet_json": { + "ref": {"uuid": "ET_DOMAIN_sample", "name": "Sample Sales", "version": "1"}, + "description": "Synthetic sales domain used for converter tests", + "zsheetType": "DOMAIN", + "domainSystemInstructions": "Only answer questions about sales data.", + "knowledge": [ + { + "name": "Fiscal year", + "content": "The fiscal year starts in February.", + "id": "ET_UNSTRUCTURED_KNOWLEDGE_1" + }, + { + "name": "Pipeline", + "content": "Pipeline refers to open orders expected to close this quarter.", + "id": "ET_UNSTRUCTURED_KNOWLEDGE_2", + "schemaAnnotation": {"relevance": "NOT_SCHEMA_RELATED"} + } + ], + "relationshipGraph": { + "zsheets": [ + {"uuid": "ET_ZSHEET_orders", "name": "orders", "version": "1"}, + {"uuid": "ET_ZSHEET_customers", "name": "customers", "version": "1"}, + {"uuid": "ET_ZSHEET_tags", "name": "tags", "version": "1"} + ], + "relationships": [ + { + "properties": { + "joinCondition": { + "leftColumn": {"name": "customer_id", "zsheetRef": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightColumn": {"name": "customer_id", "zsheetRef": {"uuid": "ET_ZSHEET_customers", "name": "customers"}} + }, + "relationshipType": "MANY_TO_ONE" + }, + "leftDataSource": {"zsheet": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightDataSource": {"zsheet": {"uuid": "ET_ZSHEET_customers", "name": "customers"}} + }, + { + "properties": { + "joinCondition": { + "leftColumn": {"name": "customer_id", "zsheetRef": {"uuid": "ET_ZSHEET_customers", "name": "customers"}}, + "rightColumn": {"name": "customer_id", "zsheetRef": {"uuid": "ET_ZSHEET_orders", "name": "orders"}} + }, + "relationshipType": "ONE_TO_MANY" + }, + "leftDataSource": {"zsheet": {"uuid": "ET_ZSHEET_customers", "name": "customers"}}, + "rightDataSource": {"zsheet": {"uuid": "ET_ZSHEET_orders", "name": "orders"}} + }, + { + "properties": { + "joinCondition": { + "leftColumn": {"name": "order_id", "zsheetRef": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightColumn": {"name": "order_id", "zsheetRef": {"uuid": "ET_ZSHEET_tags", "name": "tags"}} + }, + "relationshipType": "MANY_TO_MANY" + }, + "leftDataSource": {"zsheet": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightDataSource": {"zsheet": {"uuid": "ET_ZSHEET_tags", "name": "tags"}} + }, + { + "properties": { + "compoundJoinCondition": { + "nestedCondition": { + "logicalOperator": "AND", + "conditions": [ + { + "simpleCondition": { + "leftColumn": {"name": "order_id", "zsheetRef": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightColumn": {"name": "order_id", "zsheetRef": {"uuid": "ET_ZSHEET_tags", "name": "tags"}} + } + }, + { + "simpleCondition": { + "leftColumn": {"name": "customer_id", "zsheetRef": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightColumn": {"name": "customer_id", "zsheetRef": {"uuid": "ET_ZSHEET_tags", "name": "tags"}} + } + } + ] + } + }, + "relationshipType": "MANY_TO_ONE" + }, + "leftDataSource": {"zsheet": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightDataSource": {"zsheet": {"uuid": "ET_ZSHEET_tags", "name": "tags"}} + }, + { + "properties": { + "compoundJoinCondition": { + "nestedCondition": { + "logicalOperator": "OR", + "conditions": [ + { + "simpleCondition": { + "leftColumn": {"name": "order_id", "zsheetRef": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightColumn": {"name": "order_id", "zsheetRef": {"uuid": "ET_ZSHEET_tags", "name": "tags"}} + } + } + ] + } + }, + "relationshipType": "MANY_TO_ONE" + }, + "leftDataSource": {"zsheet": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightDataSource": {"zsheet": {"uuid": "ET_ZSHEET_tags", "name": "tags"}} + } + ] + } + } + }, + "tables": [ + { + "zsheet_uuid": "ET_ZSHEET_orders", + "zsheet_json": { + "ref": {"uuid": "ET_ZSHEET_orders", "name": "orders", "version": "1"}, + "description": "Customer orders", + "location": { + "database": "analytics", + "schema": "sales", + "dbTable": "orders", + "connectionId": "et-connection-snowflake" + }, + "columns": [ + {"name": "order_id", "properties": {"dataType": "INT64", "isPrimaryKey": true}}, + {"name": "customer_id", "properties": {"dataType": "INT64"}}, + {"name": "order_date", "properties": {"dataType": "DATE"}}, + {"name": "amount", "properties": {"dataType": "DOUBLE", "visibility": "HIDDEN"}}, + { + "name": "status", + "description": "Current order status", + "properties": {"dataType": "VARCHAR", "displayName": "Order Status", "isEnum": true} + }, + {"name": "Discount - Percent", "properties": {"dataType": "DOUBLE"}} + ], + "formulas": [ + { + "name": "is_large", + "expression": "CASE WHEN \"orders\".\"amount\" > 100 THEN TRUE ELSE FALSE END", + "description": "true when the order amount exceeds 100", + "properties": {"dataType": "BOOL", "displayName": "Is Large"}, + "id": "FORMULA_1" + } + ], + "measures": [ + { + "name": "total_amount", + "expression": "SUM(\"orders\".\"amount\")", + "description": "Total order amount", + "staleReason": "Column orders.amount is hidden in this domain and cannot be queried.", + "id": "MEASURE_1" + } + ] + } + }, + { + "zsheet_uuid": "ET_ZSHEET_customers", + "zsheet_json": { + "ref": {"uuid": "ET_ZSHEET_customers", "name": "customers", "version": "1"}, + "description": "Customer master data", + "location": { + "database": "analytics", + "schema": "sales", + "dbTable": "customers", + "connectionId": "et-connection-snowflake" + }, + "primaryKey": {"columns": ["customer_id"]}, + "columns": [ + {"name": "customer_id", "properties": {"dataType": "INT64"}}, + {"name": "name", "properties": {"dataType": "VARCHAR"}}, + {"name": "region", "properties": {"dataType": "VARCHAR"}} + ], + "formulas": [ + { + "name": "region", + "expression": "UPPER(\"customers\".\"region\")", + "properties": {"dataType": "VARCHAR"}, + "id": "FORMULA_2" + } + ], + "measures": [ + { + "name": "total_amount", + "expression": "COUNT(DISTINCT \"customers\".\"customer_id\")", + "description": "Distinct customer count (name collides with the orders measure)", + "id": "MEASURE_2" + } + ] + } + }, + { + "zsheet_uuid": "ET_ZSHEET_tags", + "zsheet_json": { + "ref": {"uuid": "ET_ZSHEET_tags", "name": "tags", "version": "1"}, + "description": "Order tags", + "location": { + "database": "tagdb", + "schema": "public", + "dbTable": "tags", + "connectionId": "et-connection-postgres" + }, + "columns": [ + {"name": "tag_id", "properties": {"dataType": "INT64"}}, + {"name": "order_id", "properties": {"dataType": "INT64"}}, + {"name": "customer_id", "properties": {"dataType": "INT64"}} + ] + } + } + ], + "connections": [ + {"connection_id": "et-connection-snowflake", "dialect": "snowflake", "name": "Snowflake"}, + {"connection_id": "et-connection-postgres", "dialect": "postgres", "name": "Postgres"} + ], + "reviewed_queries": {"ref": null, "items_json": "{}"}, + "synonym_sets": {"ref": null, "items_json": "{}"}, + "table_metadata": [ + {"zsheet_uuid": "ET_ZSHEET_orders", "connection_id": "et-connection-snowflake", "database": "analytics", "schema": "sales", "table_name": "orders"}, + {"zsheet_uuid": "ET_ZSHEET_customers", "connection_id": "et-connection-snowflake", "database": "analytics", "schema": "sales", "table_name": "customers"}, + {"zsheet_uuid": "ET_ZSHEET_tags", "connection_id": "et-connection-postgres", "database": "tagdb", "schema": "public", "table_name": "tags"} + ], + "recommended_questions": ["What is the total order amount by region?"] +} diff --git a/converters/wisdom/tests/test_wisdom_to_osi.py b/converters/wisdom/tests/test_wisdom_to_osi.py new file mode 100644 index 0000000..eaa1c2d --- /dev/null +++ b/converters/wisdom/tests/test_wisdom_to_osi.py @@ -0,0 +1,167 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +from pathlib import Path + +import pytest +import yaml + +from ossie import OSIDialect, OSIDocument +from ossie_wisdom import ConverterIssueType, WisdomToOSIConverter + +FIXTURE = Path(__file__).parent / "fixtures" / "sample_export.json" + + +@pytest.fixture(scope="module") +def result(): + export = json.loads(FIXTURE.read_text()) + return WisdomToOSIConverter().convert(export) + + +@pytest.fixture(scope="module") +def model(result): + assert len(result.output.semantic_model) == 1 + return result.output.semantic_model[0] + + +def _issues_of(result, issue_type): + return [issue for issue in result.issues if issue.issue_type is issue_type] + + +def test_model_name_and_description(model): + assert model.name == "Sample Sales" + assert model.description == "Synthetic sales domain used for converter tests" + + +def test_knowledge_becomes_model_ai_context(model): + assert model.ai_context == ( + "Only answer questions about sales data.\n" + "- The fiscal year starts in February.\n" + "- Pipeline refers to open orders expected to close this quarter." + ) + + +def test_datasets(model): + datasets = {dataset.name: dataset for dataset in model.datasets} + assert set(datasets) == {"orders", "customers", "tags"} + assert datasets["orders"].source == "analytics.sales.orders" + assert datasets["orders"].description == "Customer orders" + # Explicit primaryKey wins; otherwise per-column isPrimaryKey flags are collected. + assert datasets["customers"].primary_key == ["customer_id"] + assert datasets["orders"].primary_key == ["order_id"] + assert datasets["tags"].primary_key is None + + +def test_columns_become_fields(model): + orders = next(dataset for dataset in model.datasets if dataset.name == "orders") + fields = {field.name: field for field in orders.fields} + assert fields["order_id"].expression.dialects[0].expression == "order_id" + assert fields["order_id"].expression.dialects[0].dialect is OSIDialect.SNOWFLAKE + assert fields["order_date"].dimension.is_time is True + assert fields["order_id"].dimension is None + assert fields["status"].label == "Order Status" + assert fields["status"].description == "Current order status" + # Hidden columns are included. + assert "amount" in fields + # Non-identifier column names are quoted in the expression so they parse as SQL. + assert fields["Discount - Percent"].expression.dialects[0].expression == '"Discount - Percent"' + + +def test_formulas_become_fields(model): + orders = next(dataset for dataset in model.datasets if dataset.name == "orders") + fields = {field.name: field for field in orders.fields} + is_large = fields["is_large"] + assert is_large.expression.dialects[0].expression == 'CASE WHEN "orders"."amount" > 100 THEN TRUE ELSE FALSE END' + assert is_large.expression.dialects[0].dialect is OSIDialect.SNOWFLAKE + assert is_large.label == "Is Large" + assert is_large.description == "true when the order amount exceeds 100" + + +def test_formula_colliding_with_column_is_dropped(result, model): + customers = next(dataset for dataset in model.datasets if dataset.name == "customers") + assert [field.name for field in customers.fields].count("region") == 1 + dropped = _issues_of(result, ConverterIssueType.DUPLICATE_FIELD_DROPPED) + assert [issue.element_name for issue in dropped] == ["customers.region"] + + +def test_unsupported_dialect_falls_back_to_ansi(result, model): + tags = next(dataset for dataset in model.datasets if dataset.name == "tags") + assert all(field.expression.dialects[0].dialect is OSIDialect.ANSI_SQL for field in tags.fields) + unsupported = _issues_of(result, ConverterIssueType.UNSUPPORTED_DIALECT) + assert len(unsupported) == 1 + assert "postgres" in unsupported[0].element_name + + +def test_relationship_directions(model): + relationships = {relationship.name: relationship for relationship in model.relationships} + # MANY_TO_ONE keeps left as the many side. + many_to_one = relationships["orders_to_customers"] + assert (many_to_one.from_dataset, many_to_one.to) == ("orders", "customers") + assert many_to_one.from_columns == ["customer_id"] + assert many_to_one.to_columns == ["customer_id"] + # ONE_TO_MANY is flipped so `from` is the many side; the name is deduped. + flipped = relationships["orders_to_customers_2"] + assert (flipped.from_dataset, flipped.to) == ("orders", "customers") + + +def test_many_to_many_is_kept_with_cardinality_loss(result, model): + relationships = {relationship.name: relationship for relationship in model.relationships} + many_to_many = relationships["orders_to_tags"] + assert many_to_many.ai_context == "many-to-many relationship; cardinality is not representable in Ossie" + losses = _issues_of(result, ConverterIssueType.CARDINALITY_LOSS) + assert [issue.element_name for issue in losses] == ["orders <-> tags"] + + +def test_compound_and_join_is_flattened(model): + relationships = {relationship.name: relationship for relationship in model.relationships} + compound = relationships["orders_to_tags_2"] + assert compound.from_columns == ["order_id", "customer_id"] + assert compound.to_columns == ["order_id", "customer_id"] + + +def test_or_join_is_dropped(result, model): + assert len(model.relationships) == 4 + dropped = _issues_of(result, ConverterIssueType.RELATIONSHIP_DROPPED) + assert [issue.element_name for issue in dropped] == ["orders <-> tags"] + + +def test_measures_become_metrics(result, model): + metrics = {metric.name: metric for metric in model.metrics} + assert set(metrics) == {"total_amount", "customers_total_amount"} + total = metrics["total_amount"] + assert total.expression.dialects[0].expression == 'SUM("orders"."amount")' + assert total.expression.dialects[0].dialect is OSIDialect.SNOWFLAKE + assert total.description == "Total order amount" + collisions = _issues_of(result, ConverterIssueType.METRIC_NAME_COLLISION) + assert [issue.element_name for issue in collisions] == ["customers.total_amount"] + + +def test_stale_measure_is_kept_with_warning(result, model): + assert any(metric.name == "total_amount" for metric in model.metrics) + stale = _issues_of(result, ConverterIssueType.STALE_MEASURE) + assert [issue.element_name for issue in stale] == ["orders.total_amount"] + + +def test_output_round_trips_through_osi_yaml(result): + document = OSIDocument.model_validate(yaml.safe_load(result.output.to_osi_yaml())) + assert document == result.output + + +def test_export_without_tables_is_rejected(): + with pytest.raises(ValueError, match="no tables"): + WisdomToOSIConverter().convert({"domain": {"zsheet_json": {"ref": {"name": "empty"}}}, "tables": []}) From ad6d7bb79c9760e0bdde7bc0cff3e24f6a829d14 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Fri, 10 Jul 2026 16:01:57 -0700 Subject: [PATCH 87/89] Add WISDOM to well-known custom-extension vendors Adds WISDOM to the OSIVendor enum, the vendor examples in the spec documents and JSON schema, and the supported-vendors table in the converter guide. Co-Authored-By: Claude Fable 5 --- converters/README.md | 1 + core-spec/osi-schema.json | 2 +- core-spec/spec.md | 1 + core-spec/spec.yaml | 2 +- python/src/ossie/models.py | 1 + 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/converters/README.md b/converters/README.md index 8daf5d0..48bc065 100644 --- a/converters/README.md +++ b/converters/README.md @@ -74,6 +74,7 @@ The Ossie specification currently defines extensions for the following vendors: | `DBT` | dbt semantic models | | `DATABRICKS` | Databricks semantic layer | | `OMNI` | Omni semantic model | +| `WISDOM` | WisdomAI domain | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/core-spec/osi-schema.json b/core-spec/osi-schema.json index 385e18a..163db20 100644 --- a/core-spec/osi-schema.json +++ b/core-spec/osi-schema.json @@ -28,7 +28,7 @@ }, "Vendor": { "type": "string", - "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA"], + "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM"], "description": "Vendor name for custom extensions. Any string value is accepted." }, "AIContext": { diff --git a/core-spec/spec.md b/core-spec/spec.md index 8b9c10a..b7c285e 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -389,6 +389,7 @@ The following are well-known examples: | `DATABRICKS` | Databricks-specific attributes | | `GOODDATA` | GoodData-specific attributes | | `HONEYDEW` | Honeydew-specific attributes | +| `WISDOM` | WisdomAI-specific attributes | ### Examples diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 7229e08..2c9c73e 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -41,7 +41,7 @@ dialects: # Vendor name for custom extensions (free-form string) -# Examples: "COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA" +# Examples: "COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM" vendor_name: string diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index 1c8b646..9be74a6 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -44,6 +44,7 @@ class OSIVendor(str, Enum): DATABRICKS = "DATABRICKS" GOODDATA = "GOODDATA" SEMANTIDO = "SEMANTIDO" + WISDOM = "WISDOM" class OSIAIContextObject(BaseModel): From 96a6502013f5b5ef8c58a0f36da23dd28d09fb72 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Fri, 10 Jul 2026 16:12:47 -0700 Subject: [PATCH 88/89] Add Ossie -> wisdom domain export direction Adds osi-to-wisdom, the reverse of the wisdom converter: an Ossie document becomes a wisdom domain export (format 1.0) consumable by wisdom's importDomain API. The mapping inverts wisdom-to-osi so round-trips are stable in both directions: model ai_context splits back into system instructions and knowledge items, fields split into columns and formulas, relationship direction is read as many-to-one with ai_context notes restoring one-to-one/many-to-many and composite keys becoming compound AND join conditions, and metrics attach to the table their expression references. IDs are deterministic (derived from names) and connections are per-dialect placeholders remapped at import time. Verified: Ossie -> wisdom -> Ossie round-trips of two real domain exports are byte-identical. Co-Authored-By: Claude Fable 5 --- converters/wisdom/README.md | 32 +- .../wisdom/src/ossie_wisdom/__init__.py | 2 + converters/wisdom/src/ossie_wisdom/cli.py | 48 ++- .../src/ossie_wisdom/converter_issues.py | 6 + .../wisdom/src/ossie_wisdom/osi_to_wisdom.py | 393 ++++++++++++++++++ converters/wisdom/tests/test_osi_to_wisdom.py | 229 ++++++++++ 6 files changed, 700 insertions(+), 10 deletions(-) create mode 100644 converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py create mode 100644 converters/wisdom/tests/test_osi_to_wisdom.py diff --git a/converters/wisdom/README.md b/converters/wisdom/README.md index 166c356..4b4064f 100644 --- a/converters/wisdom/README.md +++ b/converters/wisdom/README.md @@ -17,16 +17,18 @@ under the License. --> -# WisdomAI → Apache Ossie Converter +# WisdomAI ↔ Apache Ossie Converter -Converts a [WisdomAI](https://wisdom.ai) domain export (format `1.0`, produced by wisdom's -`exportDomain` API) into an Ossie semantic model YAML document. +Converts between a [WisdomAI](https://wisdom.ai) domain export (format `1.0`, produced by +wisdom's `exportDomain` API and consumed by `importDomain`) and an Ossie semantic model +YAML document. ## Usage ```bash pip install -e ../../python -e . ossie-wisdom wisdom-to-osi -i domain-export.json -o semantic_model.yaml +ossie-wisdom osi-to-wisdom -i semantic_model.yaml -o domain-export.json ``` Conversion warnings (information loss) are printed to stderr; the output YAML validates @@ -73,9 +75,31 @@ Compound join conditions that are an `AND` of equality conditions are flattened positional `from_columns`/`to_columns` arrays; any other compound condition (e.g. `OR`) drops the relationship with a `RELATIONSHIP_DROPPED` warning. +## Ossie → wisdom (export direction) + +`osi-to-wisdom` emits a domain export mirroring wisdom's `exportDomain` JSON, inverting +the mapping above so that wisdom → Ossie → wisdom and Ossie → wisdom → Ossie round-trips +are stable: + +- Model `ai_context` splits back into `domainSystemInstructions` (leading text) and one + knowledge item per `- ` bullet. +- A field whose expression is just its own (possibly quoted) name becomes a column; + anything else becomes a formula. `dimension.is_time` becomes a `TIMESTAMP` data type + (wisdom re-derives exact types from the warehouse). +- Relationships become `MANY_TO_ONE` edges (Ossie's `from` is the many side); the + `ai_context` notes written by `wisdom-to-osi` restore `ONE_TO_ONE`/`MANY_TO_MANY`, and + composite keys become compound `AND` join conditions. +- Metrics attach to the first dataset their expression references (a + `METRIC_TABLE_UNRESOLVED` warning falls back to the first dataset). +- Connections are per-dialect placeholders (`et-connection-snowflake`, ...) expected to be + remapped when the domain is imported; all IDs are derived deterministically from names, + so re-runs produce identical output. +- Not representable in wisdom (dropped with warnings): extra semantic models beyond the + first, `unique_keys`, `custom_extensions`, and `ai_context` on fields/metrics (plus + synonyms/examples anywhere). + ## Known limitations -- One-way only (wisdom → Ossie). No Ossie → wisdom export yet. - Hidden columns and stale measures are converted anyway (with a `STALE_MEASURE` warning for the latter), so the output may reference columns wisdom itself hides from querying. - Out of scope for now: reviewed queries, recommended questions, synonym sets, row-level diff --git a/converters/wisdom/src/ossie_wisdom/__init__.py b/converters/wisdom/src/ossie_wisdom/__init__.py index 94153f4..337704d 100644 --- a/converters/wisdom/src/ossie_wisdom/__init__.py +++ b/converters/wisdom/src/ossie_wisdom/__init__.py @@ -16,11 +16,13 @@ # under the License. from ossie_wisdom.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult +from ossie_wisdom.osi_to_wisdom import OSIToWisdomConverter from ossie_wisdom.wisdom_to_osi import WisdomToOSIConverter __all__ = [ "ConverterIssue", "ConverterIssueType", "ConverterResult", + "OSIToWisdomConverter", "WisdomToOSIConverter", ] diff --git a/converters/wisdom/src/ossie_wisdom/cli.py b/converters/wisdom/src/ossie_wisdom/cli.py index 0274e62..6248697 100644 --- a/converters/wisdom/src/ossie_wisdom/cli.py +++ b/converters/wisdom/src/ossie_wisdom/cli.py @@ -19,6 +19,7 @@ Usage: ossie-wisdom wisdom-to-osi -i domain-export.json -o output.yaml + ossie-wisdom osi-to-wisdom -i input.yaml -o domain-export.json """ import argparse @@ -26,24 +27,45 @@ import sys from pathlib import Path +import yaml + +from ossie import OSIDocument from ossie_wisdom.converter_issues import ConverterIssueType +from ossie_wisdom.osi_to_wisdom import OSIToWisdomConverter from ossie_wisdom.wisdom_to_osi import WisdomToOSIConverter _ISSUE_REASON: dict[ConverterIssueType, str] = { ConverterIssueType.UNSUPPORTED_DIALECT: "the connection dialect has no Ossie equivalent; expressions were emitted verbatim as ANSI_SQL", ConverterIssueType.CARDINALITY_LOSS: "Ossie relationships encode cardinality by direction and cannot represent many-to-many", - ConverterIssueType.RELATIONSHIP_DROPPED: "the relationship uses a join Ossie cannot represent (non-table source, OR/non-equi condition, or unknown dataset)", + ConverterIssueType.RELATIONSHIP_DROPPED: "the relationship uses a join that cannot be represented (non-table source, OR/non-equi condition, or unknown dataset)", ConverterIssueType.METRIC_NAME_COLLISION: "another table defines a measure with the same name; this one was prefixed with its table name", ConverterIssueType.STALE_MEASURE: "wisdom marked this measure stale; it was converted anyway", ConverterIssueType.DUPLICATE_FIELD_DROPPED: "the dataset already has a field with this name", + ConverterIssueType.EXTRA_MODEL_DROPPED: "a wisdom domain export holds a single domain; only the first semantic model was converted", + ConverterIssueType.AI_CONTEXT_DROPPED: "wisdom has no equivalent for ai_context at this level (or for synonyms/examples)", + ConverterIssueType.METRIC_TABLE_UNRESOLVED: "the metric expression references no known dataset; it was attached to the first dataset", + ConverterIssueType.MISSING_DIALECT_EXPRESSION: "no expression was available in the dataset's dialect or ANSI_SQL; the first available dialect was used", + ConverterIssueType.UNIQUE_KEYS_DROPPED: "wisdom has no unique-key construct", + ConverterIssueType.CUSTOM_EXTENSION_DROPPED: "wisdom cannot store Ossie custom extensions", } _DROPPED_ISSUE_TYPES = { ConverterIssueType.RELATIONSHIP_DROPPED, ConverterIssueType.DUPLICATE_FIELD_DROPPED, + ConverterIssueType.EXTRA_MODEL_DROPPED, + ConverterIssueType.AI_CONTEXT_DROPPED, + ConverterIssueType.UNIQUE_KEYS_DROPPED, + ConverterIssueType.CUSTOM_EXTENSION_DROPPED, } +def _print_issues(result) -> None: + for issue in result.issues: + verb = "was dropped" if issue.issue_type in _DROPPED_ISSUE_TYPES else "was converted with loss" + reason = _ISSUE_REASON[issue.issue_type] + print(f"[WARNING] {issue.issue_type.value}: {issue.element_name} {verb} during conversion because {reason}", file=sys.stderr) + + def _cmd_wisdom_to_osi(args: argparse.Namespace) -> None: input_path = Path(args.input) output_path = Path(args.output) @@ -51,15 +73,23 @@ def _cmd_wisdom_to_osi(args: argparse.Namespace) -> None: export = json.loads(input_path.read_text()) result = WisdomToOSIConverter().convert(export) - for issue in result.issues: - verb = "was dropped" if issue.issue_type in _DROPPED_ISSUE_TYPES else "was converted with loss" - reason = _ISSUE_REASON[issue.issue_type] - print(f"[WARNING] {issue.issue_type.value}: {issue.element_name} {verb} during conversion because {reason}", file=sys.stderr) - + _print_issues(result) output_path.write_text(result.output.to_osi_yaml()) print(f"Written to {output_path}", file=sys.stderr) +def _cmd_osi_to_wisdom(args: argparse.Namespace) -> None: + input_path = Path(args.input) + output_path = Path(args.output) + + document = OSIDocument.model_validate(yaml.safe_load(input_path.read_text())) + result = OSIToWisdomConverter().convert(document) + + _print_issues(result) + output_path.write_text(json.dumps(result.output, indent=2) + "\n") + print(f"Written to {output_path}", file=sys.stderr) + + def main() -> None: parser = argparse.ArgumentParser( prog="ossie-wisdom", @@ -71,9 +101,15 @@ def main() -> None: wisdom_to_osi.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to wisdom domain export JSON") wisdom_to_osi.add_argument("-o", "--output", required=True, metavar="FILE", help="Path for output Ossie YAML") + osi_to_wisdom = subparsers.add_parser("osi-to-wisdom", help="Convert Ossie YAML → domain-export.json") + osi_to_wisdom.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to Ossie YAML") + osi_to_wisdom.add_argument("-o", "--output", required=True, metavar="FILE", help="Path for output wisdom domain export JSON") + args = parser.parse_args() if args.command == "wisdom-to-osi": _cmd_wisdom_to_osi(args) + elif args.command == "osi-to-wisdom": + _cmd_osi_to_wisdom(args) if __name__ == "__main__": diff --git a/converters/wisdom/src/ossie_wisdom/converter_issues.py b/converters/wisdom/src/ossie_wisdom/converter_issues.py index f8c8fb2..fcef92f 100644 --- a/converters/wisdom/src/ossie_wisdom/converter_issues.py +++ b/converters/wisdom/src/ossie_wisdom/converter_issues.py @@ -29,6 +29,12 @@ class ConverterIssueType(Enum): METRIC_NAME_COLLISION = "METRIC_NAME_COLLISION" STALE_MEASURE = "STALE_MEASURE" DUPLICATE_FIELD_DROPPED = "DUPLICATE_FIELD_DROPPED" + EXTRA_MODEL_DROPPED = "EXTRA_MODEL_DROPPED" + AI_CONTEXT_DROPPED = "AI_CONTEXT_DROPPED" + METRIC_TABLE_UNRESOLVED = "METRIC_TABLE_UNRESOLVED" + MISSING_DIALECT_EXPRESSION = "MISSING_DIALECT_EXPRESSION" + UNIQUE_KEYS_DROPPED = "UNIQUE_KEYS_DROPPED" + CUSTOM_EXTENSION_DROPPED = "CUSTOM_EXTENSION_DROPPED" @dataclass(frozen=True) diff --git a/converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py b/converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py new file mode 100644 index 0000000..cc245d1 --- /dev/null +++ b/converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py @@ -0,0 +1,393 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Converts an Ossie Document into a WisdomAI domain export (format 1.0). + +The output mirrors the JSON produced by wisdom's ``exportDomain`` RPC so it +can be fed to ``importDomain``. Inverse of :mod:`ossie_wisdom.wisdom_to_osi`: +model ``ai_context`` splits back into system instructions and knowledge items, +fields split back into columns and formulas, relationship direction is read as +many-to-one (with the ai_context notes restoring one-to-one/many-to-many), and +metrics attach to the table their expression references. + +IDs are derived deterministically from element names, and connections are +per-dialect placeholders expected to be remapped when the domain is imported. +""" + +import hashlib +import re +from datetime import datetime, timezone +from typing import Dict, List, Optional, Tuple + +from ossie import ( + OSIAIContextObject, + OSIDataset, + OSIDialect, + OSIDocument, + OSIExpression, + OSISemanticModel, +) +from ossie_wisdom.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult + +_WISDOM_DIALECT = { + OSIDialect.SNOWFLAKE: "snowflake", + OSIDialect.DATABRICKS: "databricks", + OSIDialect.ANSI_SQL: "ansi", +} + + +def _stable_id(prefix: str, value: str) -> str: + return f"{prefix}_{hashlib.md5(value.encode('utf-8')).hexdigest()}" + + +class OSIToWisdomConverter: + """Converts an Ossie Document into a wisdom domain export dict.""" + + def convert(self, document: OSIDocument, exported_at: Optional[str] = None) -> ConverterResult[dict]: + issues: List[ConverterIssue] = [] + + model = document.semantic_model[0] + for extra in document.semantic_model[1:]: + issues.append(ConverterIssue(issue_type=ConverterIssueType.EXTRA_MODEL_DROPPED, element_name=extra.name)) + self._report_custom_extensions(model, issues) + + domain_uuid = _stable_id("ET_DOMAIN", model.name) + zsheet_refs = { + dataset.name: {"uuid": _stable_id("ET_ZSHEET", dataset.name), "name": dataset.name, "version": "1"} + for dataset in model.datasets + } + dataset_dialects = {dataset.name: self._infer_dataset_dialect(dataset) for dataset in model.datasets} + measures_by_dataset = self._assign_metrics(model, dataset_dialects, issues) + + tables = [] + table_metadata = [] + for dataset in model.datasets: + zsheet = self._convert_dataset( + dataset, zsheet_refs, dataset_dialects[dataset.name], measures_by_dataset.get(dataset.name, []), issues + ) + tables.append({"zsheet_uuid": zsheet_refs[dataset.name]["uuid"], "zsheet_json": zsheet}) + location = zsheet["location"] + table_metadata.append( + { + "zsheet_uuid": zsheet_refs[dataset.name]["uuid"], + "connection_id": location["connectionId"], + "database": location["database"], + "schema": location["schema"], + "table_name": location["dbTable"], + } + ) + + instructions, knowledge = self._split_ai_context(model, issues) + domain_zsheet: dict = { + "ref": {"uuid": domain_uuid, "name": model.name, "version": "1"}, + "zsheetType": "DOMAIN", + "relationshipGraph": { + "zsheets": list(zsheet_refs.values()), + "relationships": self._convert_relationships(model, zsheet_refs, issues), + }, + } + if model.description: + domain_zsheet["description"] = model.description + if instructions: + domain_zsheet["domainSystemInstructions"] = instructions + if knowledge: + domain_zsheet["knowledge"] = knowledge + + connections = [ + {"connection_id": f"et-connection-{dialect}", "dialect": dialect, "name": dialect} + for dialect in sorted({_WISDOM_DIALECT[d] for d in dataset_dialects.values()}) + ] + + export = { + "version": "1.0", + "export_metadata": { + "exported_at": exported_at or datetime.now(timezone.utc).isoformat(), + "source_domain_id": domain_uuid, + "domain_name": model.name, + }, + "domain": {"zsheet_json": domain_zsheet}, + "tables": tables, + "connections": connections, + "reviewed_queries": {"ref": None, "items_json": "{}"}, + "synonym_sets": {"ref": None, "items_json": "{}"}, + "table_metadata": table_metadata, + "recommended_questions": [], + } + return ConverterResult(output=export, issues=issues) + + def _report_custom_extensions(self, model: OSISemanticModel, issues: List[ConverterIssue]) -> None: + elements = [(model.name, model.custom_extensions)] + for dataset in model.datasets: + elements.append((dataset.name, dataset.custom_extensions)) + for field in dataset.fields or []: + elements.append((f"{dataset.name}.{field.name}", field.custom_extensions)) + for relationship in model.relationships or []: + elements.append((relationship.name, relationship.custom_extensions)) + for metric in model.metrics or []: + elements.append((metric.name, metric.custom_extensions)) + for name, extensions in elements: + if extensions: + issues.append( + ConverterIssue(issue_type=ConverterIssueType.CUSTOM_EXTENSION_DROPPED, element_name=name) + ) + + def _infer_dataset_dialect(self, dataset: OSIDataset) -> OSIDialect: + for field in dataset.fields or []: + for entry in field.expression.dialects: + if entry.dialect in (OSIDialect.SNOWFLAKE, OSIDialect.DATABRICKS): + return entry.dialect + return OSIDialect.ANSI_SQL + + def _split_ai_context(self, model: OSISemanticModel, issues: List[ConverterIssue]) -> Tuple[str, List[dict]]: + ai_context = model.ai_context + if ai_context is None: + return "", [] + if isinstance(ai_context, OSIAIContextObject): + if ai_context.synonyms or ai_context.examples: + issues.append(ConverterIssue(issue_type=ConverterIssueType.AI_CONTEXT_DROPPED, element_name=model.name)) + text = ai_context.instructions or "" + else: + text = ai_context + + instruction_lines: List[str] = [] + contents: List[str] = [] + current: Optional[str] = None + for line in text.split("\n"): + if line.startswith("- "): + if current is not None: + contents.append(current) + current = line[2:] + elif current is not None: + current += "\n" + line + else: + instruction_lines.append(line) + if current is not None: + contents.append(current) + + knowledge = [ + {"name": content, "content": content, "id": _stable_id("ET_UNSTRUCTURED_KNOWLEDGE", content)} + for content in contents + if content.strip() + ] + return "\n".join(instruction_lines).strip(), knowledge + + def _convert_dataset( + self, + dataset: OSIDataset, + zsheet_refs: Dict[str, dict], + dialect: OSIDialect, + measures: List[dict], + issues: List[ConverterIssue], + ) -> dict: + ref = zsheet_refs[dataset.name] + ref_lite = {"uuid": ref["uuid"], "name": ref["name"]} + database, schema, table = self._split_source(dataset.source) + + columns: List[dict] = [] + formulas: List[dict] = [] + for field in dataset.fields or []: + expression = self._pick_expression(field.expression, dialect, f"{dataset.name}.{field.name}", issues) + if field.ai_context is not None: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.AI_CONTEXT_DROPPED, element_name=f"{dataset.name}.{field.name}" + ) + ) + properties: dict = {} + if field.label: + properties["displayName"] = field.label + if field.dimension and field.dimension.is_time: + properties["dataType"] = "TIMESTAMP" + if self._is_bare_column(expression, field.name): + column: dict = {"name": field.name} + if field.description: + column["description"] = field.description + if properties: + column["properties"] = properties + column["location"] = {"name": field.name, "zsheetRef": ref_lite} + columns.append(column) + else: + formula: dict = {"name": field.name, "expression": expression} + if field.description: + formula["description"] = field.description + if properties: + formula["properties"] = properties + formula["location"] = {"name": field.name, "zsheetRef": ref_lite} + formula["id"] = _stable_id("FORMULA", f"{dataset.name}.{field.name}") + formulas.append(formula) + + zsheet: dict = { + "ref": ref, + "location": { + "database": database, + "schema": schema, + "dbTable": table, + "connectionId": f"et-connection-{_WISDOM_DIALECT[dialect]}", + }, + "source": {"alias": dataset.name, "zsheet": ref_lite}, + "columns": columns, + } + if dataset.description: + zsheet["description"] = dataset.description + if formulas: + zsheet["formulas"] = formulas + if measures: + zsheet["measures"] = measures + if dataset.primary_key: + zsheet["primaryKey"] = {"columns": list(dataset.primary_key)} + if dataset.unique_keys: + issues.append(ConverterIssue(issue_type=ConverterIssueType.UNIQUE_KEYS_DROPPED, element_name=dataset.name)) + if dataset.ai_context is not None: + content = self._ai_context_text(dataset.ai_context, dataset.name, issues) + if content: + zsheet["knowledge"] = [ + {"name": content, "content": content, "id": _stable_id("ET_UNSTRUCTURED_KNOWLEDGE", content)} + ] + return zsheet + + def _ai_context_text(self, ai_context, element_name: str, issues: List[ConverterIssue]) -> str: + if isinstance(ai_context, OSIAIContextObject): + if ai_context.synonyms or ai_context.examples: + issues.append( + ConverterIssue(issue_type=ConverterIssueType.AI_CONTEXT_DROPPED, element_name=element_name) + ) + return ai_context.instructions or "" + return ai_context + + def _split_source(self, source: str) -> Tuple[str, str, str]: + parts = source.split(".") + if len(parts) >= 3: + return parts[0], parts[1], ".".join(parts[2:]) + if len(parts) == 2: + return "", parts[0], parts[1] + return "", "", source + + def _pick_expression( + self, expression: OSIExpression, dialect: OSIDialect, element_name: str, issues: List[ConverterIssue] + ) -> str: + by_dialect = {entry.dialect: entry.expression for entry in expression.dialects} + if dialect in by_dialect: + return by_dialect[dialect] + if OSIDialect.ANSI_SQL in by_dialect: + return by_dialect[OSIDialect.ANSI_SQL] + issues.append( + ConverterIssue(issue_type=ConverterIssueType.MISSING_DIALECT_EXPRESSION, element_name=element_name) + ) + return expression.dialects[0].expression + + def _is_bare_column(self, expression: str, name: str) -> bool: + return expression in (name, f'"{name}"', f"`{name}`") + + def _assign_metrics( + self, model: OSISemanticModel, dataset_dialects: Dict[str, OSIDialect], issues: List[ConverterIssue] + ) -> Dict[str, List[dict]]: + measures: Dict[str, List[dict]] = {} + dataset_names = [dataset.name for dataset in model.datasets] + for metric in model.metrics or []: + if metric.ai_context is not None: + issues.append( + ConverterIssue(issue_type=ConverterIssueType.AI_CONTEXT_DROPPED, element_name=metric.name) + ) + # Attach to a dataset first so the expression can be picked in that dataset's dialect. + reference_text = " ".join(entry.expression for entry in metric.expression.dialects) + dataset_name = self._find_referenced_dataset(reference_text, dataset_names) + if dataset_name is None: + dataset_name = dataset_names[0] + issues.append( + ConverterIssue(issue_type=ConverterIssueType.METRIC_TABLE_UNRESOLVED, element_name=metric.name) + ) + expression = self._pick_expression( + metric.expression, dataset_dialects[dataset_name], metric.name, issues + ) + ref = {"uuid": _stable_id("ET_ZSHEET", dataset_name), "name": dataset_name} + measure: dict = {"name": metric.name, "expression": expression} + if metric.description: + measure["description"] = metric.description + measure["location"] = {"name": metric.name, "zsheetRef": ref} + measure["id"] = _stable_id("MEASURE", f"{dataset_name}.{metric.name}") + measures.setdefault(dataset_name, []).append(measure) + return measures + + def _find_referenced_dataset(self, expression: str, dataset_names: List[str]) -> Optional[str]: + best: Optional[str] = None + best_position = len(expression) + 1 + for name in dataset_names: + pattern = re.compile(r'(? List[dict]: + edges = [] + for relationship in model.relationships or []: + left = relationship.from_dataset + right = relationship.to + if left not in zsheet_refs or right not in zsheet_refs: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.RELATIONSHIP_DROPPED, element_name=relationship.name + ) + ) + continue + relationship_type = "MANY_TO_ONE" + if isinstance(relationship.ai_context, str): + if relationship.ai_context.startswith("one-to-one"): + relationship_type = "ONE_TO_ONE" + elif relationship.ai_context.startswith("many-to-many"): + relationship_type = "MANY_TO_MANY" + else: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.AI_CONTEXT_DROPPED, element_name=relationship.name + ) + ) + elif relationship.ai_context is not None: + issues.append( + ConverterIssue(issue_type=ConverterIssueType.AI_CONTEXT_DROPPED, element_name=relationship.name) + ) + + left_ref = {"uuid": zsheet_refs[left]["uuid"], "name": left} + right_ref = {"uuid": zsheet_refs[right]["uuid"], "name": right} + conditions = [ + { + "leftColumn": {"name": from_column, "zsheetRef": left_ref}, + "rightColumn": {"name": to_column, "zsheetRef": right_ref}, + } + for from_column, to_column in zip(relationship.from_columns, relationship.to_columns) + ] + properties: dict = {"relationshipType": relationship_type} + if len(conditions) == 1: + properties["joinCondition"] = conditions[0] + else: + properties["compoundJoinCondition"] = { + "nestedCondition": { + "logicalOperator": "AND", + "conditions": [{"simpleCondition": condition} for condition in conditions], + } + } + edges.append( + { + "properties": properties, + "leftDataSource": {"zsheet": left_ref}, + "rightDataSource": {"zsheet": right_ref}, + } + ) + return edges diff --git a/converters/wisdom/tests/test_osi_to_wisdom.py b/converters/wisdom/tests/test_osi_to_wisdom.py new file mode 100644 index 0000000..1ee1717 --- /dev/null +++ b/converters/wisdom/tests/test_osi_to_wisdom.py @@ -0,0 +1,229 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +from pathlib import Path + +import pytest + +from ossie import ( + OSIDataset, + OSIDialect, + OSIDialectExpression, + OSIDocument, + OSIExpression, + OSIField, + OSIRelationship, + OSISemanticModel, +) +from ossie_wisdom import ConverterIssueType, OSIToWisdomConverter, WisdomToOSIConverter + +FIXTURE = Path(__file__).parent / "fixtures" / "sample_export.json" + + +def _snowflake(expression): + return OSIExpression(dialects=[OSIDialectExpression(dialect=OSIDialect.SNOWFLAKE, expression=expression)]) + + +@pytest.fixture(scope="module") +def osi_document(): + export = json.loads(FIXTURE.read_text()) + return WisdomToOSIConverter().convert(export).output + + +@pytest.fixture(scope="module") +def result(osi_document): + return OSIToWisdomConverter().convert(osi_document, exported_at="2026-07-10T00:00:00+00:00") + + +@pytest.fixture(scope="module") +def export(result): + return result.output + + +def _issues_of(result, issue_type): + return [issue for issue in result.issues if issue.issue_type is issue_type] + + +def _table(export, name): + return next( + table["zsheet_json"] for table in export["tables"] if table["zsheet_json"]["ref"]["name"] == name + ) + + +def test_export_envelope(export): + assert export["version"] == "1.0" + assert export["export_metadata"]["domain_name"] == "Sample Sales" + assert export["export_metadata"]["source_domain_id"] == export["domain"]["zsheet_json"]["ref"]["uuid"] + domain = export["domain"]["zsheet_json"] + assert domain["zsheetType"] == "DOMAIN" + assert domain["description"] == "Synthetic sales domain used for converter tests" + + +def test_ai_context_splits_into_instructions_and_knowledge(export): + domain = export["domain"]["zsheet_json"] + assert domain["domainSystemInstructions"] == "Only answer questions about sales data." + assert [knowledge["content"] for knowledge in domain["knowledge"]] == [ + "The fiscal year starts in February.", + "Pipeline refers to open orders expected to close this quarter.", + ] + + +def test_tables_and_locations(export): + orders = _table(export, "orders") + assert orders["location"]["database"] == "analytics" + assert orders["location"]["schema"] == "sales" + assert orders["location"]["dbTable"] == "orders" + assert orders["primaryKey"] == {"columns": ["order_id"]} + assert _table(export, "customers")["primaryKey"] == {"columns": ["customer_id"]} + metadata = {entry["zsheet_uuid"]: entry for entry in export["table_metadata"]} + assert metadata[orders["ref"]["uuid"]]["table_name"] == "orders" + + +def test_fields_split_into_columns_and_formulas(export): + orders = _table(export, "orders") + column_names = [column["name"] for column in orders["columns"]] + assert "order_id" in column_names + # A quoted bare-name expression is recognized as a plain column. + assert "Discount - Percent" in column_names + formulas = {formula["name"]: formula for formula in orders["formulas"]} + assert set(formulas) == {"is_large"} + assert formulas["is_large"]["expression"] == 'CASE WHEN "orders"."amount" > 100 THEN TRUE ELSE FALSE END' + assert formulas["is_large"]["properties"]["displayName"] == "Is Large" + status = next(column for column in orders["columns"] if column["name"] == "status") + assert status["description"] == "Current order status" + assert status["properties"]["displayName"] == "Order Status" + + +def test_metrics_attach_to_referenced_tables(export): + orders_measures = {measure["name"] for measure in _table(export, "orders")["measures"]} + customers_measures = {measure["name"] for measure in _table(export, "customers")["measures"]} + assert orders_measures == {"total_amount"} + assert customers_measures == {"customers_total_amount"} + + +def test_relationship_types_restored(export): + edges = export["domain"]["zsheet_json"]["relationshipGraph"]["relationships"] + types = [edge["properties"]["relationshipType"] for edge in edges] + assert types == ["MANY_TO_ONE", "MANY_TO_ONE", "MANY_TO_MANY", "MANY_TO_ONE"] + compound = edges[3]["properties"]["compoundJoinCondition"]["nestedCondition"] + assert compound["logicalOperator"] == "AND" + assert len(compound["conditions"]) == 2 + + +def test_connections_are_per_dialect(export): + dialects = {connection["dialect"] for connection in export["connections"]} + assert dialects == {"snowflake", "ansi"} + assert _table(export, "orders")["location"]["connectionId"] == "et-connection-snowflake" + assert _table(export, "tags")["location"]["connectionId"] == "et-connection-ansi" + + +def test_round_trip_preserves_osi_document(osi_document, export): + round_tripped = WisdomToOSIConverter().convert(export).output + assert round_tripped == osi_document + + +def test_deterministic_output(osi_document, export): + again = OSIToWisdomConverter().convert(osi_document, exported_at="2026-07-10T00:00:00+00:00").output + assert again == export + + +def test_extra_models_and_unrepresentable_elements_are_reported(): + dataset = OSIDataset( + name="orders", + source="analytics.sales.orders", + unique_keys=[["order_id"]], + fields=[ + OSIField(name="order_id", expression=_snowflake("order_id"), ai_context="the identifier"), + ], + ) + second = OSISemanticModel(name="second", datasets=[OSIDataset(name="d", source="a.b.c")]) + document = OSIDocument( + semantic_model=[ + OSISemanticModel( + name="first", + datasets=[dataset], + relationships=[ + OSIRelationship( + name="orders_to_missing", + from_dataset="orders", + to="missing", + from_columns=["x"], + to_columns=["y"], + ) + ], + ), + second, + ] + ) + result = OSIToWisdomConverter().convert(document, exported_at="2026-07-10T00:00:00+00:00") + assert [issue.element_name for issue in _issues_of(result, ConverterIssueType.EXTRA_MODEL_DROPPED)] == ["second"] + assert [issue.element_name for issue in _issues_of(result, ConverterIssueType.UNIQUE_KEYS_DROPPED)] == ["orders"] + assert [issue.element_name for issue in _issues_of(result, ConverterIssueType.AI_CONTEXT_DROPPED)] == [ + "orders.order_id" + ] + assert [issue.element_name for issue in _issues_of(result, ConverterIssueType.RELATIONSHIP_DROPPED)] == [ + "orders_to_missing" + ] + assert result.output["domain"]["zsheet_json"]["relationshipGraph"]["relationships"] == [] + + +def test_one_to_one_note_restores_relationship_type(): + document = OSIDocument( + semantic_model=[ + OSISemanticModel( + name="m", + datasets=[ + OSIDataset(name="a", source="db.s.a"), + OSIDataset(name="b", source="db.s.b"), + ], + relationships=[ + OSIRelationship( + name="a_to_b", + from_dataset="a", + to="b", + from_columns=["id"], + to_columns=["id"], + ai_context="one-to-one relationship", + ) + ], + ) + ] + ) + export = OSIToWisdomConverter().convert(document, exported_at="2026-07-10T00:00:00+00:00").output + edges = export["domain"]["zsheet_json"]["relationshipGraph"]["relationships"] + assert edges[0]["properties"]["relationshipType"] == "ONE_TO_ONE" + + +def test_unresolved_metric_attaches_to_first_dataset(): + from ossie import OSIMetric + + document = OSIDocument( + semantic_model=[ + OSISemanticModel( + name="m", + datasets=[OSIDataset(name="a", source="db.s.a"), OSIDataset(name="b", source="db.s.b")], + metrics=[OSIMetric(name="row_count", expression=_snowflake("COUNT(*)"))], + ) + ] + ) + result = OSIToWisdomConverter().convert(document, exported_at="2026-07-10T00:00:00+00:00") + export = result.output + assert [measure["name"] for measure in _table(export, "a")["measures"]] == ["row_count"] + assert [issue.element_name for issue in _issues_of(result, ConverterIssueType.METRIC_TABLE_UNRESOLVED)] == [ + "row_count" + ] From cd803b4886ada169d110e115529f09f88808c009 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Tue, 21 Jul 2026 07:42:06 -0700 Subject: [PATCH 89/89] Adapt wisdom converter to BIGQUERY dialect and UV conventions BigQuery connections now map to the BIGQUERY dialect added to the spec instead of falling back to ANSI_SQL with a warning, with backtick identifier quoting; both directions round-trip it. The pyproject follows the repo's UV layout (dependency-groups, uv sources for the in-repo apache-ossie). Co-Authored-By: Claude Fable 5 --- converters/wisdom/README.md | 4 +- converters/wisdom/pyproject.toml | 39 +++++++++++++------ .../wisdom/src/ossie_wisdom/osi_to_wisdom.py | 3 +- .../wisdom/src/ossie_wisdom/wisdom_to_osi.py | 5 ++- 4 files changed, 36 insertions(+), 15 deletions(-) diff --git a/converters/wisdom/README.md b/converters/wisdom/README.md index 4b4064f..c5884aa 100644 --- a/converters/wisdom/README.md +++ b/converters/wisdom/README.md @@ -56,8 +56,8 @@ python ../../validation/validate.py semantic_model.yaml --schema ../../core-spec | `metrics[]` | every table's `measures[]`, hoisted to the model level | Expressions are emitted verbatim under the Ossie dialect mapped from the table's connection -(`snowflake → SNOWFLAKE`, `databricks → DATABRICKS`). Connections with any other dialect fall -back to `ANSI_SQL` verbatim with an `UNSUPPORTED_DIALECT` warning. +(`snowflake → SNOWFLAKE`, `databricks → DATABRICKS`, `bigquery → BIGQUERY`). Connections with +any other dialect fall back to `ANSI_SQL` verbatim with an `UNSUPPORTED_DIALECT` warning. ### Relationship cardinality diff --git a/converters/wisdom/pyproject.toml b/converters/wisdom/pyproject.toml index 7cb7701..15052cc 100644 --- a/converters/wisdom/pyproject.toml +++ b/converters/wisdom/pyproject.toml @@ -19,33 +19,50 @@ requires = ["hatchling"] build-backend = "hatchling.build" +[dependency-groups] +dev = [ + "pytest>=8.0", +] + [project] name = "apache-ossie-wisdom" version = "0.2.0.dev0" -description = "WisdomAI domain export -> Apache Ossie converter" +description = "WisdomAI domain export <> Apache Ossie converter" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] requires-python = ">=3.11" -classifiers = [ - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Open Semantic Interchange", + "WisdomAI" ] dependencies = [ "apache-ossie>=0.2.0.dev0", "PyYAML>=6.0", ] -[project.license] -text = "Apache-2.0" - [project.scripts] ossie-wisdom = "ossie_wisdom.cli:main" +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + [tool.hatch.build.targets.wheel] packages = ["src/ossie_wisdom"] +[tool.pytest.ini_options] +testpaths = ["tests"] + [tool.uv] -dev-dependencies = [ - "pytest>=8.0", +required-version = ">=0.9.0" +default-groups = [ + "dev" ] -[tool.pytest.ini_options] -testpaths = ["tests"] +# apache-ossie is not yet published to PyPI; resolve it from the in-repo +# package for now. Remove this block once apache-ossie published to PyPI. +[tool.uv.sources] +apache-ossie = { path = "../../python", editable = true} diff --git a/converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py b/converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py index cc245d1..14aee07 100644 --- a/converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py +++ b/converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py @@ -46,6 +46,7 @@ _WISDOM_DIALECT = { OSIDialect.SNOWFLAKE: "snowflake", OSIDialect.DATABRICKS: "databricks", + OSIDialect.BIGQUERY: "bigquery", OSIDialect.ANSI_SQL: "ansi", } @@ -148,7 +149,7 @@ def _report_custom_extensions(self, model: OSISemanticModel, issues: List[Conver def _infer_dataset_dialect(self, dataset: OSIDataset) -> OSIDialect: for field in dataset.fields or []: for entry in field.expression.dialects: - if entry.dialect in (OSIDialect.SNOWFLAKE, OSIDialect.DATABRICKS): + if entry.dialect in (OSIDialect.SNOWFLAKE, OSIDialect.DATABRICKS, OSIDialect.BIGQUERY): return entry.dialect return OSIDialect.ANSI_SQL diff --git a/converters/wisdom/src/ossie_wisdom/wisdom_to_osi.py b/converters/wisdom/src/ossie_wisdom/wisdom_to_osi.py index 855641d..6fc8129 100644 --- a/converters/wisdom/src/ossie_wisdom/wisdom_to_osi.py +++ b/converters/wisdom/src/ossie_wisdom/wisdom_to_osi.py @@ -42,10 +42,13 @@ _DIALECT_MAP: Dict[str, OSIDialect] = { "snowflake": OSIDialect.SNOWFLAKE, "databricks": OSIDialect.DATABRICKS, + "bigquery": OSIDialect.BIGQUERY, "ansi": OSIDialect.ANSI_SQL, "ansi_sql": OSIDialect.ANSI_SQL, } +_BACKTICK_DIALECTS = {OSIDialect.DATABRICKS, OSIDialect.BIGQUERY} + _TIME_DATA_TYPES = {"DATE", "DATETIME", "TIMESTAMP"} _SIMPLE_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") @@ -350,7 +353,7 @@ def _quote_identifier(self, name: str, dialect: OSIDialect) -> str: """Quotes a column name for use as a SQL expression when it is not a plain identifier.""" if _SIMPLE_IDENTIFIER.match(name): return name - if dialect is OSIDialect.DATABRICKS: + if dialect in _BACKTICK_DIALECTS: return "`" + name.replace("`", "``") + "`" return '"' + name.replace('"', '""') + '"'