feat(migrate): expression/functional indexes (identity.secondary @expr + @using)

dmealing · dmealing · commit 8cf8a664462a · 2026-06-19T11:38:49.000-04:00
Adds support for functional/expression indexes — e.g. `lower(email)` (btree) and `string_to_array(tags, ',')` (GIN) — which previously could not be modeled and were silently skipped on introspect. - identity.secondary gains two db-provider attrs: - @expr — raw key EXPRESSION; used instead of @fields (which then anchors the underlying column(s) for the loader's required @fields) - @using — access method (gin/gist/hash/…); default btree (not rendered) - IndexDescriptor: expr?, using? - buildExpectedSchema: reads @expr/@using - emit (postgres): renders `USING <method> (<expr>)` - introspect (postgres): captures the access method + lifts the key expression from pg_get_indexdef (expression-key indexes are no longer skipped) - diff/indexEquals: compares using + normalized expr - registered across all ports (TS/Python/Java+Kotlin/C#); spec + embedded def updated; registry-conformance manifest, coverage snapshot, and metamodel docs regenerated - test: expression-index round-trip (expected-schema + emit) for btree functional + GIN expression indexes 🤖 Generated with [Claude Code](https://claude.com/claude-code)
diff --git a/fixtures/metamodel-docs/expected/providers.md b/fixtures/metamodel-docs/expected/providers.md
@@ -62,7 +62,7 @@ DB-domain attributes — @column / @db.indexed / @dbColumnType on every field, @
 - `field.timestamp`: `@autoSet`, `@column`, `@db.indexed`, `@dbColumnType`
 - `field.uuid`: `@column`, `@db.indexed`, `@dbColumnType`
 - `identity.reference`: `@constraintName`
-- `identity.secondary`: `@orders`, `@where`
+- `identity.secondary`: `@expr`, `@orders`, `@using`, `@where`
 - `source.rdb`: `@function`, `@kind`, `@materializedView`, `@parameterRef`, `@proc`, `@role`, `@schema`, `@table`, `@view`
 
 ## metaobjects-documentation
diff --git a/fixtures/metamodel-docs/expected/types/identity.md b/fixtures/metamodel-docs/expected/types/identity.md
@@ -56,9 +56,11 @@ A secondary index (unique by default via @unique).
 
 | Attribute | Type | Required | Default | Allowed values | Provider | Description |
 | --- | --- | --- | --- | --- | --- | --- |
+| `@expr` | string | no |  |  | metaobjects-db | Raw key EXPRESSION for a functional/expression index (e.g. "lower(email)"). Used INSTEAD of @fields — the index key is the expression rather than plain columns. RDB-physical — contributed by the db provider. |
 | `@fields` | string[] | yes |  |  | metaobjects-core-types | The field name(s) composing this identity. Single-element for a simple PK/index, multiple for a composite. |
 | `@orders` | string[] | no |  | `asc`, `desc` | metaobjects-db | Physical index-key sort direction, positional to @fields ('asc' \| 'desc'). Omit for all-ascending (the default); a shorter array leaves trailing keys ascending. Drives DESC-ordered index keys (e.g. a recency index on a timestamp). RDB-physical — contributed by the db provider, not core identity. |
 | `@unique` | boolean | no |  |  | metaobjects-core-types | When true (default), the secondary identity is a UNIQUE index; false makes it a plain (non-unique) index. |
+| `@using` | string | no |  |  | metaobjects-db | Index access method (e.g. "gin", "gist", "hash"); default "btree" (not rendered). Pair with @expr for e.g. a GIN index over an array/jsonb expression. RDB-physical — contributed by the db provider. |
 | `@where` | string | no |  |  | metaobjects-db | Partial-index predicate (raw SQL, e.g. "delivered_at IS NULL"). When set, the index covers only rows matching the predicate — smaller and cheaper for queries that always filter on it. Absent = a full index over every row. RDB-physical — contributed by the db provider. |
 
 **Allowed children**
diff --git a/fixtures/registry-conformance/coverage-report.json b/fixtures/registry-conformance/coverage-report.json
@@ -226,7 +226,9 @@
     {
       "key": "identity.secondary",
       "untestedAttrs": [
+        "expr",
         "orders",
+        "using",
         "where"
       ]
     },
diff --git a/fixtures/registry-conformance/expected-registry.json b/fixtures/registry-conformance/expected-registry.json
@@ -2050,6 +2050,13 @@
       "subType": "secondary",
       "description": "A secondary index (unique by default via @unique).",
       "attrs": [
+        {
+          "name": "expr",
+          "valueType": "string",
+          "isArray": false,
+          "required": false,
+          "description": "Raw key EXPRESSION for a functional/expression index (e.g. \"lower(email)\"). Used INSTEAD of @fields — the index key is the expression rather than plain columns. RDB-physical — contributed by the db provider."
+        },
         {
           "name": "fields",
           "valueType": "string",
@@ -2071,6 +2078,13 @@
           "required": false,
           "description": "When true (default), the secondary identity is a UNIQUE index; false makes it a plain (non-unique) index."
         },
+        {
+          "name": "using",
+          "valueType": "string",
+          "isArray": false,
+          "required": false,
+          "description": "Index access method (e.g. \"gin\", \"gist\", \"hash\"); default \"btree\" (not rendered). Pair with @expr for e.g. a GIN index over an array/jsonb expression. RDB-physical — contributed by the db provider."
+        },
         {
           "name": "where",
           "valueType": "string",
diff --git a/server/csharp/MetaObjects/Persistence/Db/DbConstants.cs b/server/csharp/MetaObjects/Persistence/Db/DbConstants.cs
@@ -30,6 +30,10 @@ public static class DbConstants
     public const string IDENTITY_ATTR_ORDERS           = "orders";
     /// <summary>identity.secondary: partial-index predicate (raw SQL). When set, the index covers only matching rows.</summary>
     public const string IDENTITY_ATTR_WHERE            = "where";
+    /// <summary>identity.secondary: raw key EXPRESSION for a functional/expression index (used instead of @fields).</summary>
+    public const string IDENTITY_ATTR_EXPR             = "expr";
+    /// <summary>identity.secondary: index access method (e.g. "gin"); default "btree".</summary>
+    public const string IDENTITY_ATTR_USING            = "using";
     /// <summary>identity.reference: physical FK constraint-name override. Absent =&gt; the auto-derived default.</summary>
     public const string IDENTITY_ATTR_CONSTRAINT_NAME  = "constraintName";
 
diff --git a/server/csharp/MetaObjects/Persistence/Db/DbProvider.cs b/server/csharp/MetaObjects/Persistence/Db/DbProvider.cs
@@ -48,7 +48,7 @@ public void RegisterTypes(TypeRegistry registry)
         registry.Extend(
             MetaObjects.Shared.BaseTypes.TYPE_IDENTITY,
             IdentityConstants.IDENTITY_SUBTYPE_SECONDARY,
-            attributes: [DbSchema.OrdersSchema, DbSchema.WhereSchema]);
+            attributes: [DbSchema.OrdersSchema, DbSchema.WhereSchema, DbSchema.ExprSchema, DbSchema.UsingSchema]);
         registry.Extend(
             MetaObjects.Shared.BaseTypes.TYPE_IDENTITY,
             IdentityConstants.IDENTITY_SUBTYPE_REFERENCE,
diff --git a/server/csharp/MetaObjects/Persistence/Db/DbSchema.cs b/server/csharp/MetaObjects/Persistence/Db/DbSchema.cs
@@ -61,6 +61,20 @@ public static class DbSchema
         Required: false,
         Description: "Partial-index predicate (raw SQL, e.g. \"delivered_at IS NULL\"). When set, the index covers only rows matching the predicate — smaller and cheaper for queries that always filter on it. Absent = a full index over every row. RDB-physical — contributed by the db provider.");
 
+    /// <summary><c>@expr</c> — raw key expression for a functional/expression index; on identity.secondary.</summary>
+    public static readonly AttrSchema ExprSchema = new AttrSchema(
+        Name: DbConstants.IDENTITY_ATTR_EXPR,
+        ValueType: AttrConstants.ATTR_SUBTYPE_STRING,
+        Required: false,
+        Description: "Raw key EXPRESSION for a functional/expression index (e.g. \"lower(email)\"). Used INSTEAD of @fields — the index key is the expression rather than plain columns. RDB-physical — contributed by the db provider.");
+
+    /// <summary><c>@using</c> — index access method (e.g. "gin"); default "btree"; on identity.secondary.</summary>
+    public static readonly AttrSchema UsingSchema = new AttrSchema(
+        Name: DbConstants.IDENTITY_ATTR_USING,
+        ValueType: AttrConstants.ATTR_SUBTYPE_STRING,
+        Required: false,
+        Description: "Index access method (e.g. \"gin\", \"gist\", \"hash\"); default \"btree\" (not rendered). Pair with @expr for e.g. a GIN index over an array/jsonb expression. RDB-physical — contributed by the db provider.");
+
     /// <summary><c>@constraintName</c> — physical FK constraint-name override; on identity.reference.</summary>
     public static readonly AttrSchema ConstraintNameSchema = new AttrSchema(
         Name: DbConstants.IDENTITY_ATTR_CONSTRAINT_NAME,
diff --git a/server/java/metadata/src/main/java/com/metaobjects/identity/SecondaryIdentity.java b/server/java/metadata/src/main/java/com/metaobjects/identity/SecondaryIdentity.java
@@ -31,6 +31,10 @@ public class SecondaryIdentity extends MetaIdentity {
     public static final String ATTR_ORDERS = "orders";
     /** Partial-index predicate (raw SQL) — db-provider attr (RDB-physical). */
     public static final String ATTR_WHERE = "where";
+    /** Raw key EXPRESSION for a functional/expression index — db-provider attr (RDB-physical). */
+    public static final String ATTR_EXPR = "expr";
+    /** Index access method (e.g. "gin"); default "btree" — db-provider attr (RDB-physical). */
+    public static final String ATTR_USING = "using";
 
     public SecondaryIdentity(String name) {
         super(SUBTYPE_SECONDARY, name);
@@ -60,6 +64,8 @@ public static void registerTypes(MetaDataRegistry registry) {
             // (asc|desc) is not part of the v1 manifest contract.
             def.optionalAttributeWithConstraints(ATTR_ORDERS).ofType(StringAttribute.SUBTYPE_STRING).asArray();
             def.optionalAttributeWithConstraints(ATTR_WHERE).ofType(StringAttribute.SUBTYPE_STRING).asSingle();
+            def.optionalAttributeWithConstraints(ATTR_EXPR).ofType(StringAttribute.SUBTYPE_STRING).asSingle();
+            def.optionalAttributeWithConstraints(ATTR_USING).ofType(StringAttribute.SUBTYPE_STRING).asSingle();
 
             // ACCEPTS ANY ATTRIBUTES (for extensibility from service providers)
             def.optionalChild(MetaAttribute.TYPE_ATTR, "*", "*");
diff --git a/server/python/src/metaobjects/meta/persistence/db/db_constants.py b/server/python/src/metaobjects/meta/persistence/db/db_constants.py
@@ -51,5 +51,9 @@
 IDENTITY_SECONDARY_ATTR_ORDERS = "orders"
 # identity.secondary: partial-index predicate (raw SQL).
 IDENTITY_SECONDARY_ATTR_WHERE = "where"
+# identity.secondary: raw key EXPRESSION for a functional/expression index (used instead of @fields).
+IDENTITY_SECONDARY_ATTR_EXPR = "expr"
+# identity.secondary: index access method (e.g. "gin"); default "btree".
+IDENTITY_SECONDARY_ATTR_USING = "using"
 # identity.reference: physical FK constraint-name override.
 IDENTITY_REFERENCE_ATTR_CONSTRAINT_NAME = "constraintName"
diff --git a/server/python/src/metaobjects/meta/persistence/db/db_provider.py b/server/python/src/metaobjects/meta/persistence/db/db_provider.py
@@ -30,6 +30,8 @@
     IDENTITY_REFERENCE_ATTR_CONSTRAINT_NAME,
     IDENTITY_SECONDARY_ATTR_ORDERS,
     IDENTITY_SECONDARY_ATTR_WHERE,
+    IDENTITY_SECONDARY_ATTR_EXPR,
+    IDENTITY_SECONDARY_ATTR_USING,
 )
 
 # Every field subtype the DB-domain attrs apply to: the shared FIELD_SUBTYPES tuple
@@ -77,6 +79,26 @@
         "RDB-physical — contributed by the db provider."
     ),
 )
+_EXPR_SCHEMA = AttrSchema(
+    name=IDENTITY_SECONDARY_ATTR_EXPR,
+    value_type=ATTR_SUBTYPE_STRING,
+    required=False,
+    description=(
+        'Raw key EXPRESSION for a functional/expression index (e.g. "lower(email)"). '
+        "Used INSTEAD of @fields — the index key is the expression rather than plain "
+        "columns. RDB-physical — contributed by the db provider."
+    ),
+)
+_USING_SCHEMA = AttrSchema(
+    name=IDENTITY_SECONDARY_ATTR_USING,
+    value_type=ATTR_SUBTYPE_STRING,
+    required=False,
+    description=(
+        'Index access method (e.g. "gin", "gist", "hash"); default "btree" (not '
+        "rendered). Pair with @expr for e.g. a GIN index over an array/jsonb "
+        "expression. RDB-physical — contributed by the db provider."
+    ),
+)
 _CONSTRAINT_NAME_SCHEMA = AttrSchema(
     name=IDENTITY_REFERENCE_ATTR_CONSTRAINT_NAME,
     value_type=ATTR_SUBTYPE_STRING,
@@ -101,7 +123,7 @@ def _register(registry: TypeRegistry) -> None:
     registry.extend(
         TYPE_IDENTITY,
         IDENTITY_SUBTYPE_SECONDARY,
-        attributes=[_ORDERS_SCHEMA, _WHERE_SCHEMA],
+        attributes=[_ORDERS_SCHEMA, _WHERE_SCHEMA, _EXPR_SCHEMA, _USING_SCHEMA],
     )
     registry.extend(
         TYPE_IDENTITY,
diff --git a/server/typescript/packages/metadata/src/persistence/db/db-constants.ts b/server/typescript/packages/metadata/src/persistence/db/db-constants.ts
@@ -19,6 +19,10 @@ export const FIELD_ATTR_DB_INDEXED = "db.indexed";
 export const IDENTITY_ATTR_ORDERS = "orders";
 /** identity.secondary: a partial-index predicate (raw SQL). When set, the index covers only matching rows. */
 export const IDENTITY_ATTR_WHERE = "where";
+/** identity.secondary: a raw key EXPRESSION for a functional/expression index (e.g. "lower(email)"); used instead of @fields. */
+export const IDENTITY_ATTR_EXPR = "expr";
+/** identity.secondary: index access method (e.g. "gin", "gist"); default "btree", which is not rendered. */
+export const IDENTITY_ATTR_USING = "using";
 /** identity.reference: physical FK constraint-name override. Absent → the auto-derived `<table>_<firstFkColumn>_fk`. */
 export const IDENTITY_ATTR_CONSTRAINT_NAME = "constraintName";
 
diff --git a/server/typescript/packages/metadata/src/persistence/db/db-definition.embedded.ts b/server/typescript/packages/metadata/src/persistence/db/db-definition.embedded.ts
@@ -197,6 +197,22 @@ export const DB_DEFINITION: ProviderDefinition = {
           "min": 0,
           "max": 1,
           "description": "Partial-index predicate (raw SQL, e.g. \"delivered_at IS NULL\"). When set, the index covers only rows matching the predicate — smaller and cheaper for queries that always filter on it. Absent = a full index over every row. RDB-physical — contributed by the db provider."
+        },
+        {
+          "type": "attr",
+          "subType": "string",
+          "name": "expr",
+          "min": 0,
+          "max": 1,
+          "description": "Raw key EXPRESSION for a functional/expression index (e.g. \"lower(email)\"). Used INSTEAD of @fields — the index key is the expression rather than plain columns. RDB-physical — contributed by the db provider."
+        },
+        {
+          "type": "attr",
+          "subType": "string",
+          "name": "using",
+          "min": 0,
+          "max": 1,
+          "description": "Index access method (e.g. \"gin\", \"gist\", \"hash\"); default \"btree\" (not rendered). Pair with @expr for e.g. a GIN index over an array/jsonb expression. RDB-physical — contributed by the db provider."
         }
       ]
     },
diff --git a/server/typescript/packages/migrate-ts/src/diff/index.ts b/server/typescript/packages/migrate-ts/src/diff/index.ts
@@ -392,6 +392,14 @@ function columnDefaultsEqual(a: ColumnDescriptor["default"], b: ColumnDescriptor
 
 function indexEquals(a: IndexDescriptor, b: IndexDescriptor): boolean {
   if (a.unique !== b.unique) return false;
+  // Access method: absent = "btree" (the default).
+  if ((a.using ?? "btree") !== (b.using ?? "btree")) return false;
+  // Expression-key index: compare the normalized key expression (same canonicalizer
+  // as predicates). Both-absent = plain index; one-absent = different.
+  if ((a.expr === undefined) !== (b.expr === undefined)) return false;
+  if (a.expr !== undefined && b.expr !== undefined) {
+    if (normalizeCheckExpr(a.expr) !== normalizeCheckExpr(b.expr)) return false;
+  }
   if (a.columns.length !== b.columns.length) return false;
   if (!a.columns.every((c, i) => c === b.columns[i])) return false;
   // Per-column ordering: an absent `orders` means all-ascending, so compare against
diff --git a/server/typescript/packages/migrate-ts/src/emit/postgres.ts b/server/typescript/packages/migrate-ts/src/emit/postgres.ts
@@ -213,16 +213,21 @@ function renderDefault(d: ColumnDefault): string {
 
 function renderCreateIndex(table: string, schema: string | undefined, ix: IndexDescriptor): string {
   const u = ix.unique ? "UNIQUE " : "";
-  // Per-column key list, appending `DESC` where @orders marks a descending key (ASC
-  // is the default and intentionally not rendered, matching PG's canonical def).
-  const keys = ix.columns
-    .map((c, i) => (ix.orders?.[i] === "desc" ? `${quote(c)} DESC` : quote(c)))
-    .join(", ");
+  // Key list: a raw expression (functional/expression index) takes precedence over
+  // the per-column list. The expression is emitted verbatim (already valid SQL).
+  const keys = ix.expr
+    ? ix.expr
+    : ix.columns
+        .map((c, i) => (ix.orders?.[i] === "desc" ? `${quote(c)} DESC` : quote(c)))
+        .join(", ");
+  // Access method: btree is the default and intentionally not rendered (matches PG's
+  // canonical def); anything else (gin/gist/hash/…) is emitted as `USING <method>`.
+  const using = ix.using && ix.using !== "btree" ? ` USING ${ix.using}` : "";
   // Partial-index predicate, when present.
   const where = ix.where ? ` WHERE (${ix.where})` : "";
   // Index name itself is unqualified in CREATE INDEX (Postgres places the index
   // in the same schema as the table being indexed). Only the ON clause needs qualification.
-  return `CREATE ${u}INDEX ${quote(ix.name)} ON ${quoteQualified(table, schema)} (${keys})${where};`;
+  return `CREATE ${u}INDEX ${quote(ix.name)} ON ${quoteQualified(table, schema)}${using} (${keys})${where};`;
 }
 
 function renderAddFk(table: string, schema: string | undefined, fk: FkDescriptor): string {
diff --git a/server/typescript/packages/migrate-ts/src/expected-schema.ts b/server/typescript/packages/migrate-ts/src/expected-schema.ts
@@ -11,6 +11,8 @@ import {
   IDENTITY_ATTR_UNIQUE,
   IDENTITY_ATTR_ORDERS,
   IDENTITY_ATTR_WHERE,
+  IDENTITY_ATTR_EXPR,
+  IDENTITY_ATTR_USING,
   IDENTITY_ATTR_CONSTRAINT_NAME,
   FIELD_ATTR_DEFAULT,
   FIELD_ATTR_MAX_LENGTH,
@@ -322,18 +324,26 @@ function buildSecondaryIndexes(
   // Drizzle emits the index using the identity's @name attr directly (no table
   // prefix), so the expected name must match.
   for (const identity of entity.secondaryIdentities()) {
+    const exprRaw = identity.ownAttr(IDENTITY_ATTR_EXPR);
+    const expr = typeof exprRaw === "string" && exprRaw.trim().length > 0 ? exprRaw.trim() : undefined;
     const fieldNames = readIdentityFields(identity);
-    if (fieldNames.length === 0) continue;
+    // An expression index keys off @expr (not @fields); a plain index needs @fields.
+    if (fieldNames.length === 0 && expr === undefined) continue;
     const cols = fieldNames.map((jsName) => {
       const field = findField(entity, jsName);
       return field ? resolveColumnName(field, strategy) : applyColumnNamingStrategy(jsName, strategy);
     });
     const uniqueAttr = identity.ownAttr(IDENTITY_ATTR_UNIQUE);
     const index: IndexDescriptor = {
       name: identity.name,
-      columns: cols,
+      columns: expr ? [] : cols,
       unique: uniqueAttr !== false,
     };
+    if (expr) index.expr = expr;
+    const usingRaw = identity.ownAttr(IDENTITY_ATTR_USING);
+    if (typeof usingRaw === "string" && usingRaw.trim().length > 0 && usingRaw.trim() !== "btree") {
+      index.using = usingRaw.trim();
+    }
     // @orders — per-field sort direction (positional to @fields). Only attach when
     // at least one field is descending (an all-ascending array is the default and
     // must serialize identically to "no orders" for diff stability).
diff --git a/server/typescript/packages/migrate-ts/src/introspect/postgres.ts b/server/typescript/packages/migrate-ts/src/introspect/postgres.ts
diff --git a/server/typescript/packages/migrate-ts/src/types.ts b/server/typescript/packages/migrate-ts/src/types.ts
diff --git a/server/typescript/packages/migrate-ts/test/expression-index.test.ts b/server/typescript/packages/migrate-ts/test/expression-index.test.ts
diff --git a/spec/metamodel/db.json b/spec/metamodel/db.json

Original file line number	Diff line number	Diff line change
`@@ -226,7 +226,9 @@`
`226`	`226`	`{`
`227`	`227`	`"key": "identity.secondary",`
`228`	`228`	`"untestedAttrs": [`
	`229`	`+ "expr",`
`229`	`230`	`"orders",`
	`231`	`+ "using",`
`230`	`232`	`"where"`
`231`	`233`	`]`
`232`	`234`	`},`