fix(tables): canonicalize filter operands where the column is always known

Greptile was right that the earlier fix was incomplete. Canonicalization lived
in the UI-facing converters, which only run it when the caller supplies column
definitions — and the workflow Table blocks build a filter from user input long
before any schema exists, so they call those converters with none. A workflow
filtering an Email column for `Ada@Example.com`, or a Phone column for
`020 1234 5678`, compiled an operand that never met the stored value and
silently returned no rows.

Moved to `fieldPredicate`, which is the one place every filter arrives with its
column in hand: both wire grammars compile through it, so raw API callers and
the copilot tool are covered too. The converter pass stays as a convenience for
the filter UI rather than as the guarantee.

Only TEXT operands are canonicalized. `date.coerce` accepts an epoch number, so
running a number through it would silently reinterpret
`{ birthDate: { $gte: 1704067200000 } }` as a date instead of letting the
range validator reject a likely mistake. Opaque ids are skipped — they are
resolved from option names upstream, where the option set is known — and an
operand the type rejects is left alone rather than nulled, so the error still
names what the user typed.

Also routes CSV number coercion AND number inference through the same decimal
parser inline edits use. A CSV could import `0x10` as 16 where typing it was
rejected; had only coercion been fixed, a hex column would have inferred
`number` and then nulled every cell.

Author: waleedlatif1

apps/sim/lib/table/__tests__/column-types-contact.test.ts

@@ -23,6 +23,7 @@ import { metadataMigrationFor } from '@/lib/table/column-types/registry.server'
 import { buildConvertedColumn } from '@/lib/table/columns/service'
 import { coerceValue } from '@/lib/table/import'
 import { filterRulesToFilter, prunePredicateForColumns } from '@/lib/table/query-builder/converters'
+import { buildFilterClause } from '@/lib/table/sql'
 import type { ColumnDefinition } from '@/lib/table/types'
 
 const column = (type: ColumnDefinition['type'], extra: Partial<ColumnDefinition> = {}) =>
@@ -510,6 +511,81 @@ describe('final-audit regressions', () => {
   })
 })
 
+describe('server-side operand canonicalization', () => {
+  const emailCol: ColumnDefinition = { id: 'c', name: 'c', type: 'email' }
+  const phoneCol: ColumnDefinition = { id: 'c', name: 'c', type: 'phone' }
+  const render = (clause: unknown) => JSON.stringify(clause)
+
+  it('canonicalizes even when the caller supplies NO column definitions', () => {
+    // The workflow Table blocks build a filter long before any schema is known
+    // and call the converters with no columns, so the client-side pass is
+    // skipped entirely. The SQL builder is the one place the column is always
+    // in hand, which is why the guarantee lives there.
+    const withoutColumns = filterRulesToFilter(
+      [
+        {
+          id: 'r1',
+          logicalOperator: 'and' as const,
+          column: 'c',
+          operator: 'eq',
+          value: 'Ada@Example.COM',
+        },
+      ],
+      []
+    )
+    // The converter left it raw...
+    expect(withoutColumns).toEqual({ c: 'Ada@Example.COM' })
+    // ...and the SQL builder still compiles the canonical operand.
+    const clause = buildFilterClause(withoutColumns as Filter, 'user_table_rows', [emailCol])
+    expect(render(clause)).toContain('ada@example.com')
+    expect(render(clause)).not.toContain('Ada@Example.COM')
+  })
+
+  it('canonicalizes a phone operand arriving formatted from a workflow', () => {
+    const clause = buildFilterClause(
+      { c: { $eq: '+44 20 7123 4567' } } as Filter,
+      'user_table_rows',
+      [phoneCol]
+    )
+    expect(render(clause)).toContain('+442071234567')
+  })
+
+  it('normalizes a fragment for a text match, not the whole value', () => {
+    const clause = buildFilterClause(
+      { c: { $contains: '(555) 123' } } as Filter,
+      'user_table_rows',
+      [phoneCol]
+    )
+    expect(render(clause)).toContain('555123')
+  })
+
+  it('is idempotent, so an already-canonical operand is unchanged', () => {
+    const once = buildFilterClause({ c: { $eq: 'ada@example.com' } } as Filter, 'user_table_rows', [
+      emailCol,
+    ])
+    expect(render(once)).toContain('ada@example.com')
+  })
+
+  it('leaves a non-text operand to the existing validation', () => {
+    // `date.coerce` accepts an epoch, so canonicalizing a NUMBER would silently
+    // reinterpret a likely mistake instead of rejecting it.
+    expect(() =>
+      buildFilterClause({ d: { $gte: 1704067200000 } } as Filter, 'user_table_rows', [
+        { id: 'd', name: 'd', type: 'date' },
+      ])
+    ).toThrow(/requires a date string, got number/)
+  })
+
+  it('leaves an unparseable operand alone rather than nulling it', () => {
+    const clause = buildFilterClause(
+      { c: { $eq: 'not an address' } } as Filter,
+      'user_table_rows',
+      [emailCol]
+    )
+    expect(render(clause)).toContain('not an address')
+  })
+})
+
 describe('date includeTime', () => {
   it('truncates to a calendar day only when includeTime is explicitly false', () => {
     const dateOnly = column('date', { includeTime: false })

apps/sim/lib/table/import.ts

@@ -16,6 +16,7 @@ import { getColumnId } from '@/lib/table/column-keys'
 import { type ColumnType, columnTypeById } from '@/lib/table/column-types'
 import { parseCurrencyInput } from '@/lib/table/currency'
 import { type NormalizeDateCellOptions, normalizeDateCellValue } from '@/lib/table/dates'
+import { parseDecimalNumber } from '@/lib/table/numeric'
 import type { ColumnDefinition, JsonValue, RowData, TableSchema } from '@/lib/table/types'
 
 /**
@@ -295,10 +296,10 @@ export function inferColumnType(values: unknown[]): InferredCsvColumnType {
   const nonEmpty = values.filter((v) => v !== null && v !== undefined && v !== '')
   if (nonEmpty.length === 0) return 'string'
 
-  const allNumber = nonEmpty.every((v) => {
-    const n = Number(v)
-    return !Number.isNaN(n) && String(v).trim() !== ''
-  })
+  // The same parser `coerceValue` uses below. If inference read `0x10` as a
+  // number while coercion rejected it, the column would be created as `number`
+  // and then have every one of those cells nulled on write.
+  const allNumber = nonEmpty.every((v) => parseDecimalNumber(v) !== null)
   if (allNumber) return 'number'
 
   const allBoolean = nonEmpty.every((v) => {
@@ -405,10 +406,11 @@ export function coerceValue(
 ): string | number | boolean | null | Record<string, unknown> | unknown[] {
   if (value === null || value === undefined || value === '') return null
   switch (colType) {
-    case 'number': {
-      const n = Number(value)
-      return Number.isNaN(n) ? null : n
-    }
+    case 'number':
+      // The same decimal parser inline edits use. Bare `Number()` read `0x10`
+      // as 16 and `Infinity` as infinity, so a CSV could import values the
+      // grid would reject if typed.
+      return parseDecimalNumber(value)
     // Importing into an existing currency column: the file carries the
     // formatted amount (`$1,234.56`) but the cell stores a bare number. The
     // column's currency is forwarded because it decides how a lone separator

apps/sim/lib/table/sql.ts

@@ -14,6 +14,7 @@ import {
   columnTypeOf,
   filterOperatorsFor,
   MULTI_SELECT_OPERATORS,
+  normalizeFilterFragment,
   predicateOperatorsFor,
   SINGLE_SELECT_OPERATORS,
   storesMultipleValues,
@@ -474,12 +475,25 @@ export function fieldPredicate(
     )
   }
 
+  // Read the operand the way the column stores its cells. This is the one place
+  // every filter reaches with its column in hand — both wire grammars compile
+  // through here — so canonicalizing at this point covers callers that cannot
+  // supply column definitions at all: the workflow Table blocks build a filter
+  // from user input long before any schema is known, and a raw API caller need
+  // not know the storage form either.
+  //
+  // The client-side pass in `converters.ts` is a convenience for the filter UI,
+  // not the guarantee. When it was the only pass, a workflow filtering an Email
+  // column for `Ada@Example.com` compiled to an operand that never met the
+  // stored `ada@example.com` and silently returned no rows.
+  const operand = column ? canonicalizeOperand(value, op, column) : value
+
   if (isMultiSelect) {
     switch (op) {
       case 'contains':
-        return buildArrayMembershipClause(tableName, field, value as JsonValue)
+        return buildArrayMembershipClause(tableName, field, operand as JsonValue)
       case 'ncontains':
-        return sql`NOT (${buildArrayMembershipClause(tableName, field, value as JsonValue)})`
+        return sql`NOT (${buildArrayMembershipClause(tableName, field, operand as JsonValue)})`
       case 'isEmpty':
         return buildEmptyClause(tableName, field, true, true)
       case 'isNotEmpty':
@@ -491,55 +505,55 @@ export function fieldPredicate(
 
   switch (op) {
     case 'eq':
-      return buildContainmentClause(tableName, field, value as JsonValue)
+      return buildContainmentClause(tableName, field, operand as JsonValue)
 
     case 'ne':
-      return sql`NOT (${buildContainmentClause(tableName, field, value as JsonValue)})`
+      return sql`NOT (${buildContainmentClause(tableName, field, operand as JsonValue)})`
 
     case 'gt':
-      return buildComparisonClause(tableName, field, '>', value as number | string, columnType)
+      return buildComparisonClause(tableName, field, '>', operand as number | string, columnType)
     case 'gte':
-      return buildComparisonClause(tableName, field, '>=', value as number | string, columnType)
+      return buildComparisonClause(tableName, field, '>=', operand as number | string, columnType)
     case 'lt':
-      return buildComparisonClause(tableName, field, '<', value as number | string, columnType)
+      return buildComparisonClause(tableName, field, '<', operand as number | string, columnType)
     case 'lte':
-      return buildComparisonClause(tableName, field, '<=', value as number | string, columnType)
+      return buildComparisonClause(tableName, field, '<=', operand as number | string, columnType)
 
     case 'in': {
-      if (!Array.isArray(value) || value.length === 0) return undefined
-      if (value.length === 1) return buildContainmentClause(tableName, field, value[0])
-      const inConditions = value.map((v) => buildContainmentClause(tableName, field, v))
+      if (!Array.isArray(operand) || operand.length === 0) return undefined
+      if (operand.length === 1) return buildContainmentClause(tableName, field, operand[0])
+      const inConditions = operand.map((v) => buildContainmentClause(tableName, field, v))
       return sql`(${sql.join(inConditions, sql.raw(' OR '))})`
     }
 
     case 'nin': {
-      if (!Array.isArray(value) || value.length === 0) return undefined
-      const ninConditions = value.map(
+      if (!Array.isArray(operand) || operand.length === 0) return undefined
+      const ninConditions = operand.map(
         (v) => sql`NOT (${buildContainmentClause(tableName, field, v)})`
       )
       return sql`(${sql.join(ninConditions, sql.raw(' AND '))})`
     }
 
     case 'contains':
-      return buildLikeClause(tableName, field, value as string, 'contains')
+      return buildLikeClause(tableName, field, operand as string, 'contains')
     case 'ncontains':
-      return buildLikeClause(tableName, field, value as string, 'contains', { negate: true })
+      return buildLikeClause(tableName, field, operand as string, 'contains', { negate: true })
     case 'startsWith':
-      return buildLikeClause(tableName, field, value as string, 'startsWith')
+      return buildLikeClause(tableName, field, operand as string, 'startsWith')
     case 'endsWith':
-      return buildLikeClause(tableName, field, value as string, 'endsWith')
+      return buildLikeClause(tableName, field, operand as string, 'endsWith')
 
     case 'like':
-      return buildPatternClause(tableName, field, value as string, { caseInsensitive: false })
+      return buildPatternClause(tableName, field, operand as string, { caseInsensitive: false })
     case 'ilike':
-      return buildPatternClause(tableName, field, value as string, { caseInsensitive: true })
+      return buildPatternClause(tableName, field, operand as string, { caseInsensitive: true })
     case 'nlike':
-      return buildPatternClause(tableName, field, value as string, {
+      return buildPatternClause(tableName, field, operand as string, {
         caseInsensitive: false,
         negate: true,
       })
     case 'nilike':
-      return buildPatternClause(tableName, field, value as string, {
+      return buildPatternClause(tableName, field, operand as string, {
         caseInsensitive: true,
         negate: true,
       })
@@ -754,6 +768,64 @@ function buildArrayMembershipClause(tableName: string, field: string, value: Jso
  * Cannot use the GIN index — falls back to a sequential scan over the table's
  * rows (bounded by the btree prefix on `table_id`).
  */
+/**
+ * Canonicalizes a filter operand into the shape the column stores.
+ *
+ * Three cases, because "the value the user typed" relates to storage three
+ * different ways:
+ *
+ * - **Opaque ids** (`select`) are resolved from option NAMES to ids upstream by
+ *   `resolveFilterSelectValues`, which needs the whole option set. Touching
+ *   them here would re-coerce an already-resolved id.
+ * - **Text matches** take a FRAGMENT, which is not a whole value and so cannot
+ *   go through `coerce` — a partial phone number fails validation. The type
+ *   normalizes it instead, if its canonical form drops characters.
+ * - **Everything else** goes through the type's own `coerce`, which is by
+ *   definition how the cell was written. Idempotent, so an operand a caller
+ *   already canonicalized passes through unchanged.
+ *
+ * An operand `coerce` rejects is left alone rather than nulled: the value is
+ * the user's, and the range-operator validator below still gets to reject it
+ * with a message naming what they actually typed.
+ */
+function canonicalizeOperand(
+  value: JsonValue | undefined,
+  op: FilterOp,
+  column: ColumnDefinition
+): JsonValue | undefined {
+  if (value === undefined || value === null) return value
+  const definition = columnTypeOf(column)
+  if (definition.storesOpaqueIds || !definition.canonicalizesValues) return value
+
+  if (TEXT_MATCH_OPS.has(op)) {
+    return typeof value === 'string' ? normalizeFilterFragment(column, value) : value
+  }
+  if (Array.isArray(value)) {
+    return value.map((entry) =>
+      typeof entry === 'string' ? canonicalizeText(entry, column) : entry
+    )
+  }
+  return typeof value === 'string' ? canonicalizeText(value, column) : value
+}
+
+/**
+ * Runs one TEXT operand through the column's own coercion.
+ *
+ * Strings only, deliberately. Canonicalization exists to reconcile what a user
+ * TYPED with how the cell was stored; an operand that arrived as another type
+ * was not typed as text and keeps its existing path — which matters because
+ * `date.coerce` accepts an epoch number, so canonicalizing one would silently
+ * reinterpret `{ birthDate: { $gte: 1704067200000 } }` as a date rather than
+ * letting `validateComparisonValue` reject a likely mistake.
+ */
+function canonicalizeText(value: string, column: ColumnDefinition): JsonValue {
+  const coerced = columnTypeOf(column).coerce(value, column)
+  return coerced.ok ? coerced.value : value
+}
+
+/** Operators whose operand is a search fragment rather than a whole value. */
+const TEXT_MATCH_OPS = new Set<FilterOp>(['contains', 'ncontains', 'startsWith', 'endsWith'])
+
 function buildComparisonClause(
   tableName: string,
   field: string,