Skip to content

Fix(table_diff): Correctly handle joins with composite keys where one or more of the key fields are null #5007

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
erindru merged 2 commits into from
Jul 24, 2025
Merged

Fix(table_diff): Correctly handle joins with composite keys where one or more of the key fields are null #5007

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f0d3cc9
Fix(table_diff): Correctly handle joins with composite keys where one…
erindru Jul 24, 2025
9cc8812
Add extra assertions
erindru Jul 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 8 additions & 13 deletions sqlmesh/core/table_diff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -421,21 +421,16 @@ def _column_expr(name: str, table: str) -> exp.Expression:
exp.select(
*s_selects.values(),
*t_selects.values(),
exp.func("IF", exp.or_(*(c.is_(exp.Null()).not_() for c in s_index)), 1, 0).as_(
"s_exists"
),
exp.func("IF", exp.or_(*(c.is_(exp.Null()).not_() for c in t_index)), 1, 0).as_(
"t_exists"
),
exp.func(
"IF", exp.column(SQLMESH_JOIN_KEY_COL, "s").is_(exp.Null()).not_(), 1, 0
).as_("s_exists"),
exp.func(
"IF", exp.column(SQLMESH_JOIN_KEY_COL, "t").is_(exp.Null()).not_(), 1, 0
).as_("t_exists"),
exp.func(
"IF",
exp.and_(
exp.column(SQLMESH_JOIN_KEY_COL, "s").eq(
exp.column(SQLMESH_JOIN_KEY_COL, "t")
),
exp.and_(
*(c.is_(exp.Null()).not_() for c in s_index + t_index),
),
exp.column(SQLMESH_JOIN_KEY_COL, "s").eq(
exp.column(SQLMESH_JOIN_KEY_COL, "t")
),
1,
0,
Expand Down
16 changes: 9 additions & 7 deletions tests/core/engine_adapter/integration/test_integration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2353,15 +2353,17 @@ def test_table_diff_grain_check_multiple_keys(ctx: TestContext):
row_diff = table_diff.row_diff()

assert row_diff.full_match_count == 7
assert row_diff.full_match_pct == 93.33
assert row_diff.s_only_count == 2
assert row_diff.t_only_count == 5
assert row_diff.stats["join_count"] == 4
assert row_diff.stats["null_grain_count"] == 4
assert row_diff.stats["s_count"] != row_diff.stats["distinct_count_s"]
assert row_diff.full_match_pct == 82.35
Copy link
Copy Markdown
Collaborator Author

@erindru erindru Jul 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test was identical to test_grain_check (just parameterized to run across all engine adapters) so had to be modified in the same way

assert row_diff.s_only_count == 0
assert row_diff.t_only_count == 3
assert row_diff.stats["join_count"] == 7
assert (
row_diff.stats["null_grain_count"] == 4
) # null grain currently (2025-07-24) means "any key column is null" as opposed to "all key columns are null"
assert row_diff.stats["distinct_count_s"] == 7
assert row_diff.stats["t_count"] != row_diff.stats["distinct_count_t"]
assert row_diff.stats["s_count"] == row_diff.stats["distinct_count_s"]
assert row_diff.stats["distinct_count_t"] == 10
assert row_diff.stats["t_count"] == row_diff.stats["distinct_count_t"]
assert row_diff.s_sample.shape == (row_diff.s_only_count, 3)
assert row_diff.t_sample.shape == (row_diff.t_only_count, 3)

Expand Down
71 changes: 62 additions & 9 deletions tests/core/test_table_diff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,15 +291,17 @@ def test_grain_check(sushi_context_fixed_date):

row_diff = diff.row_diff()
assert row_diff.full_match_count == 7
assert row_diff.full_match_pct == 93.33
assert row_diff.s_only_count == 2
assert row_diff.t_only_count == 5
assert row_diff.stats["join_count"] == 4
assert row_diff.stats["null_grain_count"] == 4
assert row_diff.stats["s_count"] != row_diff.stats["distinct_count_s"]
assert row_diff.full_match_pct == 82.35
Copy link
Copy Markdown
Collaborator Author

@erindru erindru Jul 24, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note: I checked internally with @themisvaltinos and the main thing this was testing was the full_match_count (which hasnt changed).

The other values were just what happened to be getting output at the time, which are not correct. This bug has been around for a while

Copy link
Copy Markdown
Collaborator

@themisvaltinos themisvaltinos Jul 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe worth adding assertions for row_diff.partial_match_pctand row_diff.s_only_pct being 0 and row_diff.t_only_pct being 17.65

so that we can pick up with the test if we alter the query logic in the future any possible regression. as amongst other things this fix also fixes the partial matches that before were negative.

for example a wrong output with null columns before this fix would show the same column wrongly in both s and t samples and the percentages would be wrong as well:
Screenshot 2025-07-24 at 12 12 16

after the fix:
Screenshot 2025-07-24 at 12 12 32

Copy link
Copy Markdown
Collaborator Author

@erindru erindru Jul 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, i've added some extra assertions.

Although i'm doing what the original PR did and trusting that the current results are correct because I don't understand some of the math used to calculate the percentages (like why do we multiply full_match_count by 2 to figure out the percentage for full_match_pct?)

Copy link
Copy Markdown
Collaborator

@themisvaltinos themisvaltinos Jul 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah there is some magic going on.. we multiply by 2 since inside _pct we divide by source count+target count. if full match count was 10 out of 20 source and 20 target columns this would be 10 / 40 so 25% instead of 50%. so we multiply by 2 to account for that

assert row_diff.s_only_count == 0
assert row_diff.t_only_count == 3
assert row_diff.stats["join_count"] == 7
assert (
row_diff.stats["null_grain_count"] == 4
) # null grain currently (2025-07-24) means "any key column is null" as opposed to "all key columns are null"
assert row_diff.stats["distinct_count_s"] == 7
assert row_diff.stats["t_count"] != row_diff.stats["distinct_count_t"]
assert row_diff.stats["distinct_count_t"] == 10
assert row_diff.stats["s_count"] == row_diff.stats["distinct_count_s"]
assert row_diff.stats["t_count"] == row_diff.stats["distinct_count_t"]
assert row_diff.s_sample.shape == (row_diff.s_only_count, 3)
assert row_diff.t_sample.shape == (row_diff.t_only_count, 3)

Expand Down Expand Up @@ -329,7 +331,7 @@ def test_generated_sql(sushi_context_fixed_date: Context, mocker: MockerFixture)
),
)

query_sql = 'CREATE TABLE IF NOT EXISTS "memory"."sqlmesh_temp_test"."__temp_diff_abcdefgh" AS WITH "__source" AS (SELECT "s"."key", "s"."value", "s"."key" AS "__sqlmesh_join_key" FROM "table_diff_source" AS "s"), "__target" AS (SELECT "t"."key", "t"."value", "t"."key" AS "__sqlmesh_join_key" FROM "table_diff_target" AS "t"), "__stats" AS (SELECT "s"."key" AS "s__key", "s"."value" AS "s__value", "s"."__sqlmesh_join_key" AS "s____sqlmesh_join_key", "t"."key" AS "t__key", "t"."value" AS "t__value", "t"."__sqlmesh_join_key" AS "t____sqlmesh_join_key", CASE WHEN NOT "s"."key" IS NULL THEN 1 ELSE 0 END AS "s_exists", CASE WHEN NOT "t"."key" IS NULL THEN 1 ELSE 0 END AS "t_exists", CASE WHEN "s"."__sqlmesh_join_key" = "t"."__sqlmesh_join_key" AND (NOT "s"."key" IS NULL AND NOT "t"."key" IS NULL) THEN 1 ELSE 0 END AS "row_joined", CASE WHEN "s"."key" IS NULL AND "t"."key" IS NULL THEN 1 ELSE 0 END AS "null_grain", CASE WHEN "s"."key" = "t"."key" THEN 1 WHEN ("s"."key" IS NULL) AND ("t"."key" IS NULL) THEN 1 WHEN ("s"."key" IS NULL) OR ("t"."key" IS NULL) THEN 0 ELSE 0 END AS "key_matches", CASE WHEN ROUND("s"."value", 3) = ROUND("t"."value", 3) THEN 1 WHEN ("s"."value" IS NULL) AND ("t"."value" IS NULL) THEN 1 WHEN ("s"."value" IS NULL) OR ("t"."value" IS NULL) THEN 0 ELSE 0 END AS "value_matches" FROM "__source" AS "s" FULL JOIN "__target" AS "t" ON "s"."__sqlmesh_join_key" = "t"."__sqlmesh_join_key") SELECT *, CASE WHEN "key_matches" = 1 AND "value_matches" = 1 THEN 1 ELSE 0 END AS "row_full_match" FROM "__stats"'
query_sql = 'CREATE TABLE IF NOT EXISTS "memory"."sqlmesh_temp_test"."__temp_diff_abcdefgh" AS WITH "__source" AS (SELECT "s"."key", "s"."value", "s"."key" AS "__sqlmesh_join_key" FROM "table_diff_source" AS "s"), "__target" AS (SELECT "t"."key", "t"."value", "t"."key" AS "__sqlmesh_join_key" FROM "table_diff_target" AS "t"), "__stats" AS (SELECT "s"."key" AS "s__key", "s"."value" AS "s__value", "s"."__sqlmesh_join_key" AS "s____sqlmesh_join_key", "t"."key" AS "t__key", "t"."value" AS "t__value", "t"."__sqlmesh_join_key" AS "t____sqlmesh_join_key", CASE WHEN NOT "s"."__sqlmesh_join_key" IS NULL THEN 1 ELSE 0 END AS "s_exists", CASE WHEN NOT "t"."__sqlmesh_join_key" IS NULL THEN 1 ELSE 0 END AS "t_exists", CASE WHEN "s"."__sqlmesh_join_key" = "t"."__sqlmesh_join_key" THEN 1 ELSE 0 END AS "row_joined", CASE WHEN "s"."key" IS NULL AND "t"."key" IS NULL THEN 1 ELSE 0 END AS "null_grain", CASE WHEN "s"."key" = "t"."key" THEN 1 WHEN ("s"."key" IS NULL) AND ("t"."key" IS NULL) THEN 1 WHEN ("s"."key" IS NULL) OR ("t"."key" IS NULL) THEN 0 ELSE 0 END AS "key_matches", CASE WHEN ROUND("s"."value", 3) = ROUND("t"."value", 3) THEN 1 WHEN ("s"."value" IS NULL) AND ("t"."value" IS NULL) THEN 1 WHEN ("s"."value" IS NULL) OR ("t"."value" IS NULL) THEN 0 ELSE 0 END AS "value_matches" FROM "__source" AS "s" FULL JOIN "__target" AS "t" ON "s"."__sqlmesh_join_key" = "t"."__sqlmesh_join_key") SELECT *, CASE WHEN "key_matches" = 1 AND "value_matches" = 1 THEN 1 ELSE 0 END AS "row_full_match" FROM "__stats"'
summary_query_sql = 'SELECT SUM("s_exists") AS "s_count", SUM("t_exists") AS "t_count", SUM("row_joined") AS "join_count", SUM("null_grain") AS "null_grain_count", SUM("row_full_match") AS "full_match_count", SUM("key_matches") AS "key_matches", SUM("value_matches") AS "value_matches", COUNT(DISTINCT ("s____sqlmesh_join_key")) AS "distinct_count_s", COUNT(DISTINCT ("t____sqlmesh_join_key")) AS "distinct_count_t" FROM "memory"."sqlmesh_temp_test"."__temp_diff_abcdefgh"'
compare_sql = 'SELECT ROUND(100 * (CAST(SUM("key_matches") AS DECIMAL) / COUNT("key_matches")), 9) AS "key_matches", ROUND(100 * (CAST(SUM("value_matches") AS DECIMAL) / COUNT("value_matches")), 9) AS "value_matches" FROM "memory"."sqlmesh_temp_test"."__temp_diff_abcdefgh" WHERE "row_joined" = 1'
sample_query_sql = 'WITH "source_only" AS (SELECT \'source_only\' AS "__sqlmesh_sample_type", "s__key", "s__value", "s____sqlmesh_join_key", "t__key", "t__value", "t____sqlmesh_join_key" FROM "memory"."sqlmesh_temp_test"."__temp_diff_abcdefgh" WHERE "s_exists" = 1 AND "row_joined" = 0 ORDER BY "s__key" NULLS FIRST LIMIT 20), "target_only" AS (SELECT \'target_only\' AS "__sqlmesh_sample_type", "s__key", "s__value", "s____sqlmesh_join_key", "t__key", "t__value", "t____sqlmesh_join_key" FROM "memory"."sqlmesh_temp_test"."__temp_diff_abcdefgh" WHERE "t_exists" = 1 AND "row_joined" = 0 ORDER BY "t__key" NULLS FIRST LIMIT 20), "common_rows" AS (SELECT \'common_rows\' AS "__sqlmesh_sample_type", "s__key", "s__value", "s____sqlmesh_join_key", "t__key", "t__value", "t____sqlmesh_join_key" FROM "memory"."sqlmesh_temp_test"."__temp_diff_abcdefgh" WHERE "row_joined" = 1 AND "row_full_match" = 0 ORDER BY "s__key" NULLS FIRST, "t__key" NULLS FIRST LIMIT 20) SELECT "__sqlmesh_sample_type", "s__key", "s__value", "s____sqlmesh_join_key", "t__key", "t__value", "t____sqlmesh_join_key" FROM "source_only" UNION ALL SELECT "__sqlmesh_sample_type", "s__key", "s__value", "s____sqlmesh_join_key", "t__key", "t__value", "t____sqlmesh_join_key" FROM "target_only" UNION ALL SELECT "__sqlmesh_sample_type", "s__key", "s__value", "s____sqlmesh_join_key", "t__key", "t__value", "t____sqlmesh_join_key" FROM "common_rows"'
Expand Down Expand Up @@ -369,7 +371,7 @@ def test_generated_sql(sushi_context_fixed_date: Context, mocker: MockerFixture)
where="key = 2",
)

query_sql_where = 'CREATE TABLE IF NOT EXISTS "memory"."sqlmesh_temp"."__temp_diff_abcdefgh" AS WITH "__source" AS (SELECT "s"."key", "s"."value", "s"."key" AS "__sqlmesh_join_key" FROM "table_diff_source" AS "s" WHERE "s"."key" = 2), "__target" AS (SELECT "t"."key", "t"."value", "t"."key" AS "__sqlmesh_join_key" FROM "table_diff_target" AS "t" WHERE "t"."key" = 2), "__stats" AS (SELECT "s"."key" AS "s__key", "s"."value" AS "s__value", "s"."__sqlmesh_join_key" AS "s____sqlmesh_join_key", "t"."key" AS "t__key", "t"."value" AS "t__value", "t"."__sqlmesh_join_key" AS "t____sqlmesh_join_key", CASE WHEN NOT "s"."key" IS NULL THEN 1 ELSE 0 END AS "s_exists", CASE WHEN NOT "t"."key" IS NULL THEN 1 ELSE 0 END AS "t_exists", CASE WHEN "s"."__sqlmesh_join_key" = "t"."__sqlmesh_join_key" AND (NOT "s"."key" IS NULL AND NOT "t"."key" IS NULL) THEN 1 ELSE 0 END AS "row_joined", CASE WHEN "s"."key" IS NULL AND "t"."key" IS NULL THEN 1 ELSE 0 END AS "null_grain", CASE WHEN "s"."key" = "t"."key" THEN 1 WHEN ("s"."key" IS NULL) AND ("t"."key" IS NULL) THEN 1 WHEN ("s"."key" IS NULL) OR ("t"."key" IS NULL) THEN 0 ELSE 0 END AS "key_matches", CASE WHEN ROUND("s"."value", 3) = ROUND("t"."value", 3) THEN 1 WHEN ("s"."value" IS NULL) AND ("t"."value" IS NULL) THEN 1 WHEN ("s"."value" IS NULL) OR ("t"."value" IS NULL) THEN 0 ELSE 0 END AS "value_matches" FROM "__source" AS "s" FULL JOIN "__target" AS "t" ON "s"."__sqlmesh_join_key" = "t"."__sqlmesh_join_key") SELECT *, CASE WHEN "key_matches" = 1 AND "value_matches" = 1 THEN 1 ELSE 0 END AS "row_full_match" FROM "__stats"'
query_sql_where = 'CREATE TABLE IF NOT EXISTS "memory"."sqlmesh_temp"."__temp_diff_abcdefgh" AS WITH "__source" AS (SELECT "s"."key", "s"."value", "s"."key" AS "__sqlmesh_join_key" FROM "table_diff_source" AS "s" WHERE "s"."key" = 2), "__target" AS (SELECT "t"."key", "t"."value", "t"."key" AS "__sqlmesh_join_key" FROM "table_diff_target" AS "t" WHERE "t"."key" = 2), "__stats" AS (SELECT "s"."key" AS "s__key", "s"."value" AS "s__value", "s"."__sqlmesh_join_key" AS "s____sqlmesh_join_key", "t"."key" AS "t__key", "t"."value" AS "t__value", "t"."__sqlmesh_join_key" AS "t____sqlmesh_join_key", CASE WHEN NOT "s"."__sqlmesh_join_key" IS NULL THEN 1 ELSE 0 END AS "s_exists", CASE WHEN NOT "t"."__sqlmesh_join_key" IS NULL THEN 1 ELSE 0 END AS "t_exists", CASE WHEN "s"."__sqlmesh_join_key" = "t"."__sqlmesh_join_key" THEN 1 ELSE 0 END AS "row_joined", CASE WHEN "s"."key" IS NULL AND "t"."key" IS NULL THEN 1 ELSE 0 END AS "null_grain", CASE WHEN "s"."key" = "t"."key" THEN 1 WHEN ("s"."key" IS NULL) AND ("t"."key" IS NULL) THEN 1 WHEN ("s"."key" IS NULL) OR ("t"."key" IS NULL) THEN 0 ELSE 0 END AS "key_matches", CASE WHEN ROUND("s"."value", 3) = ROUND("t"."value", 3) THEN 1 WHEN ("s"."value" IS NULL) AND ("t"."value" IS NULL) THEN 1 WHEN ("s"."value" IS NULL) OR ("t"."value" IS NULL) THEN 0 ELSE 0 END AS "value_matches" FROM "__source" AS "s" FULL JOIN "__target" AS "t" ON "s"."__sqlmesh_join_key" = "t"."__sqlmesh_join_key") SELECT *, CASE WHEN "key_matches" = 1 AND "value_matches" = 1 THEN 1 ELSE 0 END AS "row_full_match" FROM "__stats"'
spy_execute.assert_any_call(query_sql_where)


Expand Down Expand Up @@ -1137,3 +1139,54 @@ def test_data_diff_sample_limit():
assert len(diff.s_sample) == 3
assert len(diff.t_sample) == 3
assert len(diff.joined_sample) == 3


def test_data_diff_nulls_in_some_grain_columns():
engine_adapter = DuckDBConnectionConfig().create_engine_adapter()

columns_to_types = {
"key1": exp.DataType.build("int"),
"key2": exp.DataType.build("varchar"),
"key3": exp.DataType.build("int"),
"value": exp.DataType.build("varchar"),
}

engine_adapter.create_table("src", columns_to_types)
engine_adapter.create_table("target", columns_to_types)

src_records = [
(1, None, 1, "value"), # full match
(None, None, None, "null value"), # join, partial match
(2, None, None, "source only"), # source only
]

target_records = [
(1, None, 1, "value"), # full match
(None, None, None, "null value modified"), # join, partial match
(None, "three", 2, "target only"), # target only
]

src_df = pd.DataFrame(data=src_records, columns=columns_to_types.keys())
target_df = pd.DataFrame(data=target_records, columns=columns_to_types.keys())

engine_adapter.insert_append("src", src_df)
engine_adapter.insert_append("target", target_df)

table_diff = TableDiff(
adapter=engine_adapter, source="src", target="target", on=["key1", "key2", "key3"]
)

diff = table_diff.row_diff()

assert diff.join_count == 2
assert diff.s_only_count == 1
assert diff.t_only_count == 1
assert diff.full_match_count == 1
assert diff.partial_match_count == 1

assert diff.s_sample["value"].tolist() == ["source only"]
assert diff.t_sample["value"].tolist() == ["target only"]
assert diff.joined_sample[["s__value", "t__value"]].values.flatten().tolist() == [
"null value",
"null value modified",
]