From 26817f41c1e16a1711ec3bb413e1dadb75ffca79 Mon Sep 17 00:00:00 2001 From: JuliaEdom Date: Fri, 3 Jul 2026 17:16:57 +0300 Subject: [PATCH] fix(dv2): rename record_source example x5__* -> mp__* (B2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The governance admission logic (audit_28_06_26 #12) proves it is source-agnostic by citing a third record_source convention alongside 1c__/pg_ops__. That example carried the name of the Kaggle seed dataset (X5 Retail Hero); under the legend (domain.md §5.3-5.4) it is the consolidated marketplace feed, mp__. Renamed in the 5 ClickHouse bv_customer_mdm__* view headers, the PostgreSQL port (03_business_vault.sql), the PostgreSQL governance live-verify seed literal (the one place x5__ was an actual value, not just prose), and both admission-test docstrings. Added a ratchet assertion to each admission test (checked on raw text, since the example lives in a header comment that the tests otherwise strip). domain.md §5.3/§5.4 updated to reflect the rename is done. x5__ was never a real record_source value in the generator/seeds (those emit 1c__/pg_ops__/wb__) -- it only existed as documentation vocabulary for this one audit story. --- docs/domain.md | 4 +-- tests/unit/test_dv2_business_vault_ddl.py | 26 ++++++++++++++----- tests/unit/test_dv2_postgres_ddl.py | 19 +++++++++++--- .../business_vault/bv_customer_mdm__ala.sql | 2 +- .../business_vault/bv_customer_mdm__dxb.sql | 2 +- .../business_vault/bv_customer_mdm__ekb.sql | 2 +- .../business_vault/bv_customer_mdm__msk.sql | 4 +-- .../business_vault/bv_customer_mdm__spb.sql | 2 +- .../dv2/postgres/03_business_vault.sql | 4 +-- .../dv2/postgres/governance/verify_live.sh | 2 +- 10 files changed, 45 insertions(+), 22 deletions(-) diff --git a/docs/domain.md b/docs/domain.md index bec81933..1b522352 100644 --- a/docs/domain.md +++ b/docs/domain.md @@ -197,13 +197,13 @@ renames. The two exceptions are listed in §5.4. | `wb__` | Wildberries seller API | FBS orders, commissions, returns | | `excel__` | Logistics spreadsheets | Container manifests, cross-dock — the inbound-container storyline | | `pg_ops__` | Postgres OLTP (hot tier) via CDC | Operational order/customer rows promoted into the vault | -| `mp__` (currently `x5__`) | Consolidated marketplace order feed | High-volume retail order history (see §5.4) | +| `mp__` | Consolidated marketplace order feed | High-volume retail order history | ### 5.4 Planned renames / repins (the only code changes the legend requires) | Change | Scope | Status | | ------ | ----- | ------ | -| `x5__*` → `mp__*` record_source (+ governance SQL, officer probes, admission tests) | The prefix currently carries the name of the Kaggle seed dataset (X5 Retail Hero) that the demo loader replays as transaction history. Under the legend it is the **consolidated marketplace feed**, and the prefix should say so. Dataset attribution stays in the loader README | Planned (data phase) | +| `x5__*` → `mp__*` record_source (+ governance SQL, officer probes, admission tests) | The prefix carried the name of the Kaggle seed dataset (X5 Retail Hero) that the demo loader replays as transaction history. Under the legend it is the **consolidated marketplace feed**, and the prefix says so. Dataset attribution stays in the loader README | **Done** (B2) | | Demo value repin: currencies to `RUB` (primary), `AED`/`KZT` in branch stories; demo revenue/counts consistent with §1–2 | `contracts/entities/order.yaml` currency examples, NL demo answers, seeded `ORD-*` rows | Planned (data phase, after the generator spec) | Vocabulary guardrails for all public docs: the company is an **own-brand / diff --git a/tests/unit/test_dv2_business_vault_ddl.py b/tests/unit/test_dv2_business_vault_ddl.py index 48a886df..de0ed367 100644 --- a/tests/unit/test_dv2_business_vault_ddl.py +++ b/tests/unit/test_dv2_business_vault_ddl.py @@ -4,8 +4,8 @@ this file. It pins that every view parses under sqlglot's ClickHouse dialect and that the customer MDM views admit hub rows by branch via ``splitByString('__', record_source)[2]`` — NOT by a hard-coded -``record_source = '1c__'`` filter that silently dropped OLTP/X5-promoted -customers (``record_source`` ``pg_ops__`` / ``x5__``). +``record_source = '1c__'`` filter that silently dropped OLTP/marketplace- +promoted customers (``record_source`` ``pg_ops__`` / ``mp__``). This is the ClickHouse half of audit_28_06_26 #12. The PostgreSQL port was fixed in ``test_dv2_postgres_ddl.py::test_customer_mdm_views_admit_all_source_conventions``; @@ -59,15 +59,17 @@ def test_customer_mdm_views_admit_all_source_conventions(): """audit_28_06_26 #12 (ClickHouse half): the customer MDM views must select hub rows by branch via ``splitByString('__', record_source)[2]``, NOT by a hard-coded ``record_source = '1c__'`` filter that silently drops - OLTP/X5-promoted customers (record_source ``pg_ops__`` / ``x5__``). Proven - live on PostgreSQL in #99: the buggy filter returns 1 of 2 seeded customers, - the source-agnostic filter returns both. This keeps the ClickHouse views in - lock-step with the PostgreSQL port so the engines cannot diverge again.""" + OLTP/marketplace-promoted customers (record_source ``pg_ops__`` / ``mp__``). + Proven live on PostgreSQL in #99: the buggy filter returns 1 of 2 seeded + customers, the source-agnostic filter returns both. This keeps the + ClickHouse views in lock-step with the PostgreSQL port so the engines + cannot diverge again.""" for branch in MDM_BRANCHES: path = BV_DIR / f"bv_customer_mdm__{branch}.sql" # Strip block/line comments: the headers deliberately quote the old, # buggy ``record_source = '1c__'`` filter to explain the fix. - body = _strip_comments(path.read_text(encoding="utf-8")) + raw = path.read_text(encoding="utf-8") + body = _strip_comments(raw) assert f"CREATE OR REPLACE VIEW rv.bv_customer_mdm__{branch}" in body, ( f"bv_customer_mdm__{branch}.sql must define rv.bv_customer_mdm__{branch}" ) @@ -80,3 +82,13 @@ def test_customer_mdm_views_admit_all_source_conventions(): f"hard-coded record_source = '1c__' filter in " f"bv_customer_mdm__{branch} reintroduces audit #12" ) + # B2 (domain.md §5.4): the legend's marketplace-feed vocabulary replaces + # the Kaggle dataset name in the third-source-convention example (checked + # on the RAW text — the example lives in the header comment, which body + # strips). + assert "x5__" not in raw, ( + f"stale Kaggle-dataset record_source prefix x5__ leaked back into bv_customer_mdm__{branch}.sql" + ) + assert "mp__" in raw, ( + f"bv_customer_mdm__{branch}.sql should document the mp__ marketplace-feed convention (domain.md §5.3)" + ) diff --git a/tests/unit/test_dv2_postgres_ddl.py b/tests/unit/test_dv2_postgres_ddl.py index d0a0ddba..769d2b97 100644 --- a/tests/unit/test_dv2_postgres_ddl.py +++ b/tests/unit/test_dv2_postgres_ddl.py @@ -147,10 +147,12 @@ def test_business_vault_uses_postgres_collapse(): def test_customer_mdm_views_admit_all_source_conventions(): """audit_28_06_26 #12: the customer MDM views must select hub rows by branch via split_part(record_source, '__', 2), NOT by a hard-coded - record_source = '1c__' filter that silently drops OLTP/X5-promoted - customers (record_source pg_ops__/x5__). Proven live on PG: the buggy filter - returns 1 of 2 seeded customers, the split_part filter returns both.""" - body = _strip_comments((PG_DIR / "03_business_vault.sql").read_text(encoding="utf-8")).lower() + record_source = '1c__' filter that silently drops OLTP/marketplace- + promoted customers (record_source pg_ops__/mp__). Proven live on PG: the + buggy filter returns 1 of 2 seeded customers, the split_part filter returns + both.""" + raw = (PG_DIR / "03_business_vault.sql").read_text(encoding="utf-8").lower() + body = _strip_comments(raw) branches = ("msk", "spb", "ekb", "dxb", "ala") for branch in branches: assert f"view rv.bv_customer_mdm__{branch}" in body, f"missing PG view for {branch}" @@ -161,6 +163,15 @@ def test_customer_mdm_views_admit_all_source_conventions(): assert "record_source = '1c__" not in body, ( "hard-coded record_source = '1c__' filter reintroduces audit #12" ) + # B2 (domain.md §5.4): the legend's marketplace-feed vocabulary replaces the + # Kaggle dataset name in the third-source-convention example (checked on the + # RAW text — the example lives in the header comment, which body strips). + assert "x5__" not in raw, ( + "stale Kaggle-dataset record_source prefix x5__ leaked back into 03_business_vault.sql" + ) + assert "mp__" in raw, ( + "03_business_vault.sql should document the mp__ marketplace-feed convention (domain.md §5.3)" + ) def test_hubs_and_links_have_bytea_primary_keys(): diff --git a/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__ala.sql b/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__ala.sql index eecf77be..6735ef34 100644 --- a/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__ala.sql +++ b/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__ala.sql @@ -3,7 +3,7 @@ Purpose: Canonical customer record for the ALA branch. Layer: Business Vault. Branch: ala (KZ jurisdiction; Bitrix loyalty not wired in ALA). Hub admission: splitByString('__', record_source)[2] = 'ala' (source-agnostic: - 1c__/pg_ops__/x5__ all integrated, not only 1C; audit_28_06_26 #12; + 1c__/pg_ops__/mp__ all integrated, not only 1C; audit_28_06_26 #12; mirrors the PostgreSQL port's split_part(record_source,'__',2)). Security: SQL SECURITY DEFINER (ADR 0006 Phase 2) — readers query this view under the definer's rights, so the column-limited grants in diff --git a/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__dxb.sql b/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__dxb.sql index 240d03ca..06879e86 100644 --- a/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__dxb.sql +++ b/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__dxb.sql @@ -8,7 +8,7 @@ Conflict policy: keeps the loyalty columns for schema parity with bv_customer_mdm__msk so downstream marts can UNION ALL the two branches without renaming. Hub admission: splitByString('__', record_source)[2] = 'dxb' (source-agnostic: - 1c__/pg_ops__/x5__ all integrated, not only 1C; audit_28_06_26 #12; + 1c__/pg_ops__/mp__ all integrated, not only 1C; audit_28_06_26 #12; mirrors the PostgreSQL port's split_part(record_source,'__',2)). Security: SQL SECURITY DEFINER (ADR 0006 Phase 2) — readers query this view under the definer's rights, so the column-limited grants in diff --git a/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__ekb.sql b/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__ekb.sql index e4347fc1..3e98164a 100644 --- a/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__ekb.sql +++ b/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__ekb.sql @@ -3,7 +3,7 @@ Purpose: Canonical customer record for the EKB branch. Layer: Business Vault. Branch: ekb (RU jurisdiction; same conflict policy as MSK). Hub admission: splitByString('__', record_source)[2] = 'ekb' (source-agnostic: - 1c__/pg_ops__/x5__ all integrated, not only 1C; audit_28_06_26 #12; + 1c__/pg_ops__/mp__ all integrated, not only 1C; audit_28_06_26 #12; mirrors the PostgreSQL port's split_part(record_source,'__',2)). Security: SQL SECURITY DEFINER (ADR 0006 Phase 2) — readers query this view under the definer's rights, so the column-limited grants in diff --git a/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__msk.sql b/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__msk.sql index 0660e1bd..71ffdab5 100644 --- a/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__msk.sql +++ b/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__msk.sql @@ -9,9 +9,9 @@ Conflict policy: - If a customer exists only in Bitrix, PII columns are NULL but the row is still returned so loyalty-only customers stay visible. Hub admission: splitByString('__', record_source)[2] = 'msk', so a customer - promoted under ANY source convention (1c__msk, pg_ops__msk, x5__msk, + promoted under ANY source convention (1c__msk, pg_ops__msk, mp__msk, ...) is integrated, not only 1C. The old record_source = '1c__msk' - filter silently dropped OLTP/X5-promoted customers (audit_28_06_26 #12); + filter silently dropped OLTP/marketplace-promoted customers (audit_28_06_26 #12); this mirrors the PostgreSQL port's split_part(record_source,'__',2). Security: SQL SECURITY DEFINER (ADR 0006 Phase 2) — readers query this view under the definer's rights, so the column-limited grants in diff --git a/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__spb.sql b/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__spb.sql index 2b193f38..41164132 100644 --- a/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__spb.sql +++ b/warehouse/agentflow/dv2/business_vault/bv_customer_mdm__spb.sql @@ -3,7 +3,7 @@ Purpose: Canonical customer record for the SPB branch. Layer: Business Vault. Branch: spb (RU jurisdiction; same conflict policy as MSK). Hub admission: splitByString('__', record_source)[2] = 'spb' (source-agnostic: - 1c__/pg_ops__/x5__ all integrated, not only 1C; audit_28_06_26 #12; + 1c__/pg_ops__/mp__ all integrated, not only 1C; audit_28_06_26 #12; mirrors the PostgreSQL port's split_part(record_source,'__',2)). Security: SQL SECURITY DEFINER (ADR 0006 Phase 2) — readers query this view under the definer's rights, so the column-limited grants in diff --git a/warehouse/agentflow/dv2/postgres/03_business_vault.sql b/warehouse/agentflow/dv2/postgres/03_business_vault.sql index c0831b78..2233d077 100644 --- a/warehouse/agentflow/dv2/postgres/03_business_vault.sql +++ b/warehouse/agentflow/dv2/postgres/03_business_vault.sql @@ -144,10 +144,10 @@ Dialect: postgresql. Branch: one view per jurisdiction by design — PII stays in branch. Hub admission: split_part(record_source,'__',2) = '', so a customer promoted under ANY source convention (1c__, pg_ops__, - x5__, ...) is integrated. This mirrors bv_order_canonical's + mp__, ...) is integrated. This mirrors bv_order_canonical's order_branch derivation above. The ClickHouse views hard-code record_source = '1c__', which silently dropped every OLTP- and - X5-promoted customer (record_source pg_ops__/x5__) from the MDM result + marketplace-promoted customer (record_source pg_ops__/mp__) from the MDM result (audit_28_06_26 #12). The hash keys were never incompatible — customer_hk = md5(business_key) is identical across loaders; only the hub record_source filter excluded them. split_part is the source- diff --git a/warehouse/agentflow/dv2/postgres/governance/verify_live.sh b/warehouse/agentflow/dv2/postgres/governance/verify_live.sh index 0c9f3340..f1af60ad 100755 --- a/warehouse/agentflow/dv2/postgres/governance/verify_live.sh +++ b/warehouse/agentflow/dv2/postgres/governance/verify_live.sh @@ -49,7 +49,7 @@ INSERT INTO rv.hub_customer (customer_hk, customer_bk, record_source) VALUES (decode(md5('CUST-MSK-5'),'hex'),'CUST-MSK-5','1c__msk'), (decode(md5('CUST-MSK-6'),'hex'),'CUST-MSK-6','1c__msk'), (decode(md5('CUST-MSK-7'),'hex'),'CUST-MSK-7','pg_ops__msk'), - (decode(md5('CUST-MSK-8'),'hex'),'CUST-MSK-8','x5__msk'), + (decode(md5('CUST-MSK-8'),'hex'),'CUST-MSK-8','mp__msk'), (decode(md5('CUST-DXB-1'),'hex'),'CUST-DXB-1','1c__dxb'), (decode(md5('CUST-DXB-2'),'hex'),'CUST-DXB-2','1c__dxb') ON CONFLICT (customer_hk) DO NOTHING;