Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions python/yara_orm/dialects.py
Original file line number Diff line number Diff line change
Expand Up @@ -1907,6 +1907,9 @@ class SqliteDialect(BaseDialect):
"""Dialect rendering SQL for SQLite."""

name = "sqlite"
# SQLite's default SQLITE_MAX_VARIABLE_NUMBER (3.32+); the base class's
# 65535 would let bulk statements exceed it.
max_bind_params = 32766
# SQLite has no ILIKE; its LIKE is case-insensitive for ASCII text.
ilike = "LIKE"
# SQLite's human-readable plan comes from EXPLAIN QUERY PLAN.
Expand Down
9 changes: 7 additions & 2 deletions python/yara_orm/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -2667,9 +2667,14 @@ async def bulk_update(
)
for name in field_names
]
# Keep batches under the dialect's bind-parameter ceiling (65535 on
# PostgreSQL, 2000 on SQL Server): each row costs one (pk, value) pair
# per field in the CASE arms plus one pk slot in the WHERE IN list.
params_per_row = 2 * len(targets) + 1
size = min(batch_size, max(1, dialect.max_bind_params // params_per_row))
total = 0
for start in range(0, len(objects), batch_size):
batch = objects[start : start + batch_size]
for start in range(0, len(objects), size):
batch = objects[start : start + size]
params: list[Any] = []
idx = 1
set_parts = []
Expand Down
80 changes: 80 additions & 0 deletions tests/test_cc_bulk_upsert.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,86 @@ async def test_bulk_update_multiple_batches_only_named_field(db):
assert reload["s4"] == (104, "orig")


class _ExecuteSpy:
"""Executor proxy recording every (sql, params) passed to ``execute``."""

def __init__(self, inner, log):
self._inner = inner
self._log = log

def __getattr__(self, name):
return getattr(self._inner, name)

async def execute(self, sql, params=None):
self._log.append((sql, list(params or [])))
return await self._inner.execute(sql, params)


def _spy_on_execute(monkeypatch):
"""Route ``models.get_executor`` through a spy; returns the statement log."""
import yara_orm.models as models_mod

log = []
real_get_executor = models_mod.get_executor

def fake_get_executor(model, *args, **kwargs):
return _ExecuteSpy(real_get_executor(model, *args, **kwargs), log)

monkeypatch.setattr(models_mod, "get_executor", fake_get_executor)
return log


@pytest.mark.asyncio
async def test_bulk_update_emits_single_where_in_clause(db, monkeypatch):
"""
GIVEN a bulk_update writing several fields in one batch
WHEN the UPDATE statement is generated
THEN it carries exactly one WHERE and one IN clause (never one per field)
and one CASE per written field
"""
# ``name`` is explicit: Oracle treats '' (the field default) as NULL,
# which would violate the column's NOT NULL constraint on insert.
await CcuItem.bulk_create([CcuItem(sku=f"w{i}", qty=i, name="orig") for i in range(3)])
objs = await CcuItem.all().order_by("sku")
for o in objs:
o.qty += 10
o.name = "upd"
log = _spy_on_execute(monkeypatch)
await CcuItem.bulk_update(objs, ["qty", "name"])
assert len(log) == 1
sql = log[0][0]
assert sql.count("WHERE") == 1
assert sql.count(" IN (") == 1
assert sql.count("CASE") == 2


@pytest.mark.asyncio
async def test_bulk_update_clamps_batches_under_bind_param_cap(db, monkeypatch):
"""
GIVEN a dialect bind-parameter ceiling smaller than batch_size would need
WHEN bulk_update runs with the default (larger) batch_size
THEN it splits into statements that each stay under the ceiling and all
rows still update
"""
from yara_orm.connection import get_dialect

# Explicit ``name`` for Oracle ('' default would insert NULL, see above).
await CcuItem.bulk_create([CcuItem(sku=f"c{i}", qty=i, name="orig") for i in range(5)])
objs = await CcuItem.all().order_by("sku")
for o in objs:
o.qty += 100
o.name = "cap"
# 2 fields -> 2*2+1 = 5 params per row; cap 10 -> 2 rows per statement.
monkeypatch.setattr(get_dialect(CcuItem), "max_bind_params", 10)
log = _spy_on_execute(monkeypatch)
n = await CcuItem.bulk_update(objs, ["qty", "name"])
assert n == 5
assert len(log) == 3 # 2 + 2 + 1 rows
assert all(len(params) <= 10 for _, params in log)
reload = {i.sku: (i.qty, i.name) for i in await CcuItem.all()}
assert reload == {f"c{i}": (i + 100, "cap") for i in range(5)}


# -- bulk_get_or_create extra corners ----------------------------------------


Expand Down