Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
609 changes: 609 additions & 0 deletions PRPs/PRP-reliability-E5-model-exogenous-price-inertia.md

Large diffs are not rendered by default.

59 changes: 59 additions & 0 deletions app/features/forecasting/tests/test_regression_forecaster.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,65 @@ def test_handles_nan_features() -> None:
assert bool(np.all(np.isfinite(predictions)))


def test_constant_price_column_is_inert_to_future_price() -> None:
"""A model fit on a constant price column ignores future prices EXACTLY.

Discriminator test for issue #237 (hypothesis 2, the verdict): a
``RegressionForecaster`` trained on a matrix whose price column is the
constant ``1.0`` — exactly what the pre-fix seeder produced, since
``sales_daily.unit_price`` never moved off ``base_price`` and so
``price_factor = unit_price / median(unit_price) ≡ 1.0`` — predicts
byte-identically for ANY future price value. ``HistGradientBoostingRegressor``
never splits on a constant training column, so the scenario delta is
exactly 0.0, not merely small. The 0.0 the issue reproduces is
zero-learned-elasticity, not lost wiring (the wiring twin lives in
``app/features/scenarios/tests/test_feature_frame.py``).
"""
rng = np.random.default_rng(42)
n = 300
features = rng.normal(10.0, 2.0, size=(n, 5)).astype(np.float64)
features[:, 4] = 1.0 # the price column is CONSTANT, as on pre-fix seeded data
target = (40.0 + 0.5 * features[:, 0] + rng.normal(scale=0.5, size=n)).astype(np.float64)

model = RegressionForecaster(random_state=42).fit(target, features)

future = features[:14].copy()
future[:, 4] = 1.0
baseline_prediction = model.predict(14, future)
future[:, 4] = 0.60 # a deep -40% price cut
cut_prediction = model.predict(14, future)

np.testing.assert_array_equal(cut_prediction, baseline_prediction)


def test_elastic_price_column_responds_to_future_price() -> None:
"""The converse discriminator: trained price variance → a real response.

Fit on price-elastic synthetic data (price column uniform in [0.7, 1.1],
demand falling with price) and the same forecaster's prediction moves
when the future price moves — proving the inertia in the constant-column
twin is a property of the *training data*, not of the model or the
predict path (issue #237).
"""
rng = np.random.default_rng(42)
n = 300
features = rng.normal(10.0, 2.0, size=(n, 5)).astype(np.float64)
features[:, 4] = rng.uniform(0.7, 1.1, size=n)
target = (40.0 - 20.0 * features[:, 4] + rng.normal(scale=0.5, size=n)).astype(np.float64)

model = RegressionForecaster(random_state=42).fit(target, features)

future = features[:14].copy()
future[:, 4] = 1.0
baseline_prediction = model.predict(14, future)
future[:, 4] = 0.85 # a -15% cut inside the training range
cut_prediction = model.predict(14, future)

assert not np.array_equal(cut_prediction, baseline_prediction)
# Demand falls with price, so a cut lifts every horizon day's forecast.
assert bool(np.all(cut_prediction > baseline_prediction))


def test_get_and_set_params() -> None:
"""``get_params`` reflects construction; ``set_params`` mutates in place."""
model = RegressionForecaster(max_iter=150, learning_rate=0.03, max_depth=4)
Expand Down
64 changes: 64 additions & 0 deletions app/features/scenarios/tests/test_feature_frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,70 @@ def test_assemble_future_frame_shape_and_order() -> None:
assert all(len(row) == len(columns) for row in frame.matrix)


def test_scenario_vs_baseline_frame_differs_only_in_price_factor() -> None:
"""A price assumption reaches X_future, and ONLY the price_factor column.

Discriminator test for issue #237 (hypothesis 1 falsification): the
scenario frame and an assumptions-stripped baseline frame — built from
identical dates/columns/history_tail/holiday_dates/launch_date — differ
exactly in the ``price_factor`` cells inside the assumption window
(``1 + change_pct`` vs ``1.0``) and nowhere else. The future-frame wiring
therefore does NOT drop the price assumption; the observed 0.0 delta must
come from the trained model itself (see the inert-mechanism twin in
``app/features/forecasting/tests/test_regression_forecaster.py``).
"""
columns = canonical_feature_columns()
history_tail = [float(value) for value in range(HISTORY_TAIL_DAYS)]
launch = _ORIGIN - timedelta(days=100)
holiday_dates = {_HORIZON_DATES[0]}
# Window covering days 3..7 (indices 2..6) of the 14-day horizon.
assumptions = ScenarioAssumptions(
price=PriceAssumption(
change_pct=-0.15,
start_date=_HORIZON_DATES[2],
end_date=_HORIZON_DATES[6],
)
)

scenario_frame = assemble_future_frame(
dates=_HORIZON_DATES,
feature_columns=columns,
history_tail=history_tail,
assumptions=assumptions,
holiday_dates=holiday_dates,
launch_date=launch,
)
baseline_frame = assemble_future_frame(
dates=_HORIZON_DATES,
feature_columns=columns,
history_tail=history_tail,
assumptions=ScenarioAssumptions(),
holiday_dates=holiday_dates,
launch_date=launch,
)

price_index = columns.index("price_factor")
for row_index in range(_HORIZON):
scenario_row = scenario_frame.matrix[row_index]
baseline_row = baseline_frame.matrix[row_index]
for col_index in range(len(columns)):
scenario_cell = scenario_row[col_index]
baseline_cell = baseline_row[col_index]
if col_index == price_index:
if 2 <= row_index <= 6:
assert scenario_cell == 0.85
assert baseline_cell == 1.0
else:
assert scenario_cell == 1.0
assert baseline_cell == 1.0
continue
# Every non-price cell is identical (NaN-aware compare — lag
# cells whose source target lies in the horizon are NaN).
if math.isnan(scenario_cell) and math.isnan(baseline_cell):
continue
assert scenario_cell == baseline_cell


def test_assemble_future_frame_unknown_column_is_nan() -> None:
"""A requested column the builders do not produce becomes an all-NaN column."""
columns = [*canonical_feature_columns(), "mystery_feature"]
Expand Down
Loading