Skip to content

Fix encoded_missing_value being ignored and all-missing datetime columns crashing #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
devYRPauli wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Fix encoded_missing_value being ignored and all-missing datetime columns crashing #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6993ecb
Fix categorical encoder missing-value handling
devYRPauli Jul 20, 2026
dac3656
Handle all-missing datetime columns
devYRPauli Jul 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions tabfm/src/classifier_and_regressor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ def transform(self, X: Any) -> np.ndarray:

for i in range(n_features):
col = X[:, i]
missing_mask = pd.isna(col)
cats = self.categories_[i]
cat_to_idx = {c: idx for idx, c in enumerate(cats)}
mapped = pd.Series(col).map(cat_to_idx)
Expand All @@ -216,7 +217,9 @@ def transform(self, X: Any) -> np.ndarray:
if mapped.dtype.name == "category":
mapped = mapped.astype(object)

# Fill NaNs/Unknowns with unknown_value
mapped.loc[missing_mask] = self.encoded_missing_value

# Fill unknowns with unknown_value
X_out[:, i] = mapped.fillna(self.unknown_value).values.astype(self.dtype)

return X_out
Expand Down Expand Up @@ -334,7 +337,10 @@ def fit(self, X: Any, y: Any = None) -> "DatetimeTransformer":
series = pd.to_datetime(
X.iloc[:, pos], utc=True, errors="coerce", format="mixed"
)
self._fillna_map[pos] = series.mean()
fillna_value = series.mean()
if pd.isna(fillna_value):
fillna_value = pd.Timestamp(0, tz="UTC")
self._fillna_map[pos] = fillna_value
return self

def transform(self, X: Any) -> np.ndarray:
Expand Down
28 changes: 28 additions & 0 deletions tabfm/src/classifier_and_regressor_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
except ImportError:
HAS_JAX = False
from tabfm.src.classifier_and_regressor import _looks_like_datetime
from tabfm.src.classifier_and_regressor import CategoricalOrdinalEncoder
from tabfm.src.classifier_and_regressor import DatetimeTransformer
from tabfm.src.classifier_and_regressor import EnsembleGenerator
from tabfm.src.classifier_and_regressor import TabFMClassifier
from tabfm.src.classifier_and_regressor import TabFMRegressor
Expand Down Expand Up @@ -1165,6 +1167,32 @@ def test_mostly_non_date_column_not_detected(self):
self.assertFalse(_looks_like_datetime(pd.Series(vals, dtype="string")))


class CategoricalOrdinalEncoderTest(absltest.TestCase):

def test_missing_and_unknown_values_are_encoded_separately(self):
encoder = CategoricalOrdinalEncoder(
unknown_value=-1, encoded_missing_value=-99
)
encoder.fit(np.array([["known"], ["other"]], dtype=object))

encoded = encoder.transform(
np.array([[np.nan], ["unseen"], ["known"]], dtype=object)
)

np.testing.assert_array_equal(encoded[:, 0], [-99, -1, 0])


class DatetimeTransformerTest(absltest.TestCase):

def test_all_missing_column_uses_epoch_fill_value(self):
X = pd.DataFrame({"ts": pd.to_datetime([None, None, None])})

transformed = DatetimeTransformer().fit_transform(X)

self.assertEqual(transformed.shape, (3, 5))
self.assertTrue(np.isfinite(transformed).all())


class ColumnNameRobustnessTest(absltest.TestCase):

def test_duplicate_column_names_raise_a_clear_error(self):
Expand Down