From 6993ecba2e77a1c9dd3c779c3b4aa8158b33cd14 Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Mon, 20 Jul 2026 10:35:07 -0400 Subject: [PATCH 1/3] Fix categorical encoder missing-value handling --- tabfm/src/classifier_and_regressor.py | 5 ++++- tabfm/src/classifier_and_regressor_test.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tabfm/src/classifier_and_regressor.py b/tabfm/src/classifier_and_regressor.py index 4a0ebee..a116f5b 100644 --- a/tabfm/src/classifier_and_regressor.py +++ b/tabfm/src/classifier_and_regressor.py @@ -208,6 +208,7 @@ def transform(self, X: Any) -> np.ndarray: for i in range(n_features): col = X[:, i] + missing_mask = pd.isna(col) cats = self.categories_[i] cat_to_idx = {c: idx for idx, c in enumerate(cats)} mapped = pd.Series(col).map(cat_to_idx) @@ -216,7 +217,9 @@ def transform(self, X: Any) -> np.ndarray: if mapped.dtype.name == "category": mapped = mapped.astype(object) - # Fill NaNs/Unknowns with unknown_value + mapped.loc[missing_mask] = self.encoded_missing_value + + # Fill unknowns with unknown_value X_out[:, i] = mapped.fillna(self.unknown_value).values.astype(self.dtype) return X_out diff --git a/tabfm/src/classifier_and_regressor_test.py b/tabfm/src/classifier_and_regressor_test.py index 546fbee..64e2b22 100644 --- a/tabfm/src/classifier_and_regressor_test.py +++ b/tabfm/src/classifier_and_regressor_test.py @@ -27,6 +27,7 @@ except ImportError: HAS_JAX = False from tabfm.src.classifier_and_regressor import _looks_like_datetime +from tabfm.src.classifier_and_regressor import CategoricalOrdinalEncoder from tabfm.src.classifier_and_regressor import EnsembleGenerator from tabfm.src.classifier_and_regressor import TabFMClassifier from tabfm.src.classifier_and_regressor import TabFMRegressor @@ -1165,6 +1166,21 @@ def test_mostly_non_date_column_not_detected(self): self.assertFalse(_looks_like_datetime(pd.Series(vals, dtype="string"))) +class CategoricalOrdinalEncoderTest(absltest.TestCase): + + def test_missing_and_unknown_values_are_encoded_separately(self): + encoder = CategoricalOrdinalEncoder( + unknown_value=-1, encoded_missing_value=-99 + ) + encoder.fit(np.array([["known"], ["other"]], dtype=object)) + + encoded = encoder.transform( + np.array([[np.nan], ["unseen"], ["known"]], dtype=object) + ) + + np.testing.assert_array_equal(encoded[:, 0], [-99, -1, 0]) + + class ColumnNameRobustnessTest(absltest.TestCase): def test_duplicate_column_names_raise_a_clear_error(self): From dac365648f8de5eb9a4dc54cd6aecafca443b6c3 Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Mon, 20 Jul 2026 10:35:54 -0400 Subject: [PATCH 2/3] Handle all-missing datetime columns --- tabfm/src/classifier_and_regressor.py | 5 ++++- tabfm/src/classifier_and_regressor_test.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tabfm/src/classifier_and_regressor.py b/tabfm/src/classifier_and_regressor.py index a116f5b..4cc74d1 100644 --- a/tabfm/src/classifier_and_regressor.py +++ b/tabfm/src/classifier_and_regressor.py @@ -337,7 +337,10 @@ def fit(self, X: Any, y: Any = None) -> "DatetimeTransformer": series = pd.to_datetime( X.iloc[:, pos], utc=True, errors="coerce", format="mixed" ) - self._fillna_map[pos] = series.mean() + fillna_value = series.mean() + if pd.isna(fillna_value): + fillna_value = pd.Timestamp(0, tz="UTC") + self._fillna_map[pos] = fillna_value return self def transform(self, X: Any) -> np.ndarray: diff --git a/tabfm/src/classifier_and_regressor_test.py b/tabfm/src/classifier_and_regressor_test.py index 64e2b22..6937b7c 100644 --- a/tabfm/src/classifier_and_regressor_test.py +++ b/tabfm/src/classifier_and_regressor_test.py @@ -28,6 +28,7 @@ HAS_JAX = False from tabfm.src.classifier_and_regressor import _looks_like_datetime from tabfm.src.classifier_and_regressor import CategoricalOrdinalEncoder +from tabfm.src.classifier_and_regressor import DatetimeTransformer from tabfm.src.classifier_and_regressor import EnsembleGenerator from tabfm.src.classifier_and_regressor import TabFMClassifier from tabfm.src.classifier_and_regressor import TabFMRegressor @@ -1181,6 +1182,17 @@ def test_missing_and_unknown_values_are_encoded_separately(self): np.testing.assert_array_equal(encoded[:, 0], [-99, -1, 0]) +class DatetimeTransformerTest(absltest.TestCase): + + def test_all_missing_column_uses_epoch_fill_value(self): + X = pd.DataFrame({"ts": pd.to_datetime([None, None, None])}) + + transformed = DatetimeTransformer().fit_transform(X) + + self.assertEqual(transformed.shape, (3, 5)) + self.assertTrue(np.isfinite(transformed).all()) + + class ColumnNameRobustnessTest(absltest.TestCase): def test_duplicate_column_names_raise_a_clear_error(self): From f9606ba8b6d62133547c9601b45bb58a4857dff2 Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Wed, 2 Sep 2026 18:29:55 -0400 Subject: [PATCH 3/3] Decode the missing sentinel back to None in inverse_transform transform() now writes encoded_missing_value for NaN inputs, so inverse_transform() can receive that sentinel. Its mask only excluded unknown_value, so the sentinel reached cats[value]. With encoded_missing_value=-99 and three categories that raises IndexError; with 120 categories it returns cats[-99], an unrelated category, and raises nothing. Exclude the missing sentinel from the same mask. Missing values now decode to None, matching how unknown values already decode and what main returns today. The defaults and both in-repo construction sites use -1 for both sentinels, so their behavior is unchanged. --- tabfm/src/classifier_and_regressor.py | 4 +++- tabfm/src/classifier_and_regressor_test.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tabfm/src/classifier_and_regressor.py b/tabfm/src/classifier_and_regressor.py index 4cc74d1..97289a9 100644 --- a/tabfm/src/classifier_and_regressor.py +++ b/tabfm/src/classifier_and_regressor.py @@ -242,7 +242,9 @@ def inverse_transform(self, X: np.ndarray) -> np.ndarray: for i in range(n_features): col = X[:, i] cats = self.categories_[i] - valid_mask = col != self.unknown_value + valid_mask = (col != self.unknown_value) & ( + col != self.encoded_missing_value + ) if np.any(valid_mask): indices = col[valid_mask].astype(int) X_out[valid_mask, i] = cats[indices] diff --git a/tabfm/src/classifier_and_regressor_test.py b/tabfm/src/classifier_and_regressor_test.py index 6937b7c..baa52b9 100644 --- a/tabfm/src/classifier_and_regressor_test.py +++ b/tabfm/src/classifier_and_regressor_test.py @@ -1181,6 +1181,18 @@ def test_missing_and_unknown_values_are_encoded_separately(self): np.testing.assert_array_equal(encoded[:, 0], [-99, -1, 0]) + def test_inverse_transform_does_not_decode_the_missing_sentinel(self): + encoder = CategoricalOrdinalEncoder( + unknown_value=-1, encoded_missing_value=-99 + ) + encoder.fit(np.array([["known"], ["other"]], dtype=object)) + + decoded = encoder.inverse_transform(np.array([[-99], [-1], [0]])) + + self.assertIsNone(decoded[0, 0]) + self.assertIsNone(decoded[1, 0]) + self.assertEqual(decoded[2, 0], "known") + class DatetimeTransformerTest(absltest.TestCase):