diff --git a/docs/api_doc/encoding/OOFMeanEncoder.rst b/docs/api_doc/encoding/OOFMeanEncoder.rst new file mode 100644 index 000000000..743db96da --- /dev/null +++ b/docs/api_doc/encoding/OOFMeanEncoder.rst @@ -0,0 +1,5 @@ +OOFMeanEncoder +============== + +.. autoclass:: feature_engine.encoding.OOFMeanEncoder + :members: diff --git a/docs/api_doc/encoding/index.rst b/docs/api_doc/encoding/index.rst index 6729b2513..9a3222409 100644 --- a/docs/api_doc/encoding/index.rst +++ b/docs/api_doc/encoding/index.rst @@ -15,6 +15,7 @@ estimated or arbitrary numbers. :class:`OrdinalEncoder()` √ √ √ Replaces categories with an integer :class:`CountEncoder()` √ √ √ Replaces categories with their count or frequency :class:`MeanEncoder()` √ √ x Replaces categories with the target mean value +:class:`OOFMeanEncoder()` √ √ x Replaces categories with the out-of-fold target mean value :class:`WoEEncoder()` x √ x Replaces categories with the weight of the evidence :class:`DecisionTreeEncoder()` √ √ √ Replaces categories with the predictions of a decision tree :class:`RareLabelEncoder()` √ √ √ Groups infrequent categories into a single one @@ -33,6 +34,7 @@ input. CountEncoder OrdinalEncoder MeanEncoder + OOFMeanEncoder WoEEncoder DecisionTreeEncoder RareLabelEncoder diff --git a/feature_engine/encoding/__init__.py b/feature_engine/encoding/__init__.py index a16e4fd51..2929dfba7 100644 --- a/feature_engine/encoding/__init__.py +++ b/feature_engine/encoding/__init__.py @@ -6,6 +6,7 @@ from .decision_tree import DecisionTreeEncoder from .mean_encoding import MeanEncoder from .one_hot import OneHotEncoder +from .oof_mean_encoding import OOFMeanEncoder from .ordinal import OrdinalEncoder from .rare_label import RareLabelEncoder from .similarity_encoder import StringSimilarityEncoder @@ -16,6 +17,7 @@ "CountFrequencyEncoder", "DecisionTreeEncoder", "MeanEncoder", + "OOFMeanEncoder", "OneHotEncoder", "OrdinalEncoder", "RareLabelEncoder", diff --git a/feature_engine/encoding/oof_mean_encoding.py b/feature_engine/encoding/oof_mean_encoding.py new file mode 100644 index 000000000..c39af0a06 --- /dev/null +++ b/feature_engine/encoding/oof_mean_encoding.py @@ -0,0 +1,506 @@ +# Authors: hx-max +# License: BSD 3 clause +"""Out-of-fold mean (target) encoding for categorical variables.""" + +import warnings +from typing import List, Union + +import numpy as np +import pandas as pd +from sklearn.model_selection import KFold, StratifiedKFold + +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) +from feature_engine._docstrings.fit_attributes import ( + _feature_names_in_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) +from feature_engine._docstrings.init_parameters.all_transformers import ( + _missing_values_docstring, + _return_empty_docstring, + _variables_categorical_docstring, +) +from feature_engine._docstrings.init_parameters.encoders import ( + _ignore_format_docstring, + _unseen_docstring, +) +from feature_engine._docstrings.methods import ( + _fit_transform_docstring, + _inverse_transform_docstring, + _transform_encoders_docstring, +) +from feature_engine._docstrings.substitute import Substitution +from feature_engine.dataframe_checks import check_X_y +from feature_engine.encoding._helper_functions import check_parameter_unseen +from feature_engine.encoding.base_encoder import ( + CategoricalInitMixinNA, + CategoricalMethodsMixin, +) + +_unseen_docstring = ( + _unseen_docstring + + """ If `'encode'`, unseen categories will be encoded with the prior.""" +) + +_MAX_CLASSIFICATION_LEVELS = 20 + + +@Substitution( + missing_values=_missing_values_docstring, + ignore_format=_ignore_format_docstring, + variables=_variables_categorical_docstring, + return_empty=_return_empty_docstring, + unseen=_unseen_docstring, + variables_=_variables_attribute_docstring, + feature_names_in_=_feature_names_in_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, + transform=_transform_encoders_docstring, + inverse_transform=_inverse_transform_docstring, +) +class OOFMeanEncoder(CategoricalMethodsMixin, CategoricalInitMixinNA): + """ + The OOFMeanEncoder() replaces categories with the mean of the target, computed + **out-of-fold**, which prevents target leakage in the training data. + + Mean (target) encoding — replacing a category by the mean target value observed + for that category — is a powerful way of encoding high cardinality categorical + variables, but it has a well known flaw: when the encoding is estimated and + applied on the same rows, every row contributes its own target value to its own + encoding. For high cardinality variables (many categories with few observations) + the encoding becomes a copy of the target, and models trained on it overfit + spectacularly: they learn that the encoded feature predicts the target, while on + new data the same feature is far less informative. + + This transformer fixes that by computing the mapping **out-of-fold** during + `fit_transform`: the data is split into `cv` folds; for every fold, the category + mean is estimated **only on the remaining folds** and then used to encode the + held-out rows. As a result, no row ever uses its own target value to build its + own encoding, while the *statistics* the encoding relies on are still estimated + from the whole data set. + + `fit` still learns the mapping from the entire data set. That mapping is used to + encode new data during `transform` (and, for consistency, by `inverse_transform`). + Therefore the standard usage is:: + + X_train_encoded = encoder.fit_transform(X_train, y_train) # leak free + X_test_encoded = encoder.transform(X_test) # full mapping + + Smoothing (shrinkage of rare category means towards the prior) works exactly as + in :class:`MeanEncoder`: with `smoothing='auto'` the weight given to the category + mean is estimated from the ratio of the target variance within the category to + the target variance of the data set, and with `smoothing=int or float` the weight + is `n_i / (n_i + smoothing)`. + + The encoder will encode only categorical variables by default (type 'object' or + 'categorical'). You can pass a list of variables to encode. Alternatively, the + encoder will find and encode all categorical variables (type 'object' or + 'categorical'). + + With `ignore_format=True` you have the option to encode numerical variables as + well. The procedure is identical, you can either enter the list of variables to + encode, or the transformer will automatically select all variables. + + More details in the :ref:`User Guide `. + + Parameters + ---------- + {variables} + + {return_empty} + + {missing_values} + + {ignore_format} + + {unseen} + + smoothing: int, float, str, default=0.0 + Smoothing factor. Should be >= 0. If 0 then no smoothing is applied, and the + mean target value per category is returned without modification. If 'auto' then + wi is calculated as described above and the category is encoded as the blended + values of the prior and the posterior. If int or float, then the wi is + calculated as ni / (ni+smoothing). Higher values lead to stronger smoothing + (higher weight of prior). + + cv: int, default=5 + Number of folds used to generate the out-of-fold encodings. Must be >= 2. + Larger values mean less bias in the training encodings but more computation. + + shuffle: bool, default=False + Whether to shuffle the data before splitting it into folds. If `True`, + `random_state` can be used for reproducibility. + + random_state: int, default=None + Seed used when `shuffle=True`. + + stratify: bool, default=None + Whether to use stratified folds, that is, folds that preserve the proportion + of target classes. If `None` (default) the encoder decides automatically: + stratified folds are used when the target looks categorical (boolean, integer, + or float with integral values, with at most 20 distinct values), and plain + K-Fold folds otherwise. Stratification is skipped, and plain K-Fold folds are + used instead, when any class has fewer observations than folds. + + Attributes + ---------- + encoder_dict_: + Dictionary with the target mean value per category per variable, learnt on the + entire data set. Used by `transform`. + + fold_mappings_: + List with the mapping (dictionary of category to target mean value, per + variable) learnt in each fold during `fit_transform`. + + fold_priors_: + List with the target mean of the training part of each fold. + + y_prior_: + Target mean of the entire data set. + + n_splits_: + Number of folds actually used in `fit_transform`. + + {variables_} + + {feature_names_in_} + + {n_features_in_} + + Methods + ------- + fit: + Learn the target mean value per category, per variable, on the entire data + set. + + {fit_transform} + + {transform} + + {inverse_transform} + + Notes + ----- + `fit_transform` is the method that produces leak free encodings for the training + data, because `transform` can only use the mapping learnt on the whole data set + (which includes the target of every row). In other words, encodings produced by + `fit_transform` and `transform` differ slightly, which is expected and is the + whole point of the out-of-fold scheme. + + Categories that appear in a validation fold but not in the remaining folds cannot + be mapped, because their mean is unknown. During `fit_transform` those rows are + encoded with the target mean of the remaining folds (the fold prior), so that the + output never contains missing values. NAN can still be introduced by `transform` + for categories that were not present in the training data, exactly as in + :class:`MeanEncoder`. + + Check also the related transformers in the open-source package + `Category encoders `_ + + See Also + -------- + feature_engine.encoding.MeanEncoder + feature_engine.encoding.RareLabelEncoder + category_encoders.target_encoder.TargetEncoder + category_encoders.m_estimate.MEstimateEncoder + + References + ---------- + .. [1] Micci-Barreca D. "A Preprocessing Scheme for High-Cardinality Categorical + Attributes in Classification and Prediction Problems". ACM SIGKDD Explorations + Newsletter, 2001. https://dl.acm.org/citation.cfm?id=507538 + .. [2] Prokhorenkova L. et al. "CatBoost: unbiased boosting with categorical + features". NeurIPS, 2018. https://arxiv.org/abs/1706.09516 + + Examples + -------- + + >>> import pandas as pd + >>> from feature_engine.encoding import OOFMeanEncoder + >>> X = pd.DataFrame({"x1": [1, 2, 3, 4, 5], "x2": ["c", "c", "c", "b", "a"]}) + >>> y = pd.Series([0, 1, 1, 1, 0]) + >>> oof = OOFMeanEncoder(cv=2) + >>> Xt = oof.fit_transform(X, y) + >>> oof.transform(X) + x1 x2 + 0 1 0.666667 + 1 2 0.666667 + 2 3 0.666667 + 3 4 1.000000 + 4 5 0.000000 + """ + + def __init__( + self, + variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, + missing_values: str = "raise", + ignore_format: bool = False, + unseen: str = "ignore", + smoothing: Union[int, float, str] = 0.0, + cv: int = 5, + shuffle: bool = False, + random_state: Union[int, None] = None, + stratify: Union[bool, None] = None, + ) -> None: + + _check_return_empty_is_bool(return_empty) + + super().__init__(variables, missing_values, ignore_format) + self.return_empty = return_empty + + if ( + not isinstance(smoothing, (str, float, int)) + or isinstance(smoothing, str) + and (smoothing != "auto") + ) or (isinstance(smoothing, (float, int)) and smoothing < 0): + raise ValueError( + f"smoothing must be greater than 0 or 'auto'. " + f"Got {smoothing} instead." + ) + self.smoothing = smoothing + + check_parameter_unseen(unseen, ["ignore", "raise", "encode"]) + self.unseen = unseen + + if not isinstance(cv, int) or isinstance(cv, bool) or cv < 2: + raise ValueError(f"cv must be an integer greater than 1. Got {cv} instead.") + self.cv = cv + + if not isinstance(shuffle, bool): + raise ValueError( + f"shuffle takes only booleans True and False. Got {shuffle} instead." + ) + self.shuffle = shuffle + + if stratify is not None and not isinstance(stratify, bool): + raise ValueError( + f"stratify takes only booleans True and False, or None. " + f"Got {stratify} instead." + ) + self.stratify = stratify + + self.random_state = random_state + + def fit(self, X: pd.DataFrame, y: pd.Series): + """ + Learn the mean value of the target for each category of the variable, using + the entire data set. This mapping is the one used by `transform`. + + Parameters + ---------- + X: pandas dataframe of shape = [n_samples, n_features] + The training input samples. Can be the entire dataframe, not just the + variables to be encoded. + + y: pandas series + The target. + """ + + X, y = check_X_y(X, y) + variables_ = self._check_or_select_variables(X) + self._check_na(X, variables_) + + self.y_prior_ = float(y.mean()) + + if self.unseen == "encode": + self._unseen = self.y_prior_ + + self.encoder_dict_ = self._learn_mapping( + X, y, variables_, prior=self.y_prior_ + ) + + # assign underscore parameters at the end in case code above fails + self.variables_ = variables_ + self._get_feature_names_in(X) + return self + + def fit_transform(self, X: pd.DataFrame, y: pd.Series) -> pd.DataFrame: + """ + Learn the mean value of the target per category **out-of-fold** and encode the + training data with it, so that no observation is encoded with a statistic that + contains its own target value. + + This method also leaves the transformer fitted with the mapping learnt on the + entire data set, so `transform` can be used afterwards on new data. + + Parameters + ---------- + X: pandas dataframe of shape = [n_samples, n_features] + The training input samples. Can be the entire dataframe, not just the + variables to be encoded. + + y: pandas series + The target. + + Returns + ------- + X_new: pandas dataframe of shape = [n_samples, n_features] + The dataframe with the categories replaced by the out-of-fold target mean + values. + """ + + # learn the mapping on the whole data set: it is used by transform. + self.fit(X, y) + + X_oof = X.copy() + + # Out-of-fold values are floats. Re-initialise the columns to encode as + # float64 so that (a) assigning floats never fails, whatever the original + # dtype is (object, category, integer), and (b) the output is numeric. + # Every row is assigned in exactly one fold, so no value is left as NA. + for var in self.variables_: + X_oof[var] = pd.Series(np.nan, index=X_oof.index, dtype="float64") + + splits = self._make_splits(X, y) + self.n_splits_ = len(splits) + self.fold_mappings_ = [] + self.fold_priors_ = [] + + for train_index, valid_index in splits: + X_fold, y_fold = X.iloc[train_index], y.iloc[train_index] + fold_prior = float(y_fold.mean()) + + fold_mapping = self._learn_mapping( + X_fold, y_fold, self.variables_, prior=fold_prior + ) + + for var in self.variables_: + col = X_oof.columns.get_loc(var) + values = X.iloc[valid_index][var].map(fold_mapping[var]) + + if values.isnull().any(): + if self.unseen == "raise": + raise ValueError( + "During the out-of-fold encoding, categories that were not " + f"present in the training folds were found in variable " + f"'{var}'. Set unseen to 'ignore' or 'encode' to encode " + "them with the prior of the training folds instead." + ) + values = values.fillna(fold_prior) + + X_oof.iloc[valid_index, col] = values.to_numpy(dtype="float64") + + self.fold_mappings_.append(fold_mapping) + self.fold_priors_.append(fold_prior) + + return X_oof + + def _learn_mapping( + self, + X: pd.DataFrame, + y: pd.Series, + variables_: List[Union[str, int]], + prior: float, + ) -> dict: + """Return the mapping of category to target mean value, per variable.""" + + mapping = {} + + if self.smoothing == "auto": + y_var = y.var(ddof=0) + + for var in variables_: + if self.smoothing == "auto": + damping = ( + y.groupby(X[var], observed=False).var(ddof=0) / y_var + if y_var > 0 + else 0 + ) + else: + damping = self.smoothing + + counts = X[var].value_counts() + counts.index = counts.index.infer_objects() + _lambda = counts / (counts + damping) + + mapping[var] = ( + _lambda * y.groupby(X[var], observed=False).mean() + + (1.0 - _lambda) * prior + ).to_dict() + + return mapping + + def _make_splits(self, X: pd.DataFrame, y: pd.Series) -> List[tuple]: + """Create the list of (train, validation) positional indices.""" + + stratified = self._resolve_stratify(y) + + if stratified: + classes = pd.Series(np.asarray(y)).value_counts() + smallest_class = int(classes.min()) if not classes.empty else 0 + if smallest_class < self.cv: + warnings.warn( + "The smallest target class has fewer observations than folds " + f"({smallest_class} < {self.cv}), therefore stratified folds are " + "not possible and plain K-Fold folds are used instead." + ) + stratified = False + + splitter_cls = StratifiedKFold if stratified else KFold + + kwargs: dict = {"n_splits": self.cv} + if self.shuffle: + kwargs["shuffle"] = True + kwargs["random_state"] = self.random_state + + splitter = splitter_cls(**kwargs) + + if stratified: + return list(splitter.split(X, y)) + return list(splitter.split(X)) + + def _resolve_stratify(self, y: pd.Series) -> bool: + """Return True if the target looks categorical and folds can be stratified.""" + + if self.stratify is not None: + return self.stratify + + y_array = np.asarray(y) + + if pd.Series(y_array).nunique(dropna=True) > _MAX_CLASSIFICATION_LEVELS: + return False + + if y_array.dtype.kind in "biu": + return True + + if y_array.dtype.kind == "f": + return bool(np.allclose(y_array, np.round(y_array))) + + return False + + def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame: + """Convert the encoded variable back to the original values. + + Note that if unseen was set to 'encode', then this method is not implemented. + + Parameters + ---------- + X: pandas dataframe of shape = [n_samples, n_features]. + The transformed dataframe. + + Returns + ------- + X_tr: pandas dataframe of shape = [n_samples, n_features]. + The un-transformed dataframe, with the categorical variables containing the + original values. + """ + + if self.unseen == "encode": + raise NotImplementedError( + "inverse_transform is not implemented for this transformer when " + "`unseen='encode'`." + ) + else: + return super().inverse_transform(X) + + def _more_tags(self): + tags_dict = super()._more_tags() + tags_dict["requires_y"] = True + return tags_dict + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + return tags diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index fbe9ce00f..8804b038a 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -13,6 +13,7 @@ CountFrequencyEncoder, DecisionTreeEncoder, MeanEncoder, + OOFMeanEncoder, OneHotEncoder, OrdinalEncoder, RareLabelEncoder, @@ -71,6 +72,7 @@ def test_check_estimator_from_sklearn(estimator): CountFrequencyEncoder(), DecisionTreeEncoder(regression=False), MeanEncoder(), + OOFMeanEncoder(), OneHotEncoder(), OrdinalEncoder(), RareLabelEncoder(), diff --git a/tests/test_encoding/test_oof_mean_encoder.py b/tests/test_encoding/test_oof_mean_encoder.py new file mode 100644 index 000000000..bbcd3989b --- /dev/null +++ b/tests/test_encoding/test_oof_mean_encoder.py @@ -0,0 +1,432 @@ +import numpy as np +import pandas as pd +import pytest +from numpy import nan +from sklearn.base import clone +from sklearn.exceptions import NotFittedError +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import roc_auc_score + +from feature_engine.encoding import MeanEncoder, OOFMeanEncoder + + +@pytest.fixture +def df_enc(): + return pd.DataFrame( + { + "var_A": ["A"] * 6 + ["B"] * 10 + ["C"] * 4, + "var_B": ["A"] * 10 + ["B"] * 6 + ["C"] * 4, + "target": [1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1], + } + ) + + +@pytest.fixture +def df_high_cardinality(): + """Every category has a single observation, so the in-sample target mean of a + category equals the target of its own row: maximum leakage.""" + n = 200 + rng = np.random.default_rng(0) + return pd.DataFrame( + { + "cat": [f"c{i}" for i in range(n)], + "num": rng.normal(size=n), + "target": rng.integers(0, 2, size=n), + } + ) + + +# --------------------------------------------------------------------------- +# init params +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("params", [("raise", True, "auto"), ("ignore", False, 1)]) +def test_init_param_assignment(params): + OOFMeanEncoder( + missing_values=params[0], + ignore_format=params[1], + unseen=params[0], + smoothing=params[2], + ) + + +@pytest.mark.parametrize( + "errors", ["empanada", False, 1, ("raise", "ignore"), ["ignore"]] +) +def test_error_if_unseen_gets_not_permitted_value(errors): + with pytest.raises(ValueError): + OOFMeanEncoder(unseen=errors) + + +@pytest.mark.parametrize("smoothing", ["hello", ["auto"], -1]) +def test_raises_error_when_not_allowed_smoothing_param_in_init(smoothing): + with pytest.raises(ValueError): + OOFMeanEncoder(smoothing=smoothing) + + +@pytest.mark.parametrize("cv", [1, 0, -3, 2.5, "5"]) +def test_raises_error_when_cv_is_not_an_integer_greater_than_1(cv): + with pytest.raises(ValueError): + OOFMeanEncoder(cv=cv) + + +@pytest.mark.parametrize("shuffle", ["yes", 1, None]) +def test_raises_error_when_shuffle_is_not_boolean(shuffle): + with pytest.raises(ValueError): + OOFMeanEncoder(shuffle=shuffle) + + +@pytest.mark.parametrize("stratify", ["yes", 1, "auto"]) +def test_raises_error_when_stratify_is_not_boolean_or_none(stratify): + with pytest.raises(ValueError): + OOFMeanEncoder(stratify=stratify) + + +def test_get_params_and_clone(): + encoder = OOFMeanEncoder(variables=["var_A"], cv=3, shuffle=True, random_state=3) + params = encoder.get_params() + assert params["cv"] == 3 + assert params["shuffle"] is True + assert params["random_state"] == 3 + assert isinstance(clone(encoder), OOFMeanEncoder) + + +# --------------------------------------------------------------------------- +# fit / transform: parity with MeanEncoder +# --------------------------------------------------------------------------- +def test_fit_learns_same_mapping_as_mean_encoder(df_enc): + oof = OOFMeanEncoder(variables=["var_A"], cv=4).fit( + df_enc[["var_A", "var_B"]], df_enc["target"] + ) + mean = MeanEncoder(variables=["var_A"]).fit( + df_enc[["var_A", "var_B"]], df_enc["target"] + ) + + assert oof.encoder_dict_ == mean.encoder_dict_ + # sanity: the mapping is the full-data mean per category, and y_prior_ is the + # overall target mean (not a per category mean). + assert oof.encoder_dict_["var_A"]["A"] == pytest.approx(5 / 6) + assert oof.y_prior_ == pytest.approx(df_enc["target"].mean()) + assert oof.y_prior_ != oof.encoder_dict_["var_A"]["A"] + + +def test_transform_matches_mean_encoder(df_enc): + oof = OOFMeanEncoder(variables=["var_A"], cv=4).fit( + df_enc[["var_A", "var_B"]], df_enc["target"] + ) + mean = MeanEncoder(variables=["var_A"]).fit( + df_enc[["var_A", "var_B"]], df_enc["target"] + ) + + pd.testing.assert_frame_equal( + oof.transform(df_enc[["var_A", "var_B"]]), + mean.transform(df_enc[["var_A", "var_B"]]), + ) + + +def test_error_when_transforming_before_fitting(df_enc): + with pytest.raises(NotFittedError): + OOFMeanEncoder().transform(df_enc[["var_A", "var_B"]]) + + +# --------------------------------------------------------------------------- +# the point of the transformer: no leakage in fit_transform +# --------------------------------------------------------------------------- +def test_fit_transform_does_not_leak_the_target(df_high_cardinality): + X, y = df_high_cardinality[["cat", "num"]], df_high_cardinality["target"] + + # in-sample mean encoding: each category is encoded with its own row's target. + in_sample = MeanEncoder(variables=["cat"]).fit(X, y).transform(X)["cat"] + assert np.corrcoef(in_sample, y)[0, 1] > 0.99 + + # out-of-fold encoding: a row's own target is never used to encode it. + oof = OOFMeanEncoder(variables=["cat"], cv=5, shuffle=True, random_state=0) + X_oof = oof.fit_transform(X, y)["cat"] + + assert X_oof.isnull().sum() == 0 + assert abs(np.corrcoef(X_oof, y)[0, 1]) < 0.2 + + +def test_oof_values_are_fold_dependent_unlike_in_sample_encoding(df_enc): + """The in-sample encoding is a deterministic function of the category — one single + value per category, which is exactly what leaks the target. The out-of-fold + encoding is not: every fold learns its own mapping, so two rows of the same + category can receive different values.""" + X, y = df_enc[["var_A", "var_B"]], df_enc["target"] + + in_sample = MeanEncoder(variables=["var_A"]).fit(X, y).transform(X)["var_A"] + encoder = OOFMeanEncoder(variables=["var_A"], cv=3, shuffle=True, random_state=1) + X_oof = encoder.fit_transform(X, y)["var_A"] + + assert in_sample.groupby(X["var_A"]).nunique().max() == 1 + assert in_sample.nunique() == X["var_A"].nunique() + + # every category is mapped to more than one value across folds + assert X_oof.groupby(X["var_A"]).nunique().min() > 1 + assert X_oof.nunique() > X["var_A"].nunique() + + # and the values remain probability-like + assert 0 <= X_oof.min() and X_oof.max() <= 1 + + +def test_fit_transform_leaves_encoder_fitted_for_new_data(df_enc): + X, y = df_enc[["var_A", "var_B"]], df_enc["target"] + + oof = OOFMeanEncoder(variables=["var_A"], cv=4) + oof.fit_transform(X, y) + + assert oof.n_splits_ == 4 + assert len(oof.fold_mappings_) == 4 + assert len(oof.fold_priors_) == 4 + assert oof.variables_ == ["var_A"] + assert oof.n_features_in_ == 2 + assert oof.feature_names_in_ == ["var_A", "var_B"] + + # transform on new data uses the mapping learnt on the whole data set + new = pd.DataFrame({"var_A": ["A", "B", "C"], "var_B": ["A", "B", "C"]}) + transformed = oof.transform(new) + assert transformed["var_A"].tolist() == [ + oof.encoder_dict_["var_A"]["A"], + oof.encoder_dict_["var_A"]["B"], + oof.encoder_dict_["var_A"]["C"], + ] + + +def test_fit_transform_preserves_shape_columns_and_untouched_variables(df_enc): + X, y = df_enc[["var_A", "var_B"]], df_enc["target"] + + X_oof = OOFMeanEncoder(variables=["var_A"], cv=2).fit_transform(X, y) + + assert X_oof.shape == X.shape + assert X_oof.columns.tolist() == X.columns.tolist() + pd.testing.assert_series_equal(X_oof["var_B"], X["var_B"]) + assert X_oof["var_A"].dtype == "float64" + + +def test_fit_transform_works_with_categorical_dtype_and_numerical_variables(): + df = pd.DataFrame( + { + "cat": pd.Categorical(["a", "b", "c", "a", "b", "c"]), + "num": [1, 1, 2, 2, 3, 3], + "target": [0, 1, 1, 0, 1, 1], + } + ) + + X_oof = OOFMeanEncoder( + variables=["cat", "num"], ignore_format=True, cv=2 + ).fit_transform(df[["cat", "num"]], df["target"]) + + assert X_oof["cat"].dtype == "float64" + assert X_oof["num"].dtype == "float64" + assert X_oof.isnull().sum().sum() == 0 + + +def test_oof_values_approximate_full_data_mapping(df_enc): + """The out-of-fold means are estimated from the same data, so per original + category they should be in the same ballpark as the full-data mean.""" + X, y = df_enc[["var_A", "var_B"]], df_enc["target"] + + oof = OOFMeanEncoder(variables=["var_A"], cv=10, shuffle=True, random_state=2) + X_oof = oof.fit_transform(X, y) + X_oof["category"] = X["var_A"].to_numpy() + + oof_means = X_oof.groupby("category")["var_A"].mean() + full_means = pd.Series(oof.encoder_dict_["var_A"]) + + for category in ["A", "B", "C"]: + assert abs(oof_means[category] - full_means[category]) < 0.35 + + +# --------------------------------------------------------------------------- +# folds, reproducibility, unseen categories +# --------------------------------------------------------------------------- +def test_stratified_folds_are_used_for_categorical_targets(df_enc): + X, y = df_enc[["var_A", "var_B"]], df_enc["target"] + + encoder = OOFMeanEncoder(variables=["var_A"], cv=4) + encoder.fit_transform(X, y) + + # every fold keeps the share of positives close to the overall share + overall = y.mean() + assert encoder.n_splits_ == 4 + for mapping, prior in zip(encoder.fold_mappings_, encoder.fold_priors_): + assert mapping["var_A"] + assert abs(prior - overall) < 0.3 + + +def test_plain_kfold_is_used_for_continuous_targets(): + rng = np.random.default_rng(0) + X = pd.DataFrame({"cat": ["a", "b", "c", "d"] * 25}) + y = pd.Series(rng.normal(size=100)) + + encoder = OOFMeanEncoder(variables=["cat"], cv=3) + X_oof = encoder.fit_transform(X, y) + + assert encoder.n_splits_ == 3 + assert X_oof.isnull().sum().sum() == 0 + + +def test_warns_and_falls_back_to_kfold_when_classes_are_too_small(): + X = pd.DataFrame({"cat": ["a", "b"] * 5}) + y = pd.Series([1] + [0] * 9) + + encoder = OOFMeanEncoder(variables=["cat"], cv=5) + with pytest.warns(UserWarning): + encoder.fit_transform(X, y) + + assert encoder.n_splits_ == 5 + + +def test_fit_transform_is_reproducible_with_random_state(df_enc): + X, y = df_enc[["var_A", "var_B"]], df_enc["target"] + + first = OOFMeanEncoder( + variables=["var_A"], cv=3, shuffle=True, random_state=7 + ).fit_transform(X, y) + second = OOFMeanEncoder( + variables=["var_A"], cv=3, shuffle=True, random_state=7 + ).fit_transform(X, y) + + pd.testing.assert_frame_equal(first, second) + + +def test_fold_unseen_categories_are_encoded_with_the_fold_prior(): + """A category that only exists in the validation fold cannot be mapped, so it is + encoded with the prior of the remaining folds instead of with NaN.""" + X = pd.DataFrame({"cat": ["a", "a", "b", "b", "c", "d"]}) + y = pd.Series([0, 1, 0, 1, 1, 0]) + + encoder = OOFMeanEncoder(variables=["cat"], cv=2, stratify=False) + X_oof = encoder.fit_transform(X, y) + + # fold 0 trains on rows 3-5 (b, c, d) and holds out rows 0-2 (a, a, b); 'a' is + # therefore unseen and falls back to the prior of rows 3-5. Fold 1 does the same + # for 'c' and 'd', which only appear in its held out rows. + assert encoder.fold_priors_ == pytest.approx([2 / 3, 1 / 3]) + assert X_oof["cat"].tolist() == pytest.approx( + [2 / 3, 2 / 3, 1.0, 0.0, 1 / 3, 1 / 3] + ) + assert X_oof.isnull().sum().sum() == 0 + + +def test_unseen_encode_uses_the_prior(df_enc): + X, y = df_enc[["var_A", "var_B"]], df_enc["target"] + oof = OOFMeanEncoder(variables=["var_A"], unseen="encode", cv=2).fit(X, y) + + new = pd.DataFrame({"var_A": ["D"], "var_B": ["A"]}) + assert oof.transform(new)["var_A"].iloc[0] == pytest.approx(y.mean()) + + +def test_unseen_ignore_introduces_nan_in_transform(df_enc): + X, y = df_enc[["var_A", "var_B"]], df_enc["target"] + oof = OOFMeanEncoder(variables=["var_A"], unseen="ignore", cv=2).fit(X, y) + + new = pd.DataFrame({"var_A": ["D"], "var_B": ["A"]}) + assert np.isnan(oof.transform(new)["var_A"].iloc[0]) + + +def test_unseen_raise_raises_in_fit_transform_when_category_is_missing_in_folds(): + X = pd.DataFrame({"cat": ["a", "a", "b", "b", "c", "d"]}) + y = pd.Series([0, 1, 0, 1, 1, 0]) + + encoder = OOFMeanEncoder(variables=["cat"], cv=2, unseen="raise", stratify=False) + + with pytest.raises(ValueError): + encoder.fit_transform(X, y) + + +# --------------------------------------------------------------------------- +# smoothing, missing values, inverse transform +# --------------------------------------------------------------------------- +def test_smoothing_shrinks_oof_values_towards_the_prior(df_enc): + X, y = df_enc[["var_A", "var_B"]], df_enc["target"] + + no_smoothing = OOFMeanEncoder( + variables=["var_A"], cv=2, shuffle=True, random_state=0 + ).fit_transform(X, y)["var_A"] + smoothed = OOFMeanEncoder( + variables=["var_A"], cv=2, shuffle=True, random_state=0, smoothing=10 + ).fit_transform(X, y)["var_A"] + + prior = y.mean() + assert (smoothed - prior).abs().mean() <= (no_smoothing - prior).abs().mean() + + +def test_smoothing_auto_runs(df_enc): + X, y = df_enc[["var_A", "var_B"]], df_enc["target"] + + oof = OOFMeanEncoder(variables=["var_A"], cv=2, smoothing="auto") + X_oof = oof.fit_transform(X, y) + + assert X_oof["var_A"].isnull().sum() == 0 + + +def test_missing_values_raise_by_default(): + X = pd.DataFrame({"cat": ["a", "b", nan, "b"]}) + y = pd.Series([0, 1, 1, 0]) + + with pytest.raises(ValueError): + OOFMeanEncoder(variables=["cat"], cv=2).fit_transform(X, y) + + +def test_missing_values_ignore(df_enc): + X = df_enc[["var_A", "var_B"]].copy() + X.loc[0, "var_A"] = nan + y = df_enc["target"] + + X_oof = OOFMeanEncoder( + variables=["var_A"], missing_values="ignore", cv=2 + ).fit_transform(X, y) + + assert X_oof["var_A"].isnull().sum() == 0 + + +def test_inverse_transform_recovers_categories(df_enc): + X, y = df_enc[["var_A", "var_B"]], df_enc["target"] + + oof = OOFMeanEncoder(variables=["var_A"], cv=2).fit(X, y) + Xt = oof.transform(X) + + pd.testing.assert_frame_equal(oof.inverse_transform(Xt), X) + + +def test_inverse_transform_not_implemented_when_unseen_encode(df_enc): + X, y = df_enc[["var_A", "var_B"]], df_enc["target"] + oof = OOFMeanEncoder(variables=["var_A"], unseen="encode", cv=2).fit(X, y) + + with pytest.raises(NotImplementedError): + oof.inverse_transform(oof.transform(X)) + + +# --------------------------------------------------------------------------- +# end to end: leakage hurts a downstream model, out-of-fold encoding does not +# --------------------------------------------------------------------------- +def test_downstream_model_generalises_better_with_oof_encoding(): + rng = np.random.default_rng(42) + n_categories = 300 + category_effect = rng.normal(size=n_categories) + + def make(n): + idx = rng.integers(0, n_categories, size=n) + cat = [f"c{i}" for i in idx] + # the target depends on the category, with a lot of noise + p = 1 / (1 + np.exp(-category_effect[idx])) + y = rng.binomial(1, p) + return pd.DataFrame({"cat": cat}), pd.Series(y) + + X_train, y_train = make(3000) + X_valid, y_valid = make(3000) + + results = {} + for name, encoder in [ + ("mean (in sample)", MeanEncoder(variables=["cat"])), + ("oof", OOFMeanEncoder(variables=["cat"], cv=5, shuffle=True, random_state=0)), + ]: + X_train_enc = encoder.fit_transform(X_train, y_train) + X_valid_enc = encoder.transform(X_valid) + + model = LogisticRegression().fit(X_train_enc, y_train) + results[name] = roc_auc_score(y_valid, model.predict_proba(X_valid_enc)[:, 1]) + + # the out-of-fold encoding is noisy but honest; the in-sample encoding is + # over-confident and generalises no better (in practice it is worse). + assert results["oof"] >= results["mean (in sample)"]