From a85da0cead85f2ee7cbcdf6e6e32f963b32c0046 Mon Sep 17 00:00:00 2001 From: Lucas Morin Date: Sun, 18 Oct 2026 16:20:02 +0200 Subject: [PATCH 1/2] Add statistical and permutation ranking to probe selection --- .../selection/ProbeFeatureSelection.rst | 40 +++++ .../selection/probe_feature_selection.py | 122 ++++++++++++++- .../test_probe_ranking_methods.py | 146 ++++++++++++++++++ 3 files changed, 304 insertions(+), 4 deletions(-) create mode 100644 tests/test_selection/test_probe_ranking_methods.py diff --git a/docs/user_guide/selection/ProbeFeatureSelection.rst b/docs/user_guide/selection/ProbeFeatureSelection.rst index 557a462ba..dc9baf7a3 100644 --- a/docs/user_guide/selection/ProbeFeatureSelection.rst +++ b/docs/user_guide/selection/ProbeFeatureSelection.rst @@ -649,3 +649,43 @@ For more details about this and other feature selection methods check out these Both our book and course are suitable for beginners and more advanced data scientists alike. By purchasing them you are supporting `Sole `_, the main developer of feature-engine. + + +Statistical and permutation ranking +----------------------------------- + +Set ``importance_method="correlation"`` to compare absolute Pearson correlations +of real and random features with a numeric target. No estimator is required:: + + selector = ProbeFeatureSelection( + importance_method="correlation", n_probes=5, cv=5, random_state=42 + ) + X_selected = selector.fit_transform(X_train, y_train) + +``f_classif`` and ``f_regression`` provide ANOVA F-score ranking for classification +and regression. ``mutual_info_classif`` and ``mutual_info_regression`` provide +mutual-information ranking. Select the variant explicitly for your target type. +These methods compute scores on the training portion of each cross-validation +split and expose their mean and standard deviation in the usual attributes. +Mutual-information methods treat the original numeric features as continuous; +use these methods for continuous predictors. Binary, discrete-uniform and +Poisson probe columns are treated as discrete. + +For an estimator without embedded feature importances, use held-out permutation +importance:: + + from sklearn.neighbors import KNeighborsRegressor + + selector = ProbeFeatureSelection( + estimator=KNeighborsRegressor(), importance_method="permutation", + scoring="neg_mean_squared_error", cv=5, random_state=42 + ) + +The estimator is cloned for each fold, fitted only on training rows, and scored +on validation rows after five permutations per column. The importance is the +mean decrease in score. The reported standard deviation is across fold means, +not across individual permutations. ``collective`` applies only to the default +``importance_method="model"``. Existing calls preserve their behaviour. + +Fit the selector inside the training part of your evaluation pipeline. For time +series, pass an appropriate chronological splitter through ``cv``. diff --git a/feature_engine/selection/probe_feature_selection.py b/feature_engine/selection/probe_feature_selection.py index f19c5ca76..2a5213216 100644 --- a/feature_engine/selection/probe_feature_selection.py +++ b/feature_engine/selection/probe_feature_selection.py @@ -2,6 +2,16 @@ import numpy as np import pandas as pd +from sklearn.base import clone, is_classifier +from sklearn.feature_selection import ( + f_classif, + f_regression, + mutual_info_classif, + mutual_info_regression, + r_regression, +) +from sklearn.inspection import permutation_importance +from sklearn.model_selection import check_cv from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, @@ -14,10 +24,10 @@ from feature_engine._docstrings.methods import _fit_transform_docstring from feature_engine._docstrings.selection._docstring import ( _cv_docstring, - _groups_docstring, _features_to_drop_docstring, _fit_docstring, _get_support_docstring, + _groups_docstring, _scoring_docstring, _transform_docstring, _variables_attribute_docstring, @@ -80,10 +90,11 @@ class ProbeFeatureSelection(BaseSelector): Parameters ---------- - estimator: object + estimator: object, default=None A scikit-learn estimator for regression or classification. If `collective=True`, the estimator must have either a `feature_importances_` or a `coef_` attribute after fitting. + Required for ``importance_method='model'`` or ``'permutation'``. {variables} @@ -120,6 +131,18 @@ class ProbeFeatureSelection(BaseSelector): {groups} + importance_method: str, default='model' + How to rank real and probe features. 'model' preserves the behaviour of + ``collective``. 'correlation' uses absolute Pearson correlation with a + numeric target. 'f_classif' and 'f_regression' use ANOVA F scores for + classification and regression respectively. 'mutual_info_classif' and + 'mutual_info_regression' estimate mutual information, treating input + features as continuous and discrete probe distributions as discrete. + These statistical scores are calculated on each training fold. + 'permutation' fits the estimator on each training fold and measures the + decrease in ``scoring`` on its held-out fold, averaged over five shuffles. + ``collective`` is used only with 'model'. + Attributes ---------- probe_features_: @@ -181,7 +204,7 @@ class ProbeFeatureSelection(BaseSelector): def __init__( self, - estimator, + estimator=None, variables: Variables = None, collective: bool = True, scoring: str = "roc_auc", @@ -193,6 +216,7 @@ def __init__( groups=None, random_state: int = 0, confirm_variables: bool = False, + importance_method: str = "model", ): if not isinstance(collective, bool): raise ValueError( @@ -253,6 +277,7 @@ def __init__( self.n_probes = n_probes self.threshold = threshold self.random_state = random_state + self.importance_method = importance_method def fit(self, X: pd.DataFrame, y: pd.Series): """ @@ -268,6 +293,28 @@ def fit(self, X: pd.DataFrame, y: pd.Series): # check input dataframe X, y = check_X_y(X, y) + methods = [ + "model", + "correlation", + "f_classif", + "f_regression", + "mutual_info_classif", + "mutual_info_regression", + "permutation", + ] + if ( + not isinstance(self.importance_method, str) + or self.importance_method not in methods + ): + raise ValueError(f"importance_method must be one of {methods}.") + if ( + self.importance_method in ["model", "permutation"] + and self.estimator is None + ): + raise ValueError( + "An estimator is required for model or permutation importance." + ) + self.variables_ = _select_numerical_variables( X, self.variables, self.confirm_variables ) @@ -283,7 +330,9 @@ def fit(self, X: pd.DataFrame, y: pd.Series): X_new = pd.concat([X[self.variables_], self.probe_features_], axis=1) - if self.collective is True: + if self.importance_method != "model": + self._rank_features(X_new, y) + elif self.collective is True: # train model using entire dataset and derive feature importance f_importance_mean, f_importance_std = find_feature_importance( X=X_new, @@ -315,6 +364,71 @@ def fit(self, X: pd.DataFrame, y: pd.Series): return self + def _rank_features(self, X, y): + """Aggregate scores over folds, using held-out data for permutation.""" + method = self.importance_method + classification = method in ["f_classif", "mutual_info_classif"] or ( + method == "permutation" and is_classifier(self.estimator) + ) + cv = check_cv(self.cv, y=y, classifier=classification) + scores = [] + for train, test in cv.split(X, y, self.groups): + X_train, y_train = X.iloc[train], y.iloc[train] + if method == "permutation": + estimator = clone(self.estimator).fit(X_train, y_train) + score = permutation_importance( + estimator, + X.iloc[test], + y.iloc[test], + scoring=self.scoring, + random_state=self.random_state, + ).importances_mean + elif method == "correlation": + score = np.abs(r_regression(X_train, y_train)) + elif method in ["f_classif", "f_regression"]: + scorer = f_classif if method == "f_classif" else f_regression + score = scorer(X_train, y_train)[0] + else: + scorer = ( + mutual_info_classif + if method == "mutual_info_classif" + else mutual_info_regression + ) + discrete = [False] * len(self.variables_) + [ + name.startswith(("binary_", "discrete_uniform_", "poisson_")) + for name in self.probe_features_.columns + ] + score = scorer( + X_train, + y_train, + discrete_features=discrete, + random_state=self.random_state, + ) + # Constant columns carry no signal. Some sklearn versions return inf + # for their correlation after floating-point centring, rather than NaN. + if method != "permutation": + score[X_train.nunique().to_numpy() <= 1] = 0.0 + if y_train.nunique() <= 1: + score[:] = 0.0 + if method == "correlation": + score = np.clip(score, 0.0, 1.0) + # Constant variables can yield undefined F scores: they carry no signal. + scores.append(np.nan_to_num(score, nan=0.0)) + if not scores: + raise ValueError("cv must yield at least one split.") + scores = np.asarray(scores) + # F scores may reach the largest finite float for perfect predictors. + # Scale before aggregation to avoid overflowing the mean or variance. + scale = np.max(np.abs(scores), axis=0) + scale[scale == 0] = 1.0 + normalized = scores / scale + self.feature_importances_ = pd.Series( + np.mean(normalized, axis=0) * scale, index=X.columns + ) + self.feature_importances_std_ = pd.Series( + np.std(normalized, axis=0) * scale, index=X.columns + ) + def _generate_probe_features(self, n_obs: int) -> pd.DataFrame: """ Returns a dataframe comprised of the probe features using the diff --git a/tests/test_selection/test_probe_ranking_methods.py b/tests/test_selection/test_probe_ranking_methods.py new file mode 100644 index 000000000..69e43231f --- /dev/null +++ b/tests/test_selection/test_probe_ranking_methods.py @@ -0,0 +1,146 @@ +import numpy as np +import pandas as pd +import pytest +from sklearn.base import clone +from sklearn.feature_selection import f_classif, f_regression, r_regression +from sklearn.inspection import permutation_importance +from sklearn.linear_model import LinearRegression +from sklearn.model_selection import GroupKFold, KFold + +from feature_engine.selection import ProbeFeatureSelection + + +def data(): + rng = np.random.default_rng(24) + X = pd.DataFrame(rng.normal(size=(160, 3)), columns=["signal", "noise", "other"]) + y = 4 * X.signal + rng.normal(scale=0.2, size=len(X)) + return X, y + + +@pytest.mark.parametrize( + "method", + [ + "correlation", + "f_classif", + "f_regression", + "mutual_info_classif", + "mutual_info_regression", + ], +) +def test_statistical_methods_are_reproducible_and_keep_signal(method): + X, y = data() + if method.endswith("classif"): + y = (y > 0).astype(int) + kwargs = dict(importance_method=method, n_probes=3, random_state=7, cv=3) + selector = ProbeFeatureSelection(**kwargs).fit(X, y) + other = clone(selector).fit(X, y) + pd.testing.assert_series_equal( + selector.feature_importances_, other.feature_importances_ + ) + assert "signal" not in selector.features_to_drop_ + + assert len(selector.feature_importances_) == 6 + pd.testing.assert_frame_equal( + selector.transform(X), X.drop(columns=selector.features_to_drop_) + ) + + +@pytest.mark.parametrize( + "method,scorer", + [ + ("correlation", r_regression), + ("f_regression", f_regression), + ("f_classif", f_classif), + ], +) +def test_statistical_scores_match_training_fold_reference(method, scorer): + X, y = data() + if method == "f_classif": + y = (y > 0).astype(int) + cv = KFold(4) + selector = ProbeFeatureSelection(importance_method=method, cv=cv).fit(X, y) + augmented = pd.concat([X, selector.probe_features_], axis=1) + scores = [] + for train, _ in cv.split(X): + score = scorer(augmented.iloc[train], y.iloc[train]) + scores.append(np.abs(score) if method == "correlation" else score[0]) + np.testing.assert_allclose(selector.feature_importances_, np.mean(scores, axis=0)) + np.testing.assert_allclose( + selector.feature_importances_std_, np.std(scores, axis=0) + ) + + +def test_permutation_uses_held_out_folds_and_does_not_fit_original_estimator(): + X, y = data() + estimator = LinearRegression() + selector = ProbeFeatureSelection( + estimator, importance_method="permutation", scoring="r2", cv=3 + ).fit(X, y) + assert not hasattr(estimator, "coef_") + assert selector.feature_importances_["signal"] > 1 + assert "signal" not in selector.features_to_drop_ + + augmented = pd.concat([X, selector.probe_features_], axis=1) + scores = [] + for train, test in KFold(3).split(X): + fitted = clone(estimator).fit(augmented.iloc[train], y.iloc[train]) + scores.append( + permutation_importance( + fitted, augmented.iloc[test], y.iloc[test], scoring="r2", random_state=0 + ).importances_mean + ) + np.testing.assert_allclose(selector.feature_importances_, np.mean(scores, axis=0)) + + +def test_group_splitter_supported(): + X, y = data() + selector = ProbeFeatureSelection( + importance_method="correlation", + cv=GroupKFold(4), + groups=np.repeat(np.arange(8), 20), + ) + selector.fit(X, y) + assert "signal" not in selector.features_to_drop_ + + +@pytest.mark.parametrize("method", ["model", "permutation"]) +def test_model_based_methods_require_estimator(method): + with pytest.raises(ValueError, match="estimator is required"): + ProbeFeatureSelection(importance_method=method).fit(*data()) + + +def test_invalid_method_is_validated_after_set_params(): + selector = ProbeFeatureSelection().set_params(importance_method="unknown") + with pytest.raises(ValueError, match="importance_method"): + selector.fit(*data()) + + +def test_correlation_retains_negative_signal_and_handles_constants(): + X, y = data() + X["signal"] *= -1 + X["constant"] = 1.0 + selector = ProbeFeatureSelection(importance_method="correlation").fit(X, y) + assert selector.feature_importances_["signal"] > 0.9 + assert selector.feature_importances_["constant"] == 0 + assert "constant" in selector.features_to_drop_ + + +def test_perfect_anova_predictor_has_finite_aggregated_scores(): + X = pd.DataFrame({"signal": [0.0, 1.0] * 20, "constant": [1.0] * 40}) + selector = ProbeFeatureSelection(importance_method="f_classif", cv=2).fit( + X, X.signal + ) + assert np.isfinite(selector.feature_importances_).all() + assert np.isfinite(selector.feature_importances_std_).all() + + +@pytest.mark.parametrize("method", ["mutual_info_classif", "mutual_info_regression"]) +def test_mutual_information_supports_mixed_probe_distributions(method): + X, y = data() + if method.endswith("classif"): + y = (y > 0).astype(int) + selector = ProbeFeatureSelection( + importance_method=method, distribution="all", cv=2 + ).fit(X, y) + assert len(selector.feature_importances_) == 8 + assert np.isfinite(selector.feature_importances_).all() From cc53d0288d2b6bac489b9c2e5e567418943e0eb6 Mon Sep 17 00:00:00 2001 From: Lucas Morin Date: Sun, 18 Oct 2026 17:07:52 +0200 Subject: [PATCH 2/2] Fix sparse input validation in sklearn test adapter --- tests/estimator_checks/sklearn_check_wrapper.py | 4 ++++ .../test_check_input_wrapper.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/test_sklearn_compatible/test_check_input_wrapper.py diff --git a/tests/estimator_checks/sklearn_check_wrapper.py b/tests/estimator_checks/sklearn_check_wrapper.py index 5e5deb459..ac4511f07 100644 --- a/tests/estimator_checks/sklearn_check_wrapper.py +++ b/tests/estimator_checks/sklearn_check_wrapper.py @@ -20,6 +20,7 @@ import numpy as np import pandas as pd +from scipy.sparse import issparse from sklearn.base import BaseEstimator, TransformerMixin, clone from sklearn.utils.validation import check_is_fitted @@ -62,6 +63,9 @@ def _to_df(X): if hasattr(X, "iloc"): return X + if issparse(X): + raise TypeError("Sparse input is not supported by this transformer.") + arr = np.asarray(X) if arr.ndim == 0: raise ValueError( diff --git a/tests/test_sklearn_compatible/test_check_input_wrapper.py b/tests/test_sklearn_compatible/test_check_input_wrapper.py new file mode 100644 index 000000000..4da6fd393 --- /dev/null +++ b/tests/test_sklearn_compatible/test_check_input_wrapper.py @@ -0,0 +1,17 @@ +import numpy as np +import pytest +from scipy import sparse + +from tests.estimator_checks.sklearn_check_wrapper import _SklearnCheckInputWrapper + + +@pytest.mark.parametrize("format", ["bsr", "coo", "csc", "csr", "dia", "dok", "lil"]) +@pytest.mark.parametrize("kind", ["matrix", "array"]) +def test_sparse_input_raises_explicit_error(format, kind): + constructor = getattr(sparse, f"{format}_{kind}", None) + if constructor is None: + pytest.skip("Sparse array format unavailable in this SciPy version.") + X = constructor(np.eye(3)) + + with pytest.raises(TypeError, match="[Ss]parse.*not supported"): + _SklearnCheckInputWrapper._to_df(X)