Add statistical and permutation ranking to probe selection

docs/user_guide/selection/ProbeFeatureSelection.rst

@@ -649,3 +649,43 @@ For more details about this and other feature selection methods check out these
 Both our book and course are suitable for beginners and more advanced data scientists
 alike. By purchasing them you are supporting `Sole <https://linkedin.com/in/soledad-galli>`_,
 the main developer of feature-engine.
+
+
+Statistical and permutation ranking
+-----------------------------------
+
+Set ``importance_method="correlation"`` to compare absolute Pearson correlations
+of real and random features with a numeric target. No estimator is required::
+
+    selector = ProbeFeatureSelection(
+        importance_method="correlation", n_probes=5, cv=5, random_state=42
+    )
+    X_selected = selector.fit_transform(X_train, y_train)
+
+``f_classif`` and ``f_regression`` provide ANOVA F-score ranking for classification
+and regression. ``mutual_info_classif`` and ``mutual_info_regression`` provide
+mutual-information ranking. Select the variant explicitly for your target type.
+These methods compute scores on the training portion of each cross-validation
+split and expose their mean and standard deviation in the usual attributes.
+Mutual-information methods treat the original numeric features as continuous;
+use these methods for continuous predictors. Binary, discrete-uniform and
+Poisson probe columns are treated as discrete.
+
+For an estimator without embedded feature importances, use held-out permutation
+importance::
+
+    from sklearn.neighbors import KNeighborsRegressor
+
+    selector = ProbeFeatureSelection(
+        estimator=KNeighborsRegressor(), importance_method="permutation",
+        scoring="neg_mean_squared_error", cv=5, random_state=42
+    )
+
+The estimator is cloned for each fold, fitted only on training rows, and scored
+on validation rows after five permutations per column. The importance is the
+mean decrease in score. The reported standard deviation is across fold means,
+not across individual permutations. ``collective`` applies only to the default
+``importance_method="model"``. Existing calls preserve their behaviour.
+
+Fit the selector inside the training part of your evaluation pipeline. For time
+series, pass an appropriate chronological splitter through ``cv``.

feature_engine/selection/probe_feature_selection.py

@@ -2,6 +2,16 @@
 
 import numpy as np
 import pandas as pd
+from sklearn.base import clone, is_classifier
+from sklearn.feature_selection import (
+    f_classif,
+    f_regression,
+    mutual_info_classif,
+    mutual_info_regression,
+    r_regression,
+)
+from sklearn.inspection import permutation_importance
+from sklearn.model_selection import check_cv
 
 from feature_engine._docstrings.fit_attributes import (
     _feature_names_in_docstring,
@@ -14,10 +24,10 @@
 from feature_engine._docstrings.methods import _fit_transform_docstring
 from feature_engine._docstrings.selection._docstring import (
     _cv_docstring,
-    _groups_docstring,
     _features_to_drop_docstring,
     _fit_docstring,
     _get_support_docstring,
+    _groups_docstring,
     _scoring_docstring,
     _transform_docstring,
     _variables_attribute_docstring,
@@ -80,10 +90,11 @@ class ProbeFeatureSelection(BaseSelector):
 
     Parameters
     ----------
-    estimator: object
+    estimator: object, default=None
         A scikit-learn estimator for regression or classification. If `collective=True`,
         the estimator must have either a `feature_importances_` or a `coef_` attribute
         after fitting.
+        Required for ``importance_method='model'`` or ``'permutation'``.
 
     {variables}
 
@@ -120,6 +131,18 @@ class ProbeFeatureSelection(BaseSelector):
 
     {groups}
 
+    importance_method: str, default='model'
+        How to rank real and probe features. 'model' preserves the behaviour of
+        ``collective``. 'correlation' uses absolute Pearson correlation with a
+        numeric target. 'f_classif' and 'f_regression' use ANOVA F scores for
+        classification and regression respectively. 'mutual_info_classif' and
+        'mutual_info_regression' estimate mutual information, treating input
+        features as continuous and discrete probe distributions as discrete.
+        These statistical scores are calculated on each training fold.
+        'permutation' fits the estimator on each training fold and measures the
+        decrease in ``scoring`` on its held-out fold, averaged over five shuffles.
+        ``collective`` is used only with 'model'.
+
     Attributes
     ----------
     probe_features_:
@@ -181,7 +204,7 @@ class ProbeFeatureSelection(BaseSelector):
 
     def __init__(
         self,
-        estimator,
+        estimator=None,
         variables: Variables = None,
         collective: bool = True,
         scoring: str = "roc_auc",
@@ -193,6 +216,7 @@ def __init__(
         groups=None,
         random_state: int = 0,
         confirm_variables: bool = False,
+        importance_method: str = "model",
     ):
         if not isinstance(collective, bool):
             raise ValueError(
@@ -253,6 +277,7 @@ def __init__(
         self.n_probes = n_probes
         self.threshold = threshold
         self.random_state = random_state
+        self.importance_method = importance_method
 
     def fit(self, X: pd.DataFrame, y: pd.Series):
         """
@@ -268,6 +293,28 @@ def fit(self, X: pd.DataFrame, y: pd.Series):
         # check input dataframe
         X, y = check_X_y(X, y)
 
+        methods = [
+            "model",
+            "correlation",
+            "f_classif",
+            "f_regression",
+            "mutual_info_classif",
+            "mutual_info_regression",
+            "permutation",
+        ]
+        if (
+            not isinstance(self.importance_method, str)
+            or self.importance_method not in methods
+        ):
+            raise ValueError(f"importance_method must be one of {methods}.")
+        if (
+            self.importance_method in ["model", "permutation"]
+            and self.estimator is None
+        ):
+            raise ValueError(
+                "An estimator is required for model or permutation importance."
+            )
+
         self.variables_ = _select_numerical_variables(
             X, self.variables, self.confirm_variables
         )
@@ -283,7 +330,9 @@ def fit(self, X: pd.DataFrame, y: pd.Series):
 
         X_new = pd.concat([X[self.variables_], self.probe_features_], axis=1)
 
-        if self.collective is True:
+        if self.importance_method != "model":
+            self._rank_features(X_new, y)
+        elif self.collective is True:
             # train model using entire dataset and derive feature importance
             f_importance_mean, f_importance_std = find_feature_importance(
                 X=X_new,
@@ -315,6 +364,71 @@ def fit(self, X: pd.DataFrame, y: pd.Series):
 
         return self
 
+    def _rank_features(self, X, y):
+        """Aggregate scores over folds, using held-out data for permutation."""
+        method = self.importance_method
+        classification = method in ["f_classif", "mutual_info_classif"] or (
+            method == "permutation" and is_classifier(self.estimator)
+        )
+        cv = check_cv(self.cv, y=y, classifier=classification)
+        scores = []
+        for train, test in cv.split(X, y, self.groups):
+            X_train, y_train = X.iloc[train], y.iloc[train]
+            if method == "permutation":
+                estimator = clone(self.estimator).fit(X_train, y_train)
+                score = permutation_importance(
+                    estimator,
+                    X.iloc[test],
+                    y.iloc[test],
+                    scoring=self.scoring,
+                    random_state=self.random_state,
+                ).importances_mean
+            elif method == "correlation":
+                score = np.abs(r_regression(X_train, y_train))
+            elif method in ["f_classif", "f_regression"]:
+                scorer = f_classif if method == "f_classif" else f_regression
+                score = scorer(X_train, y_train)[0]
+            else:
+                scorer = (
+                    mutual_info_classif
+                    if method == "mutual_info_classif"
+                    else mutual_info_regression
+                )
+                discrete = [False] * len(self.variables_) + [
+                    name.startswith(("binary_", "discrete_uniform_", "poisson_"))
+                    for name in self.probe_features_.columns
+                ]
+                score = scorer(
+                    X_train,
+                    y_train,
+                    discrete_features=discrete,
+                    random_state=self.random_state,
+                )
+            # Constant columns carry no signal. Some sklearn versions return inf
+            # for their correlation after floating-point centring, rather than NaN.
+            if method != "permutation":
+                score[X_train.nunique().to_numpy() <= 1] = 0.0
+                if y_train.nunique() <= 1:
+                    score[:] = 0.0
+            if method == "correlation":
+                score = np.clip(score, 0.0, 1.0)
+            # Constant variables can yield undefined F scores: they carry no signal.
+            scores.append(np.nan_to_num(score, nan=0.0))
+        if not scores:
+            raise ValueError("cv must yield at least one split.")
+        scores = np.asarray(scores)
+        # F scores may reach the largest finite float for perfect predictors.
+        # Scale before aggregation to avoid overflowing the mean or variance.
+        scale = np.max(np.abs(scores), axis=0)
+        scale[scale == 0] = 1.0
+        normalized = scores / scale
+        self.feature_importances_ = pd.Series(
+            np.mean(normalized, axis=0) * scale, index=X.columns
+        )
+        self.feature_importances_std_ = pd.Series(
+            np.std(normalized, axis=0) * scale, index=X.columns
+        )
+
     def _generate_probe_features(self, n_obs: int) -> pd.DataFrame:
         """
         Returns a dataframe comprised of the probe features using the

tests/estimator_checks/sklearn_check_wrapper.py

@@ -20,6 +20,7 @@
 
 import numpy as np
 import pandas as pd
+from scipy.sparse import issparse
 from sklearn.base import BaseEstimator, TransformerMixin, clone
 from sklearn.utils.validation import check_is_fitted
 
@@ -62,6 +63,9 @@ def _to_df(X):
         if hasattr(X, "iloc"):
             return X
 
+        if issparse(X):
+            raise TypeError("Sparse input is not supported by this transformer.")
+
         arr = np.asarray(X)
         if arr.ndim == 0:
             raise ValueError(