Move skl eval_metric early_stopping rounds to model params.

These 2 parameters are now model parameters that can be set at constructor and
`set_params` method.

Doc and test.

Fix return.

non breaking.

Author: trivialfis

python-package/xgboost/dask.py

@@ -1676,8 +1676,8 @@ async def _fit_async(
             obj: Optional[Callable] = _objective_decorator(self.objective)
         else:
             obj = None
-        model, metric, params = self._configure_fit(
-            booster=xgb_model, eval_metric=eval_metric, params=params
+        model, metric, params, early_stopping_rounds = self._configure_fit(
+            xgb_model, eval_metric, params, early_stopping_rounds
         )
         results = await self.client.sync(
             _train_async,
@@ -1778,8 +1778,8 @@ async def _fit_async(
             obj: Optional[Callable] = _objective_decorator(self.objective)
         else:
             obj = None
-        model, metric, params = self._configure_fit(
-            booster=xgb_model, eval_metric=eval_metric, params=params
+        model, metric, params, early_stopping_rounds = self._configure_fit(
+            xgb_model, eval_metric, params, early_stopping_rounds
         )
         results = await self.client.sync(
             _train_async,
@@ -1903,9 +1903,9 @@ def _argmax(x: Any) -> Any:
 """,
     ["estimators", "model"],
     end_note="""
-        Note
-        ----
-        For dask implementation, group is not supported, use qid instead.
+        .. note::
+
+            For dask implementation, group is not supported, use qid instead.
 """,
 )
 class DaskXGBRanker(DaskScikitLearnBase, XGBRankerMixIn):
@@ -1963,8 +1963,8 @@ async def _fit_async(
                 raise ValueError(
                     "Custom evaluation metric is not yet supported for XGBRanker."
                 )
-        model, metric, params = self._configure_fit(
-            booster=xgb_model, eval_metric=eval_metric, params=params
+        model, metric, params, early_stopping_rounds = self._configure_fit(
+            xgb_model, eval_metric, params, early_stopping_rounds
         )
         results = await self.client.sync(
             _train_async,

python-package/xgboost/sklearn.py

@@ -90,6 +90,19 @@ def inner(preds: np.ndarray, dmatrix: DMatrix) -> Tuple[np.ndarray, np.ndarray]:
     return inner
 
 
+def _metric_decorator(func: Callable) -> Metric:
+    """Decorate a metric function from sklearn.
+
+    Converts an objective function using the typical sklearn metrics signature so that it
+    is compatible with ``xgboost.training.train``
+
+    """
+    def inner(y_score: np.ndarray, dmatrix: DMatrix) -> float:
+        y_true = dmatrix.get_label()
+        return func.__name__, func(y_true, y_score)
+    return inner
+
+
 __estimator_doc = '''
     n_estimators : int
         Number of gradient boosted trees.  Equivalent to number of boosting
@@ -184,6 +197,46 @@ def inner(preds: np.ndarray, dmatrix: DMatrix) -> Tuple[np.ndarray, np.ndarray]:
         Experimental support for categorical data.  Do not set to true unless you are
         interested in development. Only valid when `gpu_hist` and dataframe are used.
 
+    eval_metric : Optional[Union[str, List[str], Callable]]
+        Metric used for monitoring the training result and early stopping.  It can be a
+        string or list of strings as names of predefined metric in XGBoost (See
+        doc/parameter.rst), one of the metrics in :py:mod:`sklearn.metrics`, or any other
+        user defined metric that looks like `sklearn.metrics`.
+
+        Unlike scikit-learn `scoring` parameter, when a callable object is provided, it's
+        assumed to be a cost function and by default XGBoost will minimize the result
+        during early stopping.
+
+        For advanced usage on Early stopping like directly choosing to maximize instead of
+        minimize, see :py:obj:`xgboost.callback.EarlyStopping`.
+
+        .. versionadded:: 1.5.1
+
+        .. note::
+
+             This parameter replaces `eval_metric` in
+             :py:meth:`fit` method.
+
+    early_stopping_rounds : Optional[int]
+        Activates early stopping. Validation metric needs to improve at least once in
+        every **early_stopping_rounds** round(s) to continue training.  Requires at least
+        one item in **eval_set** in :py:meth:`xgboost.sklearn.XGBModel.fit`.
+
+        The method returns the model from the last iteration (not the best one).  If
+        there's more than one item in **eval_set**, the last entry will be used for early
+        stopping.  If there's more than one metric in **eval_metric**, the last metric
+        will be used for early stopping.
+
+        If early stopping occurs, the model will have three additional fields:
+        ``clf.best_score``, ``clf.best_iteration`` and ``clf.best_ntree_limit``.
+
+        .. versionadded:: 1.5.1
+
+        .. note::
+
+            This parameter replaces `early_stopping_rounds` in
+            :py:meth:`fit` method.
+
     kwargs : dict, optional
         Keyword arguments for XGBoost Booster object.  Full documentation of
         parameters can be found here:
@@ -399,6 +452,8 @@ def __init__(
         validate_parameters: Optional[bool] = None,
         predictor: Optional[str] = None,
         enable_categorical: bool = False,
+        eval_metric=None,
+        early_stopping_rounds=None,
         **kwargs: Any
     ) -> None:
         if not SKLEARN_INSTALLED:
@@ -435,6 +490,8 @@ def __init__(
         self.validate_parameters = validate_parameters
         self.predictor = predictor
         self.enable_categorical = enable_categorical
+        self.eval_metric = eval_metric
+        self.early_stopping_rounds = early_stopping_rounds
         if kwargs:
             self.kwargs = kwargs
 
@@ -545,10 +602,15 @@ def get_xgb_params(self) -> Dict[str, Any]:
         params = self.get_params()
         # Parameters that should not go into native learner.
         wrapper_specific = {
-            'importance_type', 'kwargs', 'missing', 'n_estimators', 'use_label_encoder',
+            'importance_type',
+            'kwargs',
+            'missing',
+            'n_estimators',
+            'use_label_encoder',
             "enable_categorical"
+            "early_stopping_rounds"
         }
-        filtered = {}
+        filtered = dict()
         for k, v in params.items():
             if k not in wrapper_specific and not callable(v):
                 filtered[k] = v
@@ -636,15 +698,32 @@ def _configure_fit(
         booster: Optional[Union[Booster, "XGBModel", str]],
         eval_metric: Optional[Union[Callable, str, List[str]]],
         params: Dict[str, Any],
-    ) -> Tuple[Optional[Union[Booster, str]], Optional[Metric], Dict[str, Any]]:
-        # pylint: disable=protected-access, no-self-use
-        if isinstance(booster, XGBModel):
+        early_stopping_rounds: Optional[int],
+    ) -> Tuple[Optional[Union[Booster, str, "XGBModel"]], Optional[Metric], Dict[str, Any], Optional[int]]:
+        # pylint: disable=protected-access
+        model = booster
+        if hasattr(model, "_Booster"):
             # Handle the case when xgb_model is a sklearn model object
-            model: Optional[Union[Booster, str]] = booster._Booster
-        else:
-            model = booster
+            model = model._Booster
+
+        if eval_metric is not None:
+            warnings.warn(
+                "eval_metric for `fit` method is deprecated, use `eval_metric` in "
+                "constructor or `set_params` instead.",
+                UserWarning,
+            )
 
+        # configure callable evaluation metric
         feval = eval_metric if callable(eval_metric) else None
+        if self.eval_metric is not None and feval is not None:
+            warnings.warn(
+                "Overriding `eval_metric` from `fit` with `eval_metric` from parameter",
+                UserWarning
+            )
+        if callable(self.eval_metric):
+            feval = _metric_decorator(self.eval_metric)
+
+        # configure string/list evaluation metric
         if eval_metric is not None:
             if callable(eval_metric):
                 eval_metric = None
@@ -656,7 +735,26 @@ def _configure_fit(
                 " current tree method yet."
             )
 
-        return model, feval, params
+        # configure early_stopping_rounds
+        if early_stopping_rounds is not None:
+            warnings.warn(
+                "`early_stopping_rounds` is deprecated, use `early_stopping_rounds` "
+                "in constructor or `set_params` instead.",
+                UserWarning,
+            )
+            if (
+                self.early_stopping_rounds is not None
+                and self.early_stopping_rounds != early_stopping_rounds
+            ):
+                raise ValueError("2 different `early_stopping_rounds` are provided.")
+
+        early_stopping_rounds = (
+            self.early_stopping_rounds
+            if self.early_stopping_rounds is not None
+            else early_stopping_rounds
+        )
+
+        return model, feval, params, early_stopping_rounds
 
     def _set_evaluation_result(self, evals_result: TrainingCallback.EvalsLog) -> None:
         if evals_result:
@@ -704,31 +802,10 @@ def fit(
             A list of (X, y) tuple pairs to use as validation sets, for which
             metrics will be computed.
             Validation metrics will help us track the performance of the model.
-        eval_metric :
-            If a str, should be a built-in evaluation metric to use. See doc/parameter.rst.
-
-            If a list of str, should be the list of multiple built-in evaluation metrics
-            to use.
-
-            If callable, a custom evaluation metric. The call signature is
-            ``func(y_predicted, y_true)`` where ``y_true`` will be a DMatrix object such
-            that you may need to call the ``get_label`` method. It must return a str,
-            value pair where the str is a name for the evaluation and value is the value
-            of the evaluation function. The callable custom objective is always minimized.
-        early_stopping_rounds :
-            Activates early stopping. Validation metric needs to improve at least once in
-            every **early_stopping_rounds** round(s) to continue training.
-            Requires at least one item in **eval_set**.
-
-            The method returns the model from the last iteration (not the best one).
-            If there's more than one item in **eval_set**, the last entry will be used
-            for early stopping.
-
-            If there's more than one metric in **eval_metric**, the last metric will be
-            used for early stopping.
-
-            If early stopping occurs, the model will have three additional fields:
-            ``clf.best_score``, ``clf.best_iteration``.
+        eval_metric : str, list of str, or callable, optional
+            Deprecated, use `eval_metric` in constructor or `set_params` instead.
+        early_stopping_rounds : int
+            Deprecated, use `early_stopping_rounds` in constructor instead.
         verbose :
             If `verbose` and an evaluation set is used, writes the evaluation metric
             measured on the validation set to stderr.
@@ -785,7 +862,9 @@ def fit(
         else:
             obj = None
 
-        model, feval, params = self._configure_fit(xgb_model, eval_metric, params)
+        model, feval, params, early_stopping_rounds = self._configure_fit(
+            xgb_model, eval_metric, params, early_stopping_rounds
+        )
         self._Booster = train(
             params,
             train_dmatrix,
@@ -1223,7 +1302,9 @@ def fit(
         else:
             label_transform = lambda x: x
 
-        model, feval, params = self._configure_fit(xgb_model, eval_metric, params)
+        model, feval, params, early_stopping_rounds = self._configure_fit(
+            xgb_model, eval_metric, params, early_stopping_rounds
+        )
         train_dmatrix, evals = _wrap_evaluation_matrices(
             missing=self.missing,
             X=X,
@@ -1359,8 +1440,9 @@ def evals_result(self) -> TrainingCallback.EvalsLog:
 
         If **eval_set** is passed to the `fit` function, you can call
         ``evals_result()`` to get evaluation results for all passed **eval_sets**.
-        When **eval_metric** is also passed to the `fit` function, the
-        **evals_result** will contain the **eval_metrics** passed to the `fit` function.
+
+        When **eval_metric** is also passed as a parameter, the **evals_result** will
+        contain the **eval_metric** passed to the `fit` function.
 
         Returns
         -------
@@ -1371,13 +1453,14 @@ def evals_result(self) -> TrainingCallback.EvalsLog:
 
         .. code-block:: python
 
-            param_dist = {'objective':'binary:logistic', 'n_estimators':2}
+            param_dist = {
+                'objective':'binary:logistic', 'n_estimators':2, eval_metric="logloss"
+            }
 
             clf = xgb.XGBClassifier(**param_dist)
 
             clf.fit(X_train, y_train,
                     eval_set=[(X_train, y_train), (X_test, y_test)],
-                    eval_metric='logloss',
                     verbose=True)
 
             evals_result = clf.evals_result()
@@ -1388,6 +1471,7 @@ def evals_result(self) -> TrainingCallback.EvalsLog:
 
             {'validation_0': {'logloss': ['0.604835', '0.531479']},
             'validation_1': {'logloss': ['0.41965', '0.17686']}}
+
         """
         if self.evals_result_:
             evals_result = self.evals_result_
@@ -1534,15 +1618,15 @@ def fit(
     'Implementation of the Scikit-Learn API for XGBoost Ranking.',
     ['estimators', 'model'],
     end_note='''
-        Note
-        ----
-        A custom objective function is currently not supported by XGBRanker.
-        Likewise, a custom metric function is not supported either.
+        .. note::
+
+            A custom objective function is currently not supported by XGBRanker.
+            Likewise, a custom metric function is not supported either.
 
-        Note
-        ----
-        Query group information is required for ranking tasks by either using the `group`
-        parameter or `qid` parameter in `fit` method.
+        .. note::
+
+            Query group information is required for ranking tasks by either using the
+            `group` parameter or `qid` parameter in `fit` method.
 
         Before fitting the model, your data need to be sorted by query group. When fitting
         the model, you need to provide an additional array that contains the size of each
@@ -1644,22 +1728,10 @@ def fit(
         eval_qid :
             A list in which ``eval_qid[i]`` is the array containing query ID of ``i``-th
             pair in **eval_set**.
-        eval_metric :
-            If a str, should be a built-in evaluation metric to use. See
-            doc/parameter.rst.
-            If a list of str, should be the list of multiple built-in evaluation metrics
-            to use. The custom evaluation metric is not yet supported for the ranker.
-        early_stopping_rounds :
-            Activates early stopping. Validation metric needs to improve at least once in
-            every **early_stopping_rounds** round(s) to continue training.  Requires at
-            least one item in **eval_set**.
-            The method returns the model from the last iteration (not the best one).  If
-            there's more than one item in **eval_set**, the last entry will be used for
-            early stopping.
-            If there's more than one metric in **eval_metric**, the last metric will be
-            used for early stopping.
-            If early stopping occurs, the model will have three additional fields:
-            ``clf.best_score``, ``clf.best_iteration`` and ``clf.best_ntree_limit``.
+        eval_metric : str, list of str, optional
+            The custom evaluation metric is not yet supported for the ranker.
+        early_stopping_rounds : int
+            Deprecated, use `early_stopping_rounds` in constructor instead.
         verbose :
             If `verbose` and an evaluation set is used, writes the evaluation metric
             measured on the validation set to stderr.
@@ -1724,7 +1796,9 @@ def fit(
         evals_result: TrainingCallback.EvalsLog = {}
         params = self.get_xgb_params()
 
-        model, feval, params = self._configure_fit(xgb_model, eval_metric, params)
+        model, feval, params, early_stopping_rounds = self._configure_fit(
+            xgb_model, eval_metric, params, early_stopping_rounds
+        )
         if callable(feval):
             raise ValueError(
                 'Custom evaluation metric is not yet supported for XGBRanker.'