My understanding is that in version 3.0.0, the default base_score for those GLM-inspired loss functions has been changed to use the closed-form solution, but reg:squaredlogerror shouldn't be one of them. Apparently it has been changed to use the mean of the target values. If closed-form solution is preferred over the one-step Newton approach, then at least the "exp-mean-log" should be used instead. What was the rationale for the change?
I am not sure what the "best" default choice for reg:squaredlogerror should be, but for the following dataset from a kaggle competition, one-step Newton is the only choice that would not lead to divergence. Note also how different the value from one-step Newton is from the other choices.
import numpy as np
import pandas as pd
import json
from sklearn.model_selection import train_test_split
from sklearn.metrics import root_mean_squared_log_error
from xgboost import XGBRegressor
# train.csv from https://www.kaggle.com/competitions/playground-series-s5e5/data
X = pd.read_csv('train.csv', index_col='id')
y = X.pop('Calories')
X.pop('Sex') # remove categorical variable for simplicity
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=0)
import sklearn; print(F'{sklearn.__version__=}')
import xgboost; print(F'{xgboost.__version__=}')
def rmsle(y_true, y_pred):
# y_pred could go wild when XGBRegressor diverges
return root_mean_squared_log_error(
y_true, np.clip(y_pred, -1+1e-6, None)
)
params = {'objective': 'reg:squaredlogerror', 'random_state': 0}
print('\n# 3.0.0 default')
model = XGBRegressor(**params)
print('RMSLE: ',
rmsle(
y_test,
model.fit(X_train, y_train).predict(X_test)
)
)
config = json.loads(model.get_booster().save_config())
print('Base score used: ', config['learner']['learner_model_param']['base_score'])
print('\n# base_score from mean of target')
model = XGBRegressor(**params, base_score=y_train.mean())
print('RMSLE: ',
rmsle(
y_test,
model.fit(X_train, y_train).predict(X_test)
)
)
config = json.loads(model.get_booster().save_config())
print('Base score used: ', config['learner']['learner_model_param']['base_score'])
print('\n# base_score from exp-mean-log of target')
model = XGBRegressor(**params, base_score=np.expm1(np.log1p(y_train).mean()))
print('RMSLE: ',
rmsle(
y_test,
model.fit(X_train, y_train).predict(X_test)
)
)
config = json.loads(model.get_booster().save_config())
print('Base score used: ', config['learner']['learner_model_param']['base_score'])
print('\n# base_score from one-step Newton')
model = XGBRegressor(
**params,
base_score=np.log1p(y_train).sum()/(1+np.log1p(y_train)).sum()
)
print('RMSLE: ',
rmsle(
y_test,
model.fit(X_train, y_train).predict(X_test)
)
)
config = json.loads(model.get_booster().save_config())
print('Base score used: ', config['learner']['learner_model_param']['base_score'])
# optional: xgboost 2.1.4
if 'xgboost_2_1_4' in globals():
with xgboost_2_1_4():
print('\n# 2.1.4 default')
import xgboost; print(F'{xgboost.__version__=}')
from xgboost import XGBRegressor
model = XGBRegressor(**params)
print('RMSLE: ',
rmsle(
y_test,
model.fit(X_train, y_train).predict(X_test)
)
)
config = json.loads(model.get_booster().save_config())
print('Base score used: ', config['learner']['learner_model_param']['base_score'])Expected output:
sklearn.__version__='1.6.1'
xgboost.__version__='3.0.0'
# 3.0.0 default
RMSLE: 9.073920894159356
Base score used: 8.827874E1
# base_score from mean of target
RMSLE: 9.073920894159356
Base score used: 8.827874E1
# base_score from exp-mean-log of target
RMSLE: 6.978755750166066
Base score used: 6.1856785E1
# base_score from one-step Newton
RMSLE: 0.07439065625016286
Base score used: 8.0548E-1
# 2.1.4 default
xgboost.__version__='2.1.4'
RMSLE: 0.07439065625016286
Base score used: 8.0548E-1
My understanding is that in version 3.0.0, the default
base_scorefor those GLM-inspired loss functions has been changed to use the closed-form solution, butreg:squaredlogerrorshouldn't be one of them. Apparently it has been changed to use the mean of the target values. If closed-form solution is preferred over the one-step Newton approach, then at least the "exp-mean-log" should be used instead. What was the rationale for the change?I am not sure what the "best" default choice for
reg:squaredlogerrorshould be, but for the following dataset from a kaggle competition, one-step Newton is the only choice that would not lead to divergence. Note also how different the value from one-step Newton is from the other choices.Expected output: