[Python] Multiclass focal loss in xgboost doesn't train

I have a dataframe with 60 columns as variables and a last column as target class (4 possible classes). I want to implement a custom loss function. I want that function to be the focal loss for a multiclass problem since I have a very imbalanced dataset. This is my approach following the docs from xgboost for custom functions:

import numpy as np
import xgboost
from scipy.special import log_softmax, softmax
from sklearn.utils import compute_sample_weight
from sklearn.model_selection import train_test_split

num_classes = 4

def focal_loss_multiclass_objective(inputs: np.ndarray, dtrain: xgboost.DMatrix,
alpha=1, gamma=2, num_classes=4):
# Convert logits to probabilities and log-probabilities
log_prob = log_softmax(inputs)
prob = np.exp(log_prob)
print(f"prob: {prob}")

# One hot encoding the correct class
targets = dtrain.get_label().astype(np.int64)
targets_one_hot = np.identity(num_classes)[targets]

# Gradient
grad = -alpha*targets_one_hot*((1-prob)**(gamma-1)) * (-gamma*log_prob
+ (1-prob)/(prob+1e-6))
print(f"grad: {grad}")

# Hessian
hess = -alpha*targets_one_hot*(gamma*(gamma-1)*((1-prob)**(gamma-2)) * log_prob
- 2*gamma*((1-prob)**(gamma-1))/(prob+1e-6)
- ((1-prob)**gamma)/(prob+1e-6)**2)
print("hess: ", hess)

return grad, hess


The gradient and hessian of the multiclass focal loss are kind of a hell, but I have double checked them with sympy.

The training:

X = df.iloc[:, 0:60]
y = df.iloc[:, -1]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# Sample weights:
sample_weights = compute_sample_weight('balanced', y_train)
print(np.unique(sample_weights))

# Create a DMatrix entity
dtrain = xgboost.DMatrix(X_train, label=y_train, weight=sample_weights)
dtest = xgboost.DMatrix(X_test, label=y_test)

params = {
'num_class': num_classes,
#'disable_default_eval_metric': True,
'eval_metric': 'mlogloss',
'eta': 0.3,
'max_depth': 6,
'subsample': 0.8,
'colsample_bytree': 0.8,
'tree_method': 'auto'
}


# Train the model
model = xgboost.train(params,
dtrain,
num_boost_round=100,
obj=focal_loss_multiclass_objective)


But I obtain always a 25 % probability after the training. I don't know what I am doing wrong.

y_pred_logits = model.predict(dtest, output_margin=True)
y_prob = softmax(y_pred_logits, axis=1)
print(y_pred_logits)
print(y_prob)

Continue reading...