[Python] Pipeline FutureWarning: This Pipeline instance is not fitted yet

I am working on a fairly simple machine learning problem in the form of a practicum. I am using the following code to preprocess the data:

from preprocess.date_converter import DateConverter
from sklearn.pipeline import Pipeline
from preprocess.nan_fixer import CustomImputer
import pandas as pd
from preprocess.encoding import FrecuencyEncoding
from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA
from preprocess.scaler import CustomScaler



def basic_preprocess(df, target : str):

important_features = ['amt', 'category', 'merchant', 'trans_date_trans_time', 'unix_time', 'dob', 'street', 'merch_lat', 'merch_long', 'city', 'merch_zipcode', 'city_pop', 'job', 'last', 'first', 'cc_num', 'long', 'zip', "is_fraud"]

df = df.drop(["trans_num", "Unnamed: 0"], axis=1)
df = df[important_features]
df = df.copy()

# df, df_ = train_test_split(df, test_size=0.5, shuffle=True, random_state=42, stratify=df[target])

df_train, unseen_df = train_test_split(df, test_size=0.2, shuffle=True, random_state=42, stratify=df[target])
df_val, df_test = train_test_split(unseen_df, test_size=0.5, shuffle=True, random_state=42, stratify=unseen_df[target])

pipeline = Pipeline([
("date_converter", DateConverter("trans_date_trans_time")),
("imputer", CustomImputer(strategy="most_frequent")),
("encoding", FrecuencyEncoding()),
("scaler", CustomScaler(df.drop(target, axis=1).columns.tolist())),
])



pipeline.fit(df_train.drop(target, axis=1), df_train[target])
X_train = pd.DataFrame(pipeline.transform(df_train.drop(target, axis=1)), index=df_train.index)
X_test = pd.DataFrame(pipeline.transform(df_test.drop(target, axis=1)), index=df_test.index)
X_val = pd.DataFrame(pipeline.transform(df_val.drop(target, axis=1)), index=df_val.index)


df_train = pd.concat([X_train, df_train[target]], axis=1)
df_test = pd.concat([X_test, df_test[target]], axis=1)
df_val = pd.concat([X_val, df_val[target]], axis=1)


return [df_train, df_test, df_val]


Note: preprocess is a personal library that I created to make some transformers of my own. Here is the code for each of the transformers:

class CustomScaler(BaseEstimator, TransformerMixin):
"""
Receives the numeric columns from the dataframe. Scales their values using RobustScaler.
"""
def __init__(self, attributes):
self.attributes = attributes
self.scaler = RobustScaler() # Inicializa el escalador

def fit(self, X, y=None):
# Ajusta el escalador solo con los datos de entrenamiento
scale_cols = X[self.attributes]
self.scaler.fit(scale_cols)
return self

def transform(self, X, y=None):
X_copy = X.copy()
scale_attrs = X_copy[self.attributes]
# Usa el escalador ya ajustado para transformar los datos
X_scaled = self.scaler.transform(scale_attrs)
X_scaled = pd.DataFrame(X_scaled, columns=self.attributes, index=X_copy.index)
for attr in self.attributes:
X_copy[attr] = X_scaled[attr]
return X_copy


class CustomImputer(BaseEstimator, TransformerMixin):
"""
It implements SimpleImputer but, unlike SimpleImputer, it returns
dataframes and not numpy arrays.
"""
def __init__(self, strategy : str) -\> None:
self._imputer = SimpleImputer(strategy=strategy)

def fit(self, X, y=None):
self._imputer.fit(X, y)
return self

def transform(self, X, y=None):
transformed_data = self._imputer.transform(X)
return pd.DataFrame(transformed_data, columns=X.columns)


class FrecuencyEncoding(BaseEstimator, TransformerMixin):
"""
Searches for categorical columns and replaces them
using Frequency Encoding
"""
def __init__(self):
self._frequencies = {}

def fit(self, X, y=None):
for col in X.select_dtypes(['object']).columns:
self._frequencies[col] = X[col].value_counts().to_dict()
return self

def transform(self, X, y=None):
df_encoded = X.copy()
for col in df_encoded.select_dtypes(['object']).columns:
df_encoded[col] = df_encoded[col].map(self._frequencies[col])
df_encoded[col] = df_encoded[col].fillna(0)
return df_encoded


class DateConverter(BaseEstimator, TransformerMixin):
"""
Transformer created to convert all date columns to float.

It receives the list of date columns in the constructor.

Consider that in order to convert all elements (including nan) to float,
it was necessary to convert nans to 0 and then convert 0 to nans again.
"""
def __init__(self, attributes):
self.attributes = attributes
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X = X.copy()
def converter(col):
return pd.to_datetime(col, errors='coerce').apply(lambda x: x.timestamp() if pd.notna(x) else 0).astype(float).replace(0, np.nan) if col.name in self.attributes else col
return X.apply(converter)


The problem is that, running basic_preprocess I get the following error message:

/home/santiago/.local/lib/python3.10/site-packages/sklearn/pipeline.py:62: FutureWarning: This Pipeline instance is not fitted yet. Call 'fit' with appropriate arguments before using other methods such as transform, predict, etc. This will raise an error in 1.8 instead of the current warning.
warnings.warn(
/home/santiago/.local/lib/python3.10/site-packages/sklearn/pipeline.py:62: FutureWarning: This Pipeline instance is not fitted yet. Call 'fit' with appropriate arguments before using other methods such as transform, predict, etc. This will raise an error in 1.8 instead of the current warning.
warnings.warn(


This is no longer the case if I use PCA at the end of the pipeline:

pipeline = Pipeline([
("date_converter", DateConverter("trans_date_trans_time")),
("imputer", CustomImputer(strategy="most_frequent")),
("encoding", FrecuencyEncoding()),
("scaler", CustomScaler(df.drop(target, axis=1).columns.tolist())),
("PCA", PCA(n_components=0.999))

])


How to resolve it?

I ran the transformers separately expecting to get some unexpected behavior, but they all worked perfectly.

Continue reading...