[Python] Current sales to prediction

There are columns called Customer Name,Item Number,Year,Quarter,TM Current,Month,Invoice Date,Current_Quarter_Sales,SalesQuota,Hist_SalesQuota,Prev_Quarter_Sales in the dataframe.

I need to find the prediction on the Current_Quarter_Sales column.But i am getting unique values across all the rows:



The model script i am using is :

from pyspark.ml.feature import OneHotEncoder, StringIndexer, VectorAssembler, StandardScaler
from pyspark.ml.regression import LinearRegression
from pyspark.ml import Pipeline
from pyspark.ml.evaluation import RegressionEvaluator
from pyspark.sql.functions import col

final_df = final_df.dropna(subset=["Current_Quarter_Sales"])
print("Final count before transformations:", final_df.count()) # Should be 29

columns_to_cast = ["Prev_Quarter_Sales", "SalesQuota", "Hist_SalesQuota", "Quarter"]
for column in columns_to_cast:
final_df = final_df.withColumn(column, col(column).cast("double"))

final_df = final_df.fillna(0, subset=columns_to_cast)

print("Feature Summary:")
final_df.select("Prev_Quarter_Sales", "SalesQuota", "Hist_SalesQuota", "Quarter").describe().show()

stages = []
distinct_months = final_df.select("Month").distinct().count()
if distinct_months > 1:
month_indexer = StringIndexer(inputCol="Month", outputCol="MonthIndex")
stages.append(month_indexer)

month_encoder = OneHotEncoder(inputCol="MonthIndex", outputCol="MonthVec")
stages.append(month_encoder)
feature_columns = ["Prev_Quarter_Sales", "SalesQuota", "Hist_SalesQuota", "Quarter", "MonthVec"]
else:
feature_columns = ["Prev_Quarter_Sales", "SalesQuota", "Hist_SalesQuota", "Quarter"]

assembler = VectorAssembler(inputCols=feature_columns, outputCol="raw_features")
stages.append(assembler)

scaler = StandardScaler(inputCol="raw_features", outputCol="features", withStd=True, withMean=False)
stages.append(scaler)

lr = LinearRegression(featuresCol="features", labelCol="Current_Quarter_Sales", regParam=0.1, elasticNetParam=0.5)
stages.append(lr)

pipeline = Pipeline(stages=stages)

lr_model = pipeline.fit(final_df)


predictions = lr_model.transform(final_df)
print(f"Number of predictions: {predictions.count()}") # Should be 29


evaluator = RegressionEvaluator(labelCol="Current_Quarter_Sales", predictionCol="prediction", metricName="rmse")
rmse = evaluator.evaluate(predictions)
print(f"Root Mean Squared Error (RMSE): {rmse}")

predictions.select("features", "Current_Quarter_Sales", "prediction").show()

final_df.unpersist()
predictions.unpersist()

Continue reading...