[Python] How to improve pandas DF processing time on different combinations of calculated data

Hey let’s say I got big dataset something like 100k or 1mil rows and I got function that make vector calculations that take 0.03 sec now all my columns before the process can be the same for every iteration. I want to calculate 2^n combinations of conditions I make. So currently it will take for me 2^n * 0.03 s to run it all by looping length and run the function. Is there a better way to improve performance and run all this possibilities vectorized or parallel(not python cpu parallel it help by little) the only thing I think of is create unique per iteration column and make regex calculations but then the df will be too big.

In this exmaple where each processing take 0.01ms the output is: Total number of combinations: 1023 Total time to evaluate all combinations: 20.73 seconds

import pandas as pd
import numpy as np
from itertools import combinations
import time

# Generate a larger DataFrame with 100,000 rows
data = {
'Height': np.random.uniform(150, 200, size=100000),
'Weight': np.random.uniform(50, 100, size=100000),
'Gender': np.random.choice(['Male', 'Female'], size=100000),
'Age': np.random.randint(18, 70, size=100000)
}

df = pd.DataFrame(data)

# Define vectorized functions for each condition with dynamic values
def calculate_bmi(height, weight):
height_m = height / 100
return weight / (height_m ** 2)

def condition_bmi(df, min_bmi, max_bmi):
bmi = calculate_bmi(df['Height'], df['Weight'])
return (min_bmi <= bmi) & (bmi <= max_bmi)

def condition_age(df, min_age, max_age):
return (min_age <= df['Age']) & (df['Age'] <= max_age)

def condition_height(df, min_height, max_height):
return (min_height <= df['Height']) & (df['Height'] <= max_height)

def condition_weight(df, min_weight, max_weight):
return (min_weight <= df['Weight']) & (df['Weight'] <= max_weight)

def condition_gender(df, gender):
return df['Gender'] == gender

# List of possible dynamic values for each condition (with only 2 values each)
dynamic_values = {
'BMI is within the normal range': [(18.5, 24.9), (25.0, 29.9)],
'Age is within the healthy range': [(18, 30), (31, 45)],
'Height is within the normal range': [(150, 160), (161, 170)],
'Weight is within the normal range': [(50, 60), (61, 70)],
'Gender is specified': ['Male', 'Female']
}

# Function to create combinations of conditions with dynamic values
def create_condition_combinations(dynamic_values):
condition_combinations = []
for condition_name, values in dynamic_values.items():
if isinstance(values[0], tuple): # For range conditions
for value in values:
condition_combinations.append((condition_name, value))
else: # For categorical conditions
for value in values:
condition_combinations.append((condition_name, value))
return condition_combinations

# Generate all possible combinations of conditions and dynamic values
def generate_all_combinations(condition_combinations):
all_combinations = []
for r in range(1, len(condition_combinations) + 1):
for combo in combinations(condition_combinations, r):
all_combinations.append(combo)
return all_combinations

condition_combinations = create_condition_combinations(dynamic_values)
all_combinations = generate_all_combinations(condition_combinations)

# Calculate the total number of combinations
total_combinations = len(all_combinations)
print(f"Total number of combinations: {total_combinations}")

# Apply a combination of conditions
def evaluate_combination(df, combo):
combined_condition = pd.Series([True] * len(df))
for condition_name, value in combo:
if condition_name == 'BMI is within the normal range':
min_bmi, max_bmi = value
combined_condition &= condition_bmi(df, min_bmi, max_bmi)
elif condition_name == 'Age is within the healthy range':
min_age, max_age = value
combined_condition &= condition_age(df, min_age, max_age)
elif condition_name == 'Height is within the normal range':
min_height, max_height = value
combined_condition &= condition_height(df, min_height, max_height)
elif condition_name == 'Weight is within the normal range':
min_weight, max_weight = value
combined_condition &= condition_weight(df, min_weight, max_weight)
elif condition_name == 'Gender is specified':
gender = value
combined_condition &= condition_gender(df, gender)
return combined_condition

# Measure time to run all combinations
start_time = time.time()

for combo in all_combinations:
combo_start_time = time.time()

evaluate_combination(df, combo)

combo_end_time = time.time()
combo_elapsed_time = combo_end_time - combo_start_time

end_time = time.time()
total_elapsed_time = end_time - start_time

print(f"Total time to evaluate all combinations: {total_elapsed_time:.2f} seconds")

Continue reading...