[Python] Why is Rust Polars slower than Python Polars on TPCH dataset (20GB Parquet)?

I’m experimenting with Polars (both Python and Rust) to run TPCH queries on large Parquet datasets (~10GB, multiple files). Surprisingly, I’m observing that Python Polars is faster than Rust Polars, even though I expected Rust to be faster.

Dataset

Table: lineitem (from TPCH benchmark)

Size: ~20GB split into multiple Parquet files like:

F:\test_data\lineitem\lineitem_0_10000000.parquet

# tpch_generate_partitioned.py
import duckdb
import os
import time
import math

# ========== Configuration ==========
scale_factor = 30 # TPC-H scale factor
children = 20 # Number of steps to split the data generation
output_dir = r"F:\test_data" # Root output directory (e.g., F:\test_data\lineitem\...)
db_path = os.path.join(output_dir, "tpch.duckdb") # Persistent DuckDB file
temp_dir = r"F:\duckdb_tmp" # Temporary directory for DuckDB spill (ensure sufficient disk space)
tables = ['customer', 'lineitem', 'orders', 'partsupp', 'part', 'supplier', 'nation', 'region']
# ===================================

os.makedirs(output_dir, exist_ok=True)
os.makedirs(temp_dir, exist_ok=True)

conn = duckdb.connect(database=db_path, read_only=False)

try:
temp_dir_sql = temp_dir.replace('\\', '/')
conn.execute(f"SET temp_directory = '{temp_dir_sql}'")
conn.execute("SET preserve_insertion_order = false")

try:
conn.execute("INSTALL tpch;")
except Exception as e:
print("INSTALL tpch may already be installed or network unavailable (ignored):", e)
conn.execute("LOAD tpch;")

print(f"Preparing to generate TPC-H SF={scale_factor} in {children} steps, database: {db_path}")
cumulative = {t: 0 for t in tables}

for step in range(children):
suffix = f"_part{step}"
print(f"\n=== Step {step+1}/{children}: CALL dbgen(sf={scale_factor}, children={children}, step={step}, suffix='{suffix}') ===")
t0 = time.time()
conn.execute(f"CALL dbgen(sf={scale_factor}, children={children}, step={step}, suffix='{suffix}');")
dt = time.time() - t0
print(f" Generation completed (took {dt:.1f}s), exporting data to Parquet and releasing tables...")

for table in tables:
part_table = f"{table}{suffix}"
exists = conn.execute(f"SELECT COUNT(*) FROM information_schema.tables WHERE table_name = '{part_table}';").fetchone()[0]
if exists == 0:
try:
cnt = conn.execute(f"SELECT COUNT(*) FROM {part_table};").fetchone()[0]
except Exception:
cnt = 0
else:
cnt = conn.execute(f"SELECT COUNT(*) FROM {part_table};").fetchone()[0]

if cnt == 0:
print(f" Table {part_table} has no data, skipping")
try:
conn.execute(f"DROP TABLE IF EXISTS {part_table};")
except Exception:
pass
continue

start = cumulative[table]
end = start + cnt - 1
table_dir = os.path.join(output_dir, table)
os.makedirs(table_dir, exist_ok=True)
out_fname = f"{table}_{start}_{end}.parquet"
out_path = os.path.join(table_dir, out_fname)
out_path_sql = out_path.replace("\\", "/")

print(f" Exporting {part_table} ({cnt:,} rows) -> {out_path_sql} ...", end="", flush=True)
t1 = time.time()
conn.execute(f"COPY {part_table} TO '{out_path_sql}' (FORMAT PARQUET);")
print(f" Done ({time.time() - t1:.1f}s)")

cumulative[table] += cnt
conn.execute(f"DROP TABLE IF EXISTS {part_table};")

print(f"Step {step+1} completed, total time: {time.time() - t0:.1f}s")

print("\nAll steps completed. Final row counts per table:")
for table in tables:
print(f" {table}: {cumulative[table]:,} rows")

print("\nTask completed. Parquet files saved to:", output_dir)

finally:
conn.close()


CODE

import polars as pl
import time

file_path = r"F:\test_data\lineitem\*.parquet"

print("Starting group-by aggregation analysis (Python Polars)...")
start = time.time()

df = pl.scan_parquet(file_path)

result = (
df.group_by("l_returnflag", "l_linestatus")
.agg([
pl.col("l_quantity").sum().alias("sum_qty"),
pl.col("l_extendedprice").sum().alias("sum_base_price"),
pl.col("l_extendedprice").sum().alias("sum_disc_price"), # modify if discount is applied
pl.col("l_extendedprice").mean().alias("avg_price"),
pl.col("l_quantity").mean().alias("avg_qty"),
pl.col("l_discount").mean().alias("avg_disc"),
pl.count().alias("count_order"),
])
.collect()
)

print(result)
print(f"Total execution time: {time.time() - start:.2f}s")


CODE RUST

use polars:relude::*;
use std::time::Instant;

fn main() -> PolarsResult<()> {
std::env::set_var("POLARS_MAX_THREADS", "32"); // Set thread count based on CPU

let file_path = r"F:\test_data\lineitem\*.parquet";

println!("Starting group-by aggregation analysis (Rust Polars)...");
let start = Instant::now();

let args = ScanArgsParquet {
cache: true,
parallel: ParallelStrategy::Auto,
rechunk: false,
low_memory: false,
use_statistics: true,
..Default::default()
};

let df = LazyFrame::scan_parquet(file_path, args)?;

let grouped = df
.group_by_stable([col("l_returnflag"), col("l_linestatus")])
.agg([
col("l_quantity").sum().alias("sum_qty"),
col("l_extendedprice").sum().alias("sum_base_price"),
col("l_extendedprice").mean().alias("avg_price"),
col("l_quantity").mean().alias("avg_qty"),
col("l_discount").mean().alias("avg_disc"),
col("l_extendedprice").count().alias("count_order"),
])
.collect()?;

println!("{:?}", grouped);
println!("Total execution time: {:?}", start.elapsed());

Ok(())
}


TIME

PYTHON Total execution time: 6.72s

RUST Total execution time: 10.6621531s

PY

RUST

This way, the question is:

Clear about dataset and workload.

Shows minimal reproducible code in both Rust and Python.

Explains what I expected vs. what I observed.

Asks why and how to optimize.

Continue reading...