[Python] AWS Glue/spark performace inssue

I am new to aws glue and I am facing performance issues with the below code

spark.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN")

# Define S3 path with wildcard to match all .ind files
s3_ind_path = f"s3a://{source_bucket}/{source_prefix}/*.ind"

# Read all .ind files as whole text
whole_df = spark.sparkContext.wholeTextFiles(s3_ind_path).toDF(["path", "content"])

# Step 1: Split the raw content into key-value string array
whole_df = whole_df.withColumn("kv_pairs", split(col("content"), "\\|"))

# Step 2: Convert array of key:value strings into a map
whole_df = whole_df.withColumn(
"fields_map",
map_from_entries(
expr("transform(kv_pairs, x -> struct(split(x, ':', 2)[0] as key, split(x, ':', 2)[1] as value))")
)
)
# Step 3: Extract multiple fields from the map
fields_to_extract = [
"SSN_TIN", "TIDTYPE", "FNAME", "MNAME", "LNAME", "ENTNAME", "BRK_ACCT",
"DIR_ACCT", "NONBRKAC", "SPONSOR", "REP_ID", "RR2", "REG_TY", "LOB",
"PROC_DTE", "DT_REC", "SCANDTS", "XTRACID", "NASU_ID", "DC_SRCRF",
"CHK_AMT", "CHK_NUML", "DEPOSDT", "RCVDDATE", "STK_CNUM", "STK_SHR", "DOC-TY-ID"
]
for field in fields_to_extract:
whole_df = whole_df.withColumn(field, col("fields_map").getItem(field))
# Optional: Drop intermediate columns if not needed
whole_df = whole_df.drop("kv_pairs", "fields_map","content")


client_df=whole_df.select("SSN_TIN", "TIDTYPE", "fname","lname","entname")


client_df.cache()
print(f"Total rows in client_df: {client_df.count()}")


print("converting clients to dynamic frame")
client_dynamic_frame = DynamicFrame.fromDF(client_df, glueContext, "dynamic_frame")
print("Inserting clients")

glueContext.write_dynamic_frame.from_options(
frame=client_dynamic_frame,
connection_type="JDBC",
connection_options={
"connectionName": connection_name,
"database": tgt_database,
"dbtable": "staging.bpm_migration_client1",
"useConnectionProperties": "true"
}
)

print("client insert complete")


I reading 100000 thousand files from an s3 bucket and write the content to a postgres db. This is taking more than an hour to complete. Is there any way I can make it faster? Any help will be appreciated.

Continue reading...