[Python] How to open a nested MatLab data in Python?

I have MatLab data and am trying to open it in Python data frame to conduct some statiscitcs. However RT variable (column) includes 60ish values, and I want them to be seperate: trial1, trial2... etc like this so I can conduct some descriptive statistics. In the current form, it is like one variable with lots of values inside so it does not make sense. How can I change this structure to a regular data frame, is there any other way to do it?

import os
import sys
import re
import warnings

# Attempt to import necessary libraries.
try:
from scipy.io import loadmat
import pandas as pd
import numpy as np
except ImportError as e:
print(f"Error: A required library is not installed: {e.name}")
print(f"Please install it using 'pip install {e.name}'")
sys.exit(1)

def extract_metadata_from_filename(filename):
"""
Parses a filename to extract subject, session, block, and stimulation information.
Example filename: Suj1_B1_S1_Stim1_2024-01-09_15h14m06s.mat

Args:
filename (str): The name of the file.

Returns:
A dictionary with extracted metadata or None if parsing fails.
"""
# Updated regex pattern to find SujX, Bx, Sx, and optionally StimX
# The '(?:_Stim(\d+))?' part makes the stimulation group optional.
match = re.search(r'Suj(\d+)_B(\d+)_S(\d+)(?:_Stim(\d+))?', filename)
if match:
subject, block, session, stim = match.groups()
return {
'Subject': f"Subj{subject}",
'Block': f"B{block}",
'Session': f"S{session}",
'Stim': f"Stim{stim}" if stim else None # Assign None if no stimulation found
}
return None

def process_mat_data(filepath):
"""
Loads a .mat file, extracts 'rt' data from 'trials', and returns
a list of dictionaries, each ready to be added to a DataFrame.

Args:
filepath (str): The full path to the .mat file.

Returns:
A list of dictionaries with RT data and metadata.
"""
# Extract metadata from the filename
filename = os.path.basename(filepath)
metadata = extract_metadata_from_filename(filename)

if not metadata:
print(f"Warning: Could not parse metadata from filename: {filename}")
return []

print(f"Processing file: {filename}")
print(f" Extracted metadata: {metadata}")

try:
# Load the .mat file without squeezing to preserve the original structure.
mat_data = loadmat(filepath, squeeze_me=False)

rt_records = []

# Check for the top-level 'trials' key
if 'trials' in mat_data:
# The 'trials' variable is a nested structure. Access its contents
trials_object = mat_data['trials'][0, 0]

# Now, access the fields within that structured object.
# The RT data is nested within an array inside the structured array.
if 'rt' in trials_object.dtype.names:
# Access the array of RT values
rt_values_array = trials_object['rt']

# Check if the data is what we expect (a structured array with a single field)
if isinstance(rt_values_array, np.ndarray) and rt_values_array.dtype.names:
# Access the first field's data and flatten it
rt_values = rt_values_array[rt_values_array.dtype.names[0]].flatten()
elif isinstance(rt_values_array, np.ndarray):
# For a simple NumPy array, just flatten it
rt_values = rt_values_array.flatten()
else:
print(f"Warning: The 'rt' data for file {filename} is not a valid array structure.")
return []

# Iterate through the flattened RT values and create a record for each one
for rt_value in rt_values:
record = metadata.copy()
record['RT'] = rt_value
rt_records.append(record)
else:
print(f"Warning: 'rt' field not found in 'trials' for file: {filename}")
return []

# Check the number of trials processed as a sanity check
num_trials = len(rt_records)
if num_trials != 64:
warnings.warn(f"Expected 64 trials but found {num_trials} in file: {filename}")

return rt_records

else:
print(f"Warning: 'trials' variable not found in file: {filename}")
return []

except Exception as e:
print(f"An error occurred while trying to read the file {filename}: {e}")
return []

def main():
"""
Main function to traverse the directory structure, process the files,
and perform descriptive statistics.
"""
# Define the root directory where your 'raw_data' folder is located.
# IMPORTANT: Change this path to match the location on your system.
data_root_dir = './raw_data'

if not os.path.exists(data_root_dir):
print(f"Error: The directory '{data_root_dir}' was not found.")
print("Please create this folder or change the 'data_root_dir' variable to the correct path.")
sys.exit(1)

print(f"Starting to traverse directory: {os.path.abspath(data_root_dir)}")
print("This script will extract RT data and perform descriptive statistics.")
print("\n" + "=" * 50 + "\n")

# List to hold all the data records before creating the DataFrame
all_rt_data = []

# Use os.walk to recursively traverse the directory tree.
for root, dirs, files in os.walk(data_root_dir):
files.sort()
for filename in files:
if filename.endswith('.mat'):
filepath = os.path.join(root, filename)
rt_records = process_mat_data(filepath)
all_rt_data.extend(rt_records)

# Create the pandas DataFrame from the collected data
if all_rt_data:
df = pd.DataFrame(all_rt_data)

print("\n" + "=" * 50 + "\n")
print("DataFrame created successfully. Here is a sample:")
print(df.head())
print(f"\nTotal records processed: {len(df)}")

# Perform descriptive statistics on 'RT' grouped by Subject, Session, Block, and Stim
print("\n" + "=" * 50 + "\n")
print("Descriptive Statistics for 'RT' (grouped by Subject, Session, Block, and Stim):")

# Use .groupby() to perform the grouping and .describe() to get the statistics
stats = df.groupby(['Subject', 'Session', 'Block', 'Stim'])['RT'].describe()
print(stats)
else:
print("No valid 'rt' data was found to create a DataFrame.")

if __name__ == "__main__":
main()

Continue reading...