[Python] Multiprocessing and sourcing shell file for every subprocess

I am working on a code which aims to gather simulation commands and multiprocess them, sourcing a shell file for each subprocess before running a simulation command in the subprocess. For this I gather the commands in another function in a dictionary which is used by the functions below:

def source_shell_script(self, script_path: str) -> dict:
"""
Sources a shell script and updates the environment for each subprocess.
"""
if not os.path.exists(script_path):
raise FileNotFoundError(f"Script not found: {script_path}")

# source the script and output environment variables
command = f"bash -c 'source {script_path} && env'"

try:
# run the command and capture the output
result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, text=True, check=True)
# parse the environment variables from the command output
env_vars = dict(
line.split("=", 1) for line in result.stdout.splitlines() if "=" in line
)
return env_vars
except subprocess.CalledProcessError as e:
raise RuntimeError(f"Failed to source script: {script_path}. Error: {e}")


def run_cmd(self, cmd_px: tuple) -> None:
cmd, px_src_path = cmd_px
try:
# get environment variables
env_vars = self.source_shell_script(px_src_path)
print(f"Executing command: {cmd}")
result = subprocess.run(cmd, shell=True, env=env_vars, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print(f"Output: {result.stdout}")
if result.returncode != 0:
print(f"Error: {result.stderr}")
except Exception as e:
print(f"Failed to execute command: {cmd}. Error: {e}")

def exec_sim(self) -> None:
"""
Execute all simulations in parallel using multiprocessing.
"""
# create a list of (command, px_src_path) tuples for each pixel configuration
run_queue = [(cmd, self.sim_dict[px_key]["px_src_path"])
for px_key in self.sim_dict
for cmd in self.sim_dict[px_key]["px_ddsim_cmds"]]

num_workers = os.cpu_count() # number of processes
with multiprocessing.Pool(num_workers) as pool:
pool.map(self.run_cmd, run_queue)


if __name__ == "__main__":
# initialize program
eic_object = HandleEIC()
eic_object.init_path_var()
pixel_sizes = eic_object.setup_json()
eic_object.pixel_sizes = pixel_sizes
os.chmod(eic_object.execution_path, 0o777)

eic_object.setup_sim()
print("Simulation dictionary:", eic_object.sim_dict)

eic_object.exec_sim()

# create backup for simulation
eic_object.mk_sim_backup()


Instead of running properly, my program gets stuck and the console prints the commands:

Console output:

Simulation dictionary: {'2.0_0.1': {'px_epic_path': '/data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/epic', 'px_compact_path': '/data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/epic/install/share/epic/compact', 'px_ip6_path': '/data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/epic/install/share/epic/epic_ip6_extended.xml', 'px_src_path': '/data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/epic/install/bin/thisepic.sh', 'px_out_path': '/data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px', 'px_ddsim_cmds': ['ddsim --inputFiles /data/user/Analysis_epic_new/simulations/genEvents/results/beamEffectsElectrons_20.hepmc --outputFile /data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/output_20edm4hep.root --compactFile /data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/epic/install/share/epic/epic_ip6_extended.xml -N 5']}, '0.1_0.1': {'px_epic_path': '/data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/epic', 'px_compact_path': '/data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/epic/install/share/epic/compact', 'px_ip6_path': '/data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/epic/install/share/epic/epic_ip6_extended.xml', 'px_src_path': '/data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/epic/install/bin/thisepic.sh', 'px_out_path': '/data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px', 'px_ddsim_cmds': ['ddsim --inputFiles /data/user/Analysis_epic_new/simulations/genEvents/results/beamEffectsElectrons_20.hepmc --outputFile /data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/output_20edm4hep.root --compactFile /data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/epic/install/share/epic/epic_ip6_extended.xml -N 5']}}
Executing command: ddsim --inputFiles /data/user/Analysis_epic_new/simulations/genEvents/results/beamEffectsElectrons_20.hepmc --outputFile /data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/output_20edm4hep.root --compactFile /data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/epic/install/share/epic/epic_ip6_extended.xml -N 5
Executing command: ddsim --inputFiles /data/user/Analysis_epic_new/simulations/genEvents/results/beamEffectsElectrons_20.hepmc --outputFile /data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/output_20edm4hep.root --compactFile /data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/epic/install/share/epic/epic_ip6_extended.xml -N 5


Why is my code not running as expected? What am I doing wrong?

Continue reading...