[Python] Combine graphlib TopologicalSorter and multiprocessing Process in python

I'm trying to run tasks coming from a topological sorter in parallel, but I fail. so far I was able to write something like this, I hope it is representative enough:

from graphlib import TopologicalSorter
from multiprocessing import Queue, Process


def run_task_in_parallel(
task: Callable,
args: tuple | None = None,
number_of_processes: int = 1,
) -> None:
args = args or ()
processes = []
for _ in range(number_of_processes):
process = Process(target=task, args=args)
processes.append(process)
for process in processes:
process.start()
for process in processes:
process.join()


def run_step(
task_queue: Queue,
completed_task_queue: Queue,
) -> None:
while True:
step = task_queue.get()
run_steps([step])
completed_task_queue.put(step)

if __name__ == "__main__":
ts = TopologicalSorter(graph)
ts.prepare()
task_queue = Queue()
completed_task_queue = Queue()
run_task_in_parallel(
run_step,
args=(task_queue, completed_task_queue),
number_of_processes=4,
)
while ts.is_active():
for node in ts.get_ready():
print(f"Starting: {node}")
task_queue.put(node)
node = completed_task_queue.get()
print(f"Completed: {node}")
ts.done(node)


It seems it gets stuck right after running run_task_in_parallel before starting the loop on ts

If on the other hand I remove the .join() operation on the processes, it procceeds further but then it gets stuck again

Continue reading...