[Python] Terminating pool early intermittently crashes when return size is variable

I have a script that's trying to analyse some images in parellel. For some reason, I get intermittent crashes if the func passed to the pool returns variable sized data. This is only if I try to exit early and call pool.terminate(), so clearly I'm missing some clean up.

Tried googling if that's intentional with multiprocessing pools but couldn't find anything.

import glob, multiprocessing, time, cv2
import numpy as np
from PIL import Image
from signal import signal, SIGINT

def pool_initializer():
signal(SIGINT, lambda signum, frame: print(signum, frame))

def preview_analyse(preview):
time_start = time.perf_counter()

with Image.open(preview) as pil_image:
data_raw = np.asarray(pil_image).reshape(pil_image.size[0], pil_image.size[1], 4)
data = np.swapaxes(data_raw, 0,1)

w1 = cv2.inRange(data, np.asarray((38, 64, 73, 255)), np.asarray((38, 64, 73, 255)))

## cc_stats size varies, it's a (int, MatLike, MatLike, MatLike), MatLikes are 2d arrays of varying rows
cc_stats = cv2.connectedComponentsWithStats(cv2.bitwise_not(w1), 4, cv2.CV_32S)

## This intermittently crashes, variable size output
return (cc_stats, time.perf_counter() - time_start)
## This is fine, fixed size output
# return (cc_stats[3][:10], time.perf_counter() - time_start)

if __name__ == "__main__":
previews = glob.glob("*.png")

with multiprocessing.Pool(processes=int(multiprocessing.cpu_count()/2), initializer=pool_initializer) as pool:
for result, preview in zip(pool.imap(preview_analyse, previews[:10]), previews):
## Not exiting early results in zero crashes regardless of which return is chosen
if True:
break

## Intermittently crashes in pool.terminate()
pool.terminate()
pool.join()

print("Great success!")


Traceback (most recent call last):
File "H:\wouldntyouliketoknowwaterboy\test.py", line 36, in <module>
pool.terminate()
~~~~~~~~~~~~~~^^
File "C:\Users\wouldntyouliketoknowwaterboy\AppData\Local\Programs\Python\Python313\Lib\multiprocessing\pool.py", line 657, in terminate
self._terminate()
~~~~~~~~~~~~~~~^^
File "C:\Users\wouldntyouliketoknowwaterboy\AppData\Local\Programs\Python\Python313\Lib\multiprocessing\util.py", line 216, in __call__
res = self._callback(*self._args, **self._kwargs)
File "C:\Users\wouldntyouliketoknowwaterboy\AppData\Local\Programs\Python\Python313\Lib\multiprocessing\pool.py", line 703, in _terminate_pool
outqueue.put(None) # sentinel
~~~~~~~~~~~~^^^^^^
File "C:\Users\wouldntyouliketoknowwaterboy\AppData\Local\Programs\Python\Python313\Lib\multiprocessing\queues.py", line 394, in put
self._writer.send_bytes(obj)
~~~~~~~~~~~~~~~~~~~~~~~^^^^^
File "C:\Users\wouldntyouliketoknowwaterboy\AppData\Local\Programs\Python\Python313\Lib\multiprocessing\connection.py", line 200, in send_bytes
self._send_bytes(m[offsetffset + size])
~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\wouldntyouliketoknowwaterboy\AppData\Local\Programs\Python\Python313\Lib\multiprocessing\connection.py", line 287, in _send_bytes
raise ValueError("concurrent send_bytes() calls "
"are not supported")
ValueError: concurrent send_bytes() calls are not supported


I could just filter the output and call it a day but then I would be left ignorant. Can someone explain what is happening here?

Continue reading...