Given the following program structure I am looking for a way to:
Detect child failures in the parent process
Terminate all processes (parent and children) and exit with status code 1
Achieve the above without relaxing the max queue size
Here's the code:
import multiprocessing
def worker1(queue1, queue2):
while True:
item = queue1.get()
if item == 10:
raise
if item == 'stop':
return
# do something with item that generates multiple queue entries
for x in xrange(1000):
queue2.put(x)
def worker2(queue2, queue3):
while True:
item = queue2.get()
if item == 'stop':
return
# do something with item, only one result to pass to the next queue
queue3.put(1)
def worker3(queue3):
while True:
item = queue3.get()
if item == 'stop':
return
# do something with item
def main():
queue1 = multiprocessing.Queue()
queue2 = multiprocessing.Queue(100)
queue3 = multiprocessing.Queue(100)
pool1 = multiprocessing.Pool(2, worker1, (queue1,queue2,))
pool2 = multiprocessing.Pool(8, worker2, (queue2,queue3,))
pool3 = multiprocessing.Pool(4, worker3, (queue3,))
for i in xrange(100):
queue1.put(i)
for _ in range(pool1.__dict__['_processes']):
queue1.put('stop')
pool1.close()
pool1.join()
for _ in range(pool2.__dict__['_processes']):
queue2.put('stop')
pool2.close()
pool2.join()
for _ in range(pool3.__dict__['_processes']):
queue3.put('stop')
pool3.close()
pool3.join()
print 'finished'
if __name__ == '__main__':
main()
Thanks!
try: ... except: raise ~exception type here~