How to do asynchronous file copying in Python?

Since your problem is a IO bound, I would recommend you to look at threading module. With combination of Queue module you will achieve that.

I agree that a thread that communicates via a Queue is a good solution. Here's an example class:

import os, shutil, threading, Queue

class FileCopy(threading.Thread):
    def __init__(self, queue, files, dirs):
        threading.Thread.__init__(self)
        self.queue = queue
        self.files = list(files)  # copy list
        self.dirs = list(dirs)    # copy list
        for f in files:
            if not os.path.exists(f):
                raise ValueError('%s does not exist' % f)
        for d in dirs:
            if not os.path.isdir(d):
                raise ValueError('%s is not a directory' % d)

    def run(self):
        # This puts one object into the queue for each file,
        # plus a None to indicate completion
        try:
            for f in self.files:
                try:
                    for d in self.dirs:
                        shutil.copy(f, d)
                except IOError, e:
                    self.queue.put(e)
                else:
                    self.queue.put(f)
        finally:
            self.queue.put(None)  # signal completion

Here's an example of how this class can be used:

queue = Queue.Queue()
files = ['a', 'b', 'c']
dirs = ['./x', './y', './z']
copythread = FileCopy(queue, files, dirs)
copythread.start()
while True:
    x = queue.get()
    if x is None:
        break
    print(x)
copythread.join()

The threads in the earlier answer always execute sequentially without any parallelism. With a few changes, one can have the IO bound jobs run together so that the completion of threads depends on the size of files.

I made minor changes to the code in that answer and experimented to verify that the smallest files finish first while the other IO operations are continuing in the background.

import os, shutil, threading, queue

class FileCopy(threading.Thread):
    def __init__(self, queue, files, dirs):
        threading.Thread.__init__(self)
        self.queue = queue
        self.files = list(files)  # copy list
        self.dirs = list(dirs)    # copy list
        for f in files:
            if not os.path.exists(f):
                raise ValueError('%s does not exist' % f)
        for d in dirs:
            if not os.path.isdir(d):
                raise ValueError('%s is not a directory' % d)

    def run(self):
        # This puts one object into the queue for each file
        try:
            for f in self.files:
                try:
                    for d in self.dirs:
                        shutil.copy(f, d)
                except IOError as e:
                    self.queue.put(e)
                else:
                    self.queue.put(f)
        finally:
            pass


queue = queue.Queue()
files = ['a', 'b', 'c']
dirs = ['./x', './y', './z']
thlist = []

for file in files:
    copythread = FileCopy(queue, [file], dirs)
    thlist.append(copythread)

for th in thlist:
    th.start()

for file in files:
     x = queue.get()
     print("Finished copying " + x)

for th in thlist:
    th.join()