I am applying Multi-threading to a python script to improve its performance. I don't understand why there is no improvement in the execution time.
This is the code snippet of my implementation:
from queue import Queue
from threading import Thread
from datetime import datetime
import time
class WP_TITLE_DOWNLOADER(Thread):
def __init__(self, queue,name):
Thread.__init__(self)
self.queue = queue
self.name = name
def download_link(self,linkss):
####some test function
###later some processing will be done on this list.
#####this will be processed on CPU.
for idx,link in enumerate(linkss):
##time.sleep(0.01)
test.append(idx)
for idx,i in enumerate(testv):
i=i.append(2)
##
def run(self):
while True:
# Get the work from the queue
linkss = self.queue.get()
try:
self.download_link(linkss)
finally:
self.queue.task_done()
######with threading
testv=[[i for i in range(5000)] for j in range(20)]
links_list=[[i for i in range(100000)] for j in range(20)]
test=[]
start_time =time.time()
queue = Queue()
thread_count=8
for x in range(thread_count):
worker = WP_TITLE_DOWNLOADER(queue,str(x))
# Setting daemon to True will let the main thread exit even though the workers are blocking
worker.daemon = True
worker.start()
##FILL UP Queue for threads
for links in links_list:
queue.put(links)
##print("queing time={}".format(time.time()-start_time))
#print(test)
#wait for all to end
j_time =time.time()
queue.join()
t_time = time.time()-start_time
print("With threading time={}".format(t_time))
#############without threading,
###following function is same as the one in threading.
test=[]
def download_link(links1):
for idx,link in enumerate(links1):
##time.sleep(0.01)
test.append(idx)
for idx,i in enumerate(testv):
i=i.append(2)
start_time =time.time()
for links in links_list:
download_link(links)
t_time = time.time()-start_time
print("without threading time={}".format(t_time))
With threading time=0.564049482345581 without threading time=0.13332700729370117
NOTE: When I uncomment time.sleep, with threading time is lower than without threading. My test case is I have a list of lists, each list has more than 10000s elements, the idea of using multi-threading is that instead of processing a single list item, multiple lists can be processed simultaneously, resulting in a decrease in overall time. But the results are not as expected.
