Python中运行并行任务技巧
示例
标准线程多进程，生产者/消费者示例：
Worker越多,问题越大
#-*-coding:utf8-*-
importos importtime importQueue importthreading fromPILimportImage
defcreate_thumbnail(filename,size=(128,128)): try: fp,fmt=filename.rsplit('.',1) im=Image.open(filename) im.thumbnail(size,Image.ANTIALIAS) im.save((fp+'_'+'x'.join(str(i)foriinsize)+'.'+fmt),im.format) return'%sthumbnailsuccess!'%filename exceptException: return'%sthumbnailfailed!'%filename
defget_image_paths(folder): return[os.path.join(folder,f)forfinos.listdir(folder)if'png'inf]
classConsumer(threading.Thread): def__init__(self,queue): threading.Thread.__init__(self) self._queue=queue
defrun(self): whileTrue: content=self._queue.get() ifisinstance(content,str)andcontent=='quit': break respone=create_thumbnail(content) print'Byebye!'
defProducer(): filenames=get_image_paths('images') queue=Queue.Queue() worker_threads=build_worker_pool(queue,4) start_time=time.time()
forfilenameinfilenames: queue.put(filename) forworkerinworker_threads: queue.put('quit') forworkerinworker_threads: worker.join()
printtime.time()-start_time
defbuild_worker_pool(queue,size): workers=[] for_inrange(size): worker=Consumer(queue) worker.start() workers.append(worker) returnworkers
if__name__=='__main__': Producer()