python中threading和queue库实现多线程编程
摘要
本文主要介绍了利用python的threading和queue库实现多线程编程，并封装为一个类，方便读者嵌入自己的业务逻辑。最后以机器学习的一个超参数选择为例进行演示。
多线程实现逻辑封装
实例化该类后，在.object_func函数中加入自己的业务逻辑，再调用.run方法即可。
#-*-coding:utf-8-*-
#@Time:2021/2/414:36
#@Author:CyrusMayWJ
#@FileName:run.py
#@Software:PyCharm
#@Blog：https://blog.csdn.net/Cyrus_May
importqueue
importthreading
classCyrusThread(object):
def__init__(self,num_thread=10,logger=None):
"""
:paramnum_thread:线程数
:paramlogger:日志对象
"""
self.num_thread=num_thread
self.logger=logger
defobject_func(self,args_queue,max_q):
while1:
try:
arg=args_queue.get_nowait()
step=args_queue.qsize()
self.logger.info("progress:{}\{}".format(max_q,step))
except:
self.logger.info("nomoreargforargs_queue!")
break
"""
此处加入自己的业务逻辑代码
"""
defrun(self,args):
args_queue=queue.Queue()
forvalueinargs:
args_queue.put(value)
threads=[]
foriinrange(self.num_thread):
threads.append(threading.Thread(target=self.object_func,args=args_queue))
fortinthreads:
t.start()
fortinthreads:
t.join()
模型参数选择实例
#-*-coding:utf-8-*-
#@Time:2021/2/414:36
#@Author:CyrusMayWJ
#@FileName:run.py
#@Software:PyCharm
#@Blog：https://blog.csdn.net/Cyrus_May
importqueue
importthreading
importnumpyasnp
fromsklearn.datasetsimportload_boston
fromsklearn.svmimportSVR
importlogging
importsys
classCyrusThread(object):
def__init__(self,num_thread=10,logger=None):
"""
:paramnum_thread:线程数
:paramlogger:日志对象
"""
self.num_thread=num_thread
self.logger=logger
defobject_func(self,args_queue,max_q):
while1:
try:
arg=args_queue.get_nowait()
step=args_queue.qsize()
self.logger.info("progress:{}\{}".format(max_q,max_q-step))
except:
self.logger.info("nomoreargforargs_queue!")
break
#业务代码
C,epsilon,gamma=arg[0],arg[1],arg[2]
svr_model=SVR(C=C,epsilon=epsilon,gamma=gamma)
x,y=load_boston()["data"],load_boston()["target"]
svr_model.fit(x,y)
self.logger.info("score:{}".format(svr_model.score(x,y)))
defrun(self,args):
args_queue=queue.Queue()
max_q=0
forvalueinargs:
args_queue.put(value)
max_q+=1
threads=[]
foriinrange(self.num_thread):
threads.append(threading.Thread(target=self.object_func,args=(args_queue,max_q)))
fortinthreads:
t.start()
fortinthreads:
t.join()
#创建日志对象
logger=logging.getLogger()
logger.setLevel(logging.INFO)
screen_handler=logging.StreamHandler(sys.stdout)
screen_handler.setLevel(logging.INFO)
formatter=logging.Formatter('%(asctime)s-%(module)s.%(funcName)s:%(lineno)d-%(levelname)s-%(message)s')
screen_handler.setFormatter(formatter)
logger.addHandler(screen_handler)
#创建需要调整参数的集合
args=[]
forCin[iforiinnp.arange(0.01,1,0.01)]:
forepsilonin[iforiinnp.arange(0.001,1,0.01)]+[iforiinrange(1,10,1)]:
forgammain[iforiinnp.arange(0.001,1,0.01)]+[iforiinrange(1,10,1)]:
args.append([C,epsilon,gamma])
#创建多线程工具
threading_tool=CyrusThread(num_thread=20,logger=logger)
threading_tool.run(args)
运行结果
2021-02-0420:52:22,824-run.object_func:31-INFO-progress:1176219\1
2021-02-0420:52:22,824-run.object_func:31-INFO-progress:1176219\2
2021-02-0420:52:22,826-run.object_func:31-INFO-progress:1176219\3
2021-02-0420:52:22,833-run.object_func:31-INFO-progress:1176219\4
2021-02-0420:52:22,837-run.object_func:31-INFO-progress:1176219\5
2021-02-0420:52:22,838-run.object_func:31-INFO-progress:1176219\6
2021-02-0420:52:22,841-run.object_func:31-INFO-progress:1176219\7
2021-02-0420:52:22,862-run.object_func:31-INFO-progress:1176219\8
2021-02-0420:52:22,873-run.object_func:31-INFO-progress:1176219\9
2021-02-0420:52:22,884-run.object_func:31-INFO-progress:1176219\10
2021-02-0420:52:22,885-run.object_func:31-INFO-progress:1176219\11
2021-02-0420:52:22,897-run.object_func:31-INFO-progress:1176219\12
2021-02-0420:52:22,900-run.object_func:31-INFO-progress:1176219\13
2021-02-0420:52:22,904-run.object_func:31-INFO-progress:1176219\14
2021-02-0420:52:22,912-run.object_func:31-INFO-progress:1176219\15
2021-02-0420:52:22,920-run.object_func:31-INFO-progress:1176219\16
2021-02-0420:52:22,920-run.object_func:39-INFO-score:-0.01674283914287855
2021-02-0420:52:22,929-run.object_func:31-INFO-progress:1176219\17
2021-02-0420:52:22,932-run.object_func:39-INFO-score:-0.007992354170952565
2021-02-0420:52:22,932-run.object_func:31-INFO-progress:1176219\18
2021-02-0420:52:22,945-run.object_func:31-INFO-progress:1176219\19
2021-02-0420:52:22,954-run.object_func:31-INFO-progress:1176219\20
2021-02-0420:52:22,978-run.object_func:31-INFO-progress:1176219\21
2021-02-0420:52:22,984-run.object_func:39-INFO-score:-0.018769934807246536
2021-02-0420:52:22,985-run.object_func:31-INFO-progress:1176219\22
到此这篇关于python中threading和queue库实现多线程编程的文章就介绍到这了,更多相关python多线程编程内容请搜索毛票票以前的文章或继续浏览下面的相关文章希望大家以后多多支持毛票票！