Python实现多线程下载文件的代码实例
实现简单的多线程下载,需要关注如下几点:
1.文件的大小:可以从reponseheader中提取,如“Content-Length:911”表示大小是911字节
2.任务拆分:指定各个线程下载的文件的哪一块,可以通过requestheader中添加“Range:bytes=300-400”(表示下载300~400byte的内容),注意可以请求的文件的range是[0,size-1]字节的。
3.下载文件的聚合:各个线程将自己下载的文件块保存为临时文件,所有线程都完成后,再将这些临时文件按顺序聚合写入到最终的一个文件中。
实现代码:
#!/usr/bin/python #-*-coding:utf-8-*- #filename:paxel.py #FROM:http://jb51.net/code/view/58/full/ #Jaymodifieditalittleandsaveforfurtherpotentialusage. '''Itisamulti-threaddownloadingtool Itwasdevelopedfollowingaxel. Author:volans E-mail:volansw[at]gmail.com ''' importsys importos importtime importurllib fromthreadingimportThread #incaseyouwanttousehttp_proxy local_proxies={'http':'http://131.139.58.200:8080'} classAxelPython(Thread,urllib.FancyURLopener): '''Multi-threaddownloadingclass. run()isavituralmethodofThread. ''' def__init__(self,threadname,url,filename,ranges=0,proxies={}): Thread.__init__(self,name=threadname) urllib.FancyURLopener.__init__(self,proxies) self.name=threadname self.url=url self.filename=filename self.ranges=ranges self.downloaded=0 defrun(self): '''vertualfunctioninThread''' try: self.downloaded=os.path.getsize(self.filename) exceptOSError: #print'neverdownloaded' self.downloaded=0 #rebuildstartpoind self.startpoint=self.ranges[0]+self.downloaded #Thispartiscompleted ifself.startpoint>=self.ranges[1]: print'Part%shasbeendownloadedover.'%self.filename return self.oneTimeSize=16384 #16kByte/time print'task%swilldownloadfrom%dto%d'%(self.name,self.startpoint,self.ranges[1]) self.addheader("Range","bytes=%d-%d"%(self.startpoint,self.ranges[1])) self.urlhandle=self.open(self.url) data=self.urlhandle.read(self.oneTimeSize) whiledata: filehandle=open(self.filename,'ab+') filehandle.write(data) filehandle.close() self.downloaded+=len(data) #print"%s"%(self.name) #progress=u'\r...' data=self.urlhandle.read(self.oneTimeSize) defGetUrlFileSize(url,proxies={}): urlHandler=urllib.urlopen(url,proxies=proxies) headers=urlHandler.info().headers length=0 forheaderinheaders: ifheader.find('Length')!=-1: length=header.split(':')[-1].strip() length=int(length) returnlength defSpliteBlocks(totalsize,blocknumber): blocksize=totalsize/blocknumber ranges=[] foriinrange(0,blocknumber-1): ranges.append((i*blocksize,i*blocksize+blocksize-1)) ranges.append((blocksize*(blocknumber-1),totalsize-1)) returnranges defislive(tasks): fortaskintasks: iftask.isAlive(): returnTrue returnFalse defpaxel(url,output,blocks=6,proxies=local_proxies): '''paxel ''' size=GetUrlFileSize(url,proxies) ranges=SpliteBlocks(size,blocks) threadname=["thread_%d"%iforiinrange(0,blocks)] filename=["tmpfile_%d"%iforiinrange(0,blocks)] tasks=[] foriinrange(0,blocks): task=AxelPython(threadname[i],url,filename[i],ranges[i]) task.setDaemon(True) task.start() tasks.append(task) time.sleep(2) whileislive(tasks): downloaded=sum([task.downloadedfortaskintasks]) process=downloaded/float(size)*100 show=u'\rFilesize:%dDownloaded:%dCompleted:%.2f%%'%(size,downloaded,process) sys.stdout.write(show) sys.stdout.flush() time.sleep(0.5) filehandle=open(output,'wb+') foriinfilename: f=open(i,'rb') filehandle.write(f.read()) f.close() try: os.remove(i) pass except: pass filehandle.close() if__name__=='__main__': url='http://dldir1.qq.com/qqfile/QQforMac/QQ_V3.1.1.dmg' output='download.file' paxel(url,output,blocks=4,proxies={})