java线程池实现批量下载文件
本文实例为大家分享了java线程池实现批量下载文件的具体代码,供大家参考,具体内容如下
1创建线程池
packagecom.cheng.webb.thread; importjava.util.concurrent.ArrayBlockingQueue; importjava.util.concurrent.ExecutorService; importjava.util.concurrent.ThreadFactory; importjava.util.concurrent.ThreadPoolExecutor; importjava.util.concurrent.TimeUnit; importjava.util.concurrent.atomic.AtomicInteger; publicclassThreadUtil{ /** *创建批量下载线程池 * *@paramthreadSize下载线程数 *@returnExecutorService */ publicstaticExecutorServicebuildDownloadBatchThreadPool(intthreadSize){ intkeepAlive=0; Stringprefix="download-batch"; ThreadFactoryfactory=ThreadUtil.buildThreadFactory(prefix); returnnewThreadPoolExecutor(threadSize, threadSize, keepAlive, TimeUnit.SECONDS, newArrayBlockingQueue<>(threadSize), factory); } /** *创建自定义线程工厂 * *@paramprefix名称前缀 *@returnThreadFactory */ publicstaticThreadFactorybuildThreadFactory(Stringprefix){ returnnewCustomThreadFactory(prefix); } /** *自定义线程工厂 */ publicstaticclassCustomThreadFactoryimplementsThreadFactory{ privateStringthreadNamePrefix; privateAtomicIntegercounter=newAtomicInteger(1); /** *自定义线程工厂 * *@paramthreadNamePrefix工厂名称前缀 */ CustomThreadFactory(StringthreadNamePrefix){ this.threadNamePrefix=threadNamePrefix; } @Override publicThreadnewThread(Runnabler){ StringthreadName=threadNamePrefix+"-t"+counter.getAndIncrement(); returnnewThread(r,threadName); } } }
2批量下载文件
packagecom.cheng.webb.thread; importorg.slf4j.Logger; importorg.slf4j.LoggerFactory; importjava.io.File; importjava.io.FileOutputStream; importjava.io.InputStream; importjava.net.HttpURLConnection; importjava.net.URL; importjava.util.ArrayList; importjava.util.List; importjava.util.Map; importjava.util.concurrent.*; /** *文件下载类 * *@authorshucheng *@creation2019年1月30日下午4:41:32 */ publicclassDownloadUtil{ privatestaticLoggerlogger=LoggerFactory.getLogger(DownloadUtil.class); /** *下载线程数 */ privatestaticfinalintDOWNLOAD_THREAD_NUM=14; /** *下载线程池 */ privatestaticExecutorServicedownloadExecutorService=ThreadUtil .buildDownloadBatchThreadPool(DOWNLOAD_THREAD_NUM); /** *文件下载 * *@paramfileUrl *文件url,如:https://img3.doubanio.com//view//photo//s_ratio_poster//public//p2369390663.webp
*@parampath *存放路径,如:/opt/img/douban/my.webp */ publicstaticvoiddownload(StringfileUrl,Stringpath){ //判断存储文件夹是否已经存在或者创建成功 if(!createFolderIfNotExists(path)){ logger.error("Wecan'tcreatefolder:{}",getFolder(path)); return; } InputStreamin=null; FileOutputStreamout=null; try{ URLurl=newURL(fileUrl); HttpURLConnectionconn=(HttpURLConnection)url.openConnection(); conn.setRequestMethod("GET"); //2s conn.setConnectTimeout(10000); in=conn.getInputStream(); out=newFileOutputStream(path); intlen; byte[]arr=newbyte[1024*1000]; while(-1!=(len=in.read(arr))){ out.write(arr,0,len); } out.flush(); conn.disconnect(); }catch(Exceptione){ logger.error("Failtodownload:{}by{}",fileUrl,e.getMessage()); }finally{ try{ if(null!=out){ out.close(); } if(null!=in){ in.close(); } }catch(Exceptione){ //donothing } } } /** *创建文件夹,如果文件夹已经存在或者创建成功返回true * *@parampath *路径 *@returnboolean */ privatestaticbooleancreateFolderIfNotExists(Stringpath){ StringfolderName=getFolder(path); if(folderName.equals(path)){ returntrue; } Filefolder=newFile(getFolder(path)); if(!folder.exists()){ synchronized(DownloadUtil.class){ if(!folder.exists()){ returnfolder.mkdirs(); } } } returntrue; } /** *获取文件夹 * *@parampath *文件路径 *@returnString */ privatestaticStringgetFolder(Stringpath){ intindex=path.lastIndexOf("/"); return-1!=index?path.substring(0,index):path; } /** *下载资源 **issue:线程池创建过多 *
*最大批量下载为5,请知悉 * *@paramresourceMap *资源map,key为资源下载url,value为资源存储位置 */ publicstaticvoidbatch(Map
resourceMap){ if(resourceMap==null||resourceMap.isEmpty()){ return; } try{ List keys=newArrayList<>(resourceMap.keySet()); intsize=keys.size(); intpageNum=getPageNum(size); for(intindex=0;index urlList=keys.subList(start,last); for(Stringurl:urlList){ //提交任务 Runnabletask=newDownloadWorker(latch,url,resourceMap.get(url)); downloadExecutorService.submit(task); } latch.await(); } }catch(Exceptione){ logger.error("{}",e); } logger.info("Downloadresourcemapisalldone"); } /** *获取最后一个元素 * *@paramsize *列表长度 *@paramindex *下标 *@returnint */ privatestaticintgetLastNum(intsize,intindex){ returnindex>size?size:index; } /** *获取划分页面数量 * *@paramsize *列表长度 *@returnint */ privatestaticintgetPageNum(intsize){ inttmp=size/DOWNLOAD_THREAD_NUM; returnsize%DOWNLOAD_THREAD_NUM==0?tmp:tmp+1; } /** *下载线程 */ staticclassDownloadWorkerimplementsRunnable{ privateCountDownLatchlatch; privateStringurl; privateStringpath; DownloadWorker(CountDownLatchlatch,Stringurl,Stringpath){ this.latch=latch; this.url=url; this.path=path; } @Override publicvoidrun(){ logger.debug("Startbatch:[{}]into:[{}]",url,path); DownloadUtil.download(url,path); logger.debug("Download:[{}]into:[{}]isdone",url,path); latch.countDown(); } } }
3测试批量下载文件
packagecom.cheng.webb.thread; importjava.util.HashMap; importjava.util.Map; importorg.junit.Test; importcom.alibaba.fastjson.JSON; publicclassDownLoadTest{ Stringjson="{\r\n" +"\"http://www.xxx.com/111/123.mp4\":\"myFile/111/123.mp4\",\r\n" +"\"http://www.xxx.com/111/124.mp4\":\"myFile/111/124.mp4\",\r\n" +"\"http://www.xxx.com/111/125.mp4\":\"myFile/111/125.mp4\"\r\n" +"}"; @SuppressWarnings("unchecked") @Test publicvoidtest(){ Mapmap=newHashMap<>(); Map resMap=JSON.parseObject(json,map.getClass()); inttimes=1; for(intindex=0;index 以上就是本文的全部内容,希望对大家的学习有所帮助,也希望大家多多支持毛票票。