自定义线程池，批量下载文件

妖狐艹你老母 2024-04-18 23:23 47阅读 0赞

#### 一、创建线程池 ####

package threadpool;
    
    import java.util.concurrent.*;
    import java.util.concurrent.atomic.AtomicInteger;
    
    /**
     * 创建自定义线程池
     *
     * author:JZ
     * date:2019/9/13
     */
    public class CustomThreadPool {
    
        /**
         * 创建批量下载线程池
         * @param poolSize
         * @return
         */
        public static ExecutorService buildDownloadBatchThreadPool(int poolSize) {
            int keepAlive = 0;
            String threadPoolName = "download-batch";
            ThreadFactory factory = buildThreadFactory(threadPoolName);
    
            //当corePoolSize与maximumPoolSize相等时，相当于构建了一个固定大小的线程池
            return new ThreadPoolExecutor(poolSize,
                                        poolSize,
                                        keepAlive,
                                        TimeUnit.SECONDS,
                                        new ArrayBlockingQueue<>(poolSize),
                                        factory);
        }
    
        /**
         * 创建自定义线程工厂
         * @param threadFactoryName
         * @return ThreadFactory
         */
        private static ThreadFactory buildThreadFactory(String threadFactoryName) {
            return new CustomThreadFactory(threadFactoryName);
        }
    
        /**
         * 自定义线程工厂
         */
        public static class CustomThreadFactory implements ThreadFactory {
    
            private String threadFactoryName;
    
            private AtomicInteger counter = new AtomicInteger(1);
    
            /**
             * 自定义线程工厂
             * @param threadFactoryName 线程工厂名
             */
            public CustomThreadFactory(String threadFactoryName) {
                this.threadFactoryName = threadFactoryName;
            }
    
            @Override
            public Thread newThread(Runnable r) {
                String threadName = threadFactoryName + "-thread" + counter.getAndIncrement();
                return new Thread(r, threadName);
            }
        }
    }

#### 二、批量下载文件 ####

package threadpool;
    
    import java.io.*;
    import java.net.HttpURLConnection;
    import java.net.URL;
    import java.util.ArrayList;
    import java.util.List;
    import java.util.Map;
    import java.util.concurrent.CountDownLatch;
    import java.util.concurrent.ExecutorService;
    import java.util.concurrent.Executors;
    import java.util.concurrent.TimeUnit;
    
    /**
     * 下载文件工具
     * author:JZ
     * date:2019/9/13
     */
    
    public class DownloadUtil {
    
        /**
         * 下载线程数
         */
        private static final int DOWNLOAD_THREAD_NUM = 14;
    
        /**
         * 下载文件线程池
         */
        private static final ExecutorService downloadThreadPool = CustomThreadPool.buildDownloadBatchThreadPool(DOWNLOAD_THREAD_NUM);
    
    //    private static final ExecutorService downloadThreadPool = Executors.newFixedThreadPool(DOWNLOAD_THREAD_NUM);
    
        /**
         * 批量下载文件
         * @param resourceMap 文件rul与存储路径（包括文件名）的Map
         * @param fileSizeMap 文件对应的大小
         */
        public static void batchDownloadFile(Map<String, String> resourceMap, Map<String, Integer> fileSizeMap) {
            if(resourceMap == null || resourceMap.isEmpty()) {
                return;
            }
    
            try {
                List<String> urls = new ArrayList<>(resourceMap.keySet());
                int size = urls.size();
                int pageNum = getPageNum(size);
                for(int i = 0; i < pageNum; i++) {
                    int start = i * DOWNLOAD_THREAD_NUM;
                    int last = getLastNum(size, start + DOWNLOAD_THREAD_NUM);
                    final CountDownLatch latch = new CountDownLatch(last - start);
                    //获取当前需要下载的文件列表
                    List<String> urlList = urls.subList(start, last);
                    for(String url : urlList) {
                        Runnable task = new DownloadWorker(latch, url, resourceMap.get(url), fileSizeMap.get(url));
                        downloadThreadPool.submit(task);
    //                    downloadThreadPool.execute(task);
                    }
                    latch.await();
                }
            } catch(Exception e) {
                e.printStackTrace();
            }
        }
    
        /**
         * 用线程下载单个文件
         * @param fileUrl
         * @param path
         * @param fileSize
         */
        public static void downFile(String fileUrl, String path, int fileSize) {
            final CountDownLatch latch = new CountDownLatch(5);
            Runnable task = new DownloadWorker(latch, fileUrl, path, fileSize);
            downloadThreadPool.submit(task);
            try {
                latch.await(10, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    
        /**
         * 通过url下载一个文件
         * @param fileUrl
         * @param path
         * @param fileSize
         */
        public static void downloadFileByUrl(String fileUrl, String path, int fileSize) {
            if(!createFolder(path)) {
                return;
            }
    
            InputStream inputStream = null;
            BufferedInputStream bufferedInputStream = null;
            BufferedOutputStream bufferedOutputStream = null;
            DataOutputStream dataOutputStream = null;
            try {
                URL url = new URL(fileUrl);
                HttpURLConnection connection = (HttpURLConnection) url.openConnection();
                connection.setDoInput(true);
                connection.setDoInput(true);
                connection.setRequestMethod("GET");
                connection.setConnectTimeout(10000);
                connection.setRequestProperty("Content-Type", "application/octet-stream");
    //            connection.setRequestProperty("User-Agent", "User-Agent\", \"Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1)");
                connection.connect();
                if(connection.getResponseCode() == HttpURLConnection.HTTP_OK) {
                    bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(path));
                    dataOutputStream = new DataOutputStream(bufferedOutputStream);
    
                    inputStream = connection.getInputStream();
                    bufferedInputStream = new BufferedInputStream(inputStream);
                    DataInputStream dis = new DataInputStream(bufferedInputStream);
    
                    try {
                        while (true) {
                            dataOutputStream.write(dis.readByte());
                        }
                    } catch(EOFException e){
                        System.out.println("下载完成：" + path);
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
    
                    connection.disconnect();
                } else {
                    System.out.println(fileUrl + "   下载失败");
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                try {
                    if(inputStream != null) {
                        inputStream.close();
                    }
                    if(bufferedInputStream != null) {
                        bufferedInputStream.close();
                    }
                    if(bufferedOutputStream != null) {
                        bufferedOutputStream.close();
                    }
                    if(dataOutputStream != null) {
                        dataOutputStream.close();
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            return;
        }
    
        /**
         * 创建目录
         *
         * @param filePath 文件的绝对路径
         * @return Boolean
         */
        public static Boolean createFolder(String filePath) {
            String folderPath = filePath.substring(0, filePath.lastIndexOf("/"));
            File file = null;
            try {
                file = new File(folderPath);
                if(!file.exists()) {
                    file.mkdirs();
                }
            } catch (Exception e) {
                e.printStackTrace();
                return false;
            }
    
            return true;
        }
    
        /**
         * 获取划分页数
         * @param size 批量下载文件的个数
         * @return int
         */
        private static int getPageNum(int size) {
            int tmp = size / DOWNLOAD_THREAD_NUM;
            return size % DOWNLOAD_THREAD_NUM == 0 ? tmp : tmp + 1;
        }
    
        /**
         *
         * @param size 批量下载文件的总个数
         * @param index
         * @return
         */
        private static int getLastNum(int size, int index) {
            return index > size ? size : index;
        }
    
        static class DownloadWorker implements Runnable {
    
            private CountDownLatch latch;
            private String fileUrl;
            private String filePath;
            private int fileSize;
    
            DownloadWorker(CountDownLatch latch, String fileUrl, String filePath, int fileSize) {
                this.latch = latch;
                this.fileUrl = fileUrl;
                this.filePath = filePath;
                this.fileSize = fileSize;
            }
    
            @Override
            public void run() {
                DownloadUtil.downloadFileByUrl(fileUrl, filePath, fileSize);
                System.out.println(fileUrl);
                latch.countDown();
            }
        }
    
    }

#### 三、测试类 ####

package threadpool;
    
    import java.util.HashMap;
    import java.util.Map;
    
    /**
     * author:JZ
     * date:2019/9/13
     */
    public class demo {
    
        public static void main(String[] args) {
            Map<String, String> map = new HashMap<>();
            map.put("url", "D:\\123.jpg");
            Map<String, Integer> size = new HashMap<>();
            size.put("url", 1024*5);
            DownloadUtil .batchDownloadFile(map, size);
    
        }
    
    }

参考自：https://blog.csdn.net/wsc912406860/article/details/86705481