Java大文件分片上传(minio版)，超详细

妖狐艹你老母 2023-10-16 00:14 68阅读 0赞

> 本文使用spring boot 结合minio文件服务做的大文件分片上传，思路：①：初始化文件调用后端接口，后端再调用minio把文件分片成几份，生成每个分片的minio上传url②：把提起分片好的文件依次调用上一把返回的url ③：调用合并分片文件接口，完成上传 PS：文件并未经过后端服务器，而是直接上传到minio文件服务的

#### 文章目录 ####

*   *  第一步：pom引入minio
     *  第二步：配置minio
     *  第三步：自定义minio客户端与工具类
     *  1. 自定义客户端
     *  2.封装minio配置
     *  3.上传工具类
     *  4.service
     *  5.service实现
     *  6.接口控制器
     *  第四步：文件切片
     *  第五步：示例截图

### 第一步：pom引入minio ###

<dependency>
                <groupId>io.minio</groupId>
                <artifactId>minio</artifactId>
                <version>8.0.3</version>
            </dependency>

### 第二步：配置minio ###

# minio 配置（大文件分片上传使用）
    minio:
      endpoint: ${
        MINIO_ENDPOINT:http://minio服务IP:端口号}
      accesskey: ${
        MINIO_ASSESSKEY:密钥}
      secretkey: ${
        MINIO_SECRETKEY:密匙}
      bucket: ${
        MINIO_BUCKET:桶名称}

![在这里插入图片描述][fcac4b9c37b54d34b61ec57000db3a67.png]

### 第三步：自定义minio客户端与工具类 ###

### 1. 自定义客户端 ###

package com.gcddd.visual.uav.largeFileUpload.util;
    
    import com.google.common.collect.Multimap;
    import io.minio.CreateMultipartUploadResponse;
    import io.minio.ListPartsResponse;
    import io.minio.MinioClient;
    import io.minio.ObjectWriteResponse;
    import io.minio.errors.*;
    import io.minio.messages.Part;
    
    import java.io.IOException;
    import java.security.InvalidKeyException;
    import java.security.NoSuchAlgorithmException;
    
    /**
     * 自定义minio
     * @date 2023-09-15
     */
    public class CustomMinioClient extends MinioClient {
        
    
        public CustomMinioClient(MinioClient client) {
        
            super(client);
        }
    
        public String initMultiPartUpload(String bucket, String region, String object, Multimap<String, String> headers, Multimap<String, String> extraQueryParams) throws IOException, InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException {
        
            CreateMultipartUploadResponse response = this.createMultipartUpload(bucket, region, object, headers, extraQueryParams);
    
            return response.result().uploadId();
        }
    
        public ObjectWriteResponse mergeMultipartUpload(String bucketName, String region, String objectName, String uploadId, Part[] parts, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams) throws IOException, InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException {
        
    
            return this.completeMultipartUpload(bucketName, region, objectName, uploadId, parts, extraHeaders, extraQueryParams);
        }
    
        public ListPartsResponse listMultipart(String bucketName, String region, String objectName, Integer maxParts, Integer partNumberMarker, String uploadId, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams) throws NoSuchAlgorithmException, InsufficientDataException, IOException, InvalidKeyException, ServerException, XmlParserException, ErrorResponseException, InternalException, InvalidResponseException {
        
            return this.listParts(bucketName, region, objectName, maxParts, partNumberMarker, uploadId, extraHeaders, extraQueryParams);
        }
    }

### 2.封装minio配置 ###

package com.gcddd.visual.uav.largeFileUpload.configurer;
    
    import lombok.Getter;
    import lombok.Setter;
    import org.springframework.boot.context.properties.ConfigurationProperties;
    
    /**
     * @Description: MinioProperties
     * minio配置类
     */
    @ConfigurationProperties(prefix = "minio")
    @Getter
    @Setter
    public class MinioProperties {
        
    
        private String endpoint;
    
        private String accesskey;
    
        private String secretkey;
    
        private String bucket;
    }

### 3.上传工具类 ###

package com.gcddd.visual.uav.largeFileUpload.configurer;
    
    import cn.hutool.core.util.StrUtil;
    import com.gcddd.visual.uav.largeFileUpload.util.CustomMinioClient;
    import com.google.common.collect.HashMultimap;
    import io.minio.*;
    import io.minio.http.Method;
    import io.minio.messages.Part;
    import org.springframework.boot.context.properties.EnableConfigurationProperties;
    import org.springframework.context.annotation.Configuration;
    import org.springframework.stereotype.Component;
    
    import javax.annotation.PostConstruct;
    import java.util.ArrayList;
    import java.util.HashMap;
    import java.util.List;
    import java.util.Map;
    import java.util.concurrent.TimeUnit;
    
    /**
    * @Description: MinIoUtils
     * 工具类
     */
    @Component
    @Configuration
    @EnableConfigurationProperties({
        MinioProperties.class})
    public class MinIoUtils {
        
        private final MinioProperties minioProperties;
        private CustomMinioClient customMinioClient;
    
        public MinIoUtils(MinioProperties minioProperties) {
        
            this.minioProperties = minioProperties;
        }
    
        @PostConstruct
        public void init() {
        
            MinioClient minioClient = MinioClient.builder()
                    .endpoint(minioProperties.getEndpoint())
                    .credentials(minioProperties.getAccesskey(), minioProperties.getSecretkey())
                    .build();
            customMinioClient = new CustomMinioClient(minioClient);
        }
    
        /**
         * 单文件签名上传
         *
         * @param objectName 文件全路径名称
         * @return /
         */
        public String getUploadObjectUrl(String objectName) {
        
            try {
        
                return customMinioClient.getPresignedObjectUrl(
                        GetPresignedObjectUrlArgs.builder()
                                .method(Method.PUT)
                                .bucket(minioProperties.getBucket())
                                .object(objectName)
                                .expiry(1, TimeUnit.DAYS)
                                //.extraHeaders(headers)
                                .build()
                );
            } catch (Exception e) {
        
                e.printStackTrace();
                return null;
            }
        }
    
        /**
         *  初始化分片上传
         *
         * @param objectName 文件全路径名称
         * @param partCount 分片数量
         * @param contentType 类型，如果类型使用默认流会导致无法预览
         * @return /
         */
        public Map<String, Object> initMultiPartUpload(String objectName, int partCount, String contentType) {
        
            Map<String, Object> result = new HashMap<>();
            try {
        
                if (StrUtil.isBlank(contentType)) {
        
                    contentType = "application/octet-stream";
                }
                HashMultimap<String, String> headers = HashMultimap.create();
                headers.put("Content-Type", contentType);
                String uploadId = customMinioClient.initMultiPartUpload(minioProperties.getBucket(), null, objectName, headers, null);
    
                result.put("uploadId", uploadId);
                List<String> partList = new ArrayList<>();
    
                Map<String, String> reqParams = new HashMap<>();
                //reqParams.put("response-content-type", "application/json");
                reqParams.put("uploadId", uploadId);
                for (int i = 1; i <= partCount; i++) {
        
                    reqParams.put("partNumber", String.valueOf(i));
                    String uploadUrl = customMinioClient.getPresignedObjectUrl(
                            GetPresignedObjectUrlArgs.builder()
                                    .method(Method.PUT)
                                    .bucket(minioProperties.getBucket())
                                    .object(objectName)
                                    .expiry(1, TimeUnit.DAYS)
                                    .extraQueryParams(reqParams)
                                    .build());
                    partList.add(uploadUrl);
                }
                result.put("uploadUrls", partList);
            } catch (Exception e) {
        
                e.printStackTrace();
                return null;
            }
    
            return result;
        }
    
        /**
         * 分片上传完后合并
         *
         * @param objectName 文件全路径名称
         * @param uploadId 返回的uploadId
         * @return /
         */
        public boolean mergeMultipartUpload(String objectName, String uploadId) {
        
            try {
        
                //TODO::目前仅做了最大1000分片
                Part[] parts = new Part[1000];
                ListPartsResponse partResult = customMinioClient.listMultipart(minioProperties.getBucket(), null, objectName, 1000, 0, uploadId, null, null);
                int partNumber = 1;
                for (Part part : partResult.result().partList()) {
        
                    parts[partNumber - 1] = new Part(partNumber, part.etag());
                    partNumber++;
                }
                customMinioClient.mergeMultipartUpload(minioProperties.getBucket(), null, objectName, uploadId, parts, null, null);
            } catch (Exception e) {
        
                e.printStackTrace();
                return false;
            }
    
            return true;
        }
    }

### 4.service ###

package com.gcddd.visual.uav.largeFileUpload.service;
    
    import java.util.Map;
    
    /**
     * @date 2023-09-13
     */
    public interface UploadService {
        
    
        /**
         * 分片上传初始化
         *
         * @param path        路径
         * @param filename    文件名
         * @param partCount   分片数量
         * @param contentType /
         * @return /
         */
        Map<String, Object> initMultiPartUpload(String path, String filename, Integer partCount, String contentType);
    
        /**
         * 完成分片上传
         *
         * @param objectName 文件名
         * @param uploadId 标识
         * @return /
         */
         boolean mergeMultipartUpload(String objectName, String uploadId);
    }

### 5.service实现 ###

package com.gcddd.visual.uav.largeFileUpload.service.impl;
    
    import com.gcddd.visual.uav.largeFileUpload.configurer.MinIoUtils;
    import com.gcddd.visual.uav.largeFileUpload.service.UploadService;
    import com.google.common.collect.ImmutableList;
    import com.google.common.collect.ImmutableMap;
    import lombok.RequiredArgsConstructor;
    import org.springframework.stereotype.Service;
    
    import java.util.Map;
    
    /**
     * @date 2023-09-13
     */
    @Service
    @RequiredArgsConstructor
    public class UploadServiceImpl implements UploadService {
        
    
        private final MinIoUtils minIoUtils;
    
        @Override
        public Map<String, Object> initMultiPartUpload(String path, String filename, Integer partCount, String contentType) {
        
            path = path.replaceAll("/+", "/");
            if (path.indexOf("/") == 0) {
        
                path = path.substring(1);
            }
            String filePath = path + "/" + filename;
    
            Map<String, Object> result;
            // 单文件，直接上传
            if (partCount == 1) {
        
                String uploadObjectUrl = minIoUtils.getUploadObjectUrl(filePath);
                result = ImmutableMap.of("uploadUrls", ImmutableList.of(uploadObjectUrl));
            } else {
        //多文件，分片上传
                result = minIoUtils.initMultiPartUpload(filePath, partCount, contentType);
            }
    
            return result;
        }
    
        @Override
        public boolean mergeMultipartUpload(String objectName, String uploadId) {
        
            return minIoUtils.mergeMultipartUpload(objectName, uploadId);
        }
    }

### 6.接口控制器 ###

import cn.hutool.core.lang.Assert;
    import cn.hutool.json.JSONObject;
    import com.gcddd.visual.uav.largeFileUpload.service.UploadService;
    import com.gcddd.visual.uav.util.DateUtils;
    import com.google.common.collect.ImmutableMap;
    import io.swagger.v3.oas.annotations.security.SecurityRequirement;
    import io.swagger.v3.oas.annotations.tags.Tag;
    import lombok.RequiredArgsConstructor;
    import org.springframework.http.HttpHeaders;
    import org.springframework.http.HttpStatus;
    import org.springframework.http.ResponseEntity;
    import org.springframework.web.bind.annotation.*;
    
    import java.util.Map;
    
    /**
     * 大文件上传控制器
     */
    @RestController
    @RequiredArgsConstructor
    @RequestMapping("/largeFileUpload" )
    @Tag(description = "largeFileUpload" , name = "大文件分片上传" )
    @SecurityRequirement(name = HttpHeaders.AUTHORIZATION)
    public class uploadController {
        
    
        private final UploadService uploadService;
    
    //    1.用户调用初始化接口，后端调用minio初始化，得到uploadId，生成每个分片的minio上传url
    //初始化 示例： $ curl --location --request POST 'http://127.0.0.1:9998/admin/largeFileUpload/multipart/init' \
    //            --header 'Content-Type: application/json' \
    //            --data-raw '{
        
        //            "filename": "result.png",
        //            "partCount": 2
    //              }'
        //    返回示例：
    //{
        
    //    "uploadId": "eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3",
    //        "uploadUrls": [
    //    "http://ip:端口号/uav/20230913/result.png?uploadId=eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3&partNumber=1&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=lengleng%2F20230913%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230913T031427Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=9e7bd28d6f4d42d0ed447750cadcd8782ccc2648a66ecb25a1e7639593ad7dd6",
    //            "http://ip:端口号/uav/20230913/result.png?uploadId=eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3&partNumber=2&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=lengleng%2F20230913%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230913T031427Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=60d31989b9dc3bdacf6bbc9c13cb5c982f79b1e3a0bbcb948e0ec90fede2f9f3"
    //    ]
    //}
    //    2.用户调用对应分片的上传地址，多次上传会覆盖
    //    示例：
    //    curl --location --request PUT 'http://ip:端口号/uav/20230913/result.png?uploadId=eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3&partNumber=1&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=lengleng%2F20230913%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230913T031427Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=9e7bd28d6f4d42d0ed447750cadcd8782ccc2648a66ecb25a1e7639593ad7dd6' \
    //            --header 'Content-Type: application/octet-stream' \
    //            --data-binary '@/D:/jpgone'
    //    curl --location --request PUT 'http://ip:端口号/uav/20230913/result.png?uploadId=eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3&partNumber=2&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=lengleng%2F20230913%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230913T031427Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=60d31989b9dc3bdacf6bbc9c13cb5c982f79b1e3a0bbcb948e0ec90fede2f9f3' \
    //            --header 'Content-Type: application/octet-stream' \
    //            --data-binary '@/D:/jpgtwo'
    //    3.调用完成接口，后端查询所有上传的分片并合并，uploadId为第一步返回的uploadId
    //    示例：
    //    curl --location --request PUT '127.0.0.1:8006/multipart/complete' \
    //            --header 'Content-Type: application/json' \
    //            --data-raw '{
        
                        //    "objectName":"20230913/result.png",
                        //    "uploadId":"eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3"
    //                      }'
    
        /**
         * 分片初始化
         *
         * @param requestParam 请求参数
         * @return /
         */
        @PostMapping("/multipart/init")
        public ResponseEntity<Object> initMultiPartUpload(@RequestBody JSONObject requestParam) {
        
            // 路径（存入minio的路径）
            String pathText = DateUtils.getTodays("yyyyMMdd");//当前日期
            String path = requestParam.getStr("path", pathText);
            // 文件名
            String filename = requestParam.getStr("filename", "test.obj");
            // content-type
            String contentType = requestParam.getStr("contentType", "application/octet-stream");
            // md5-可进行秒传判断
            String md5 = requestParam.getStr("md5", "");
            // 分片数量
            Integer partCount = requestParam.getInt("partCount", 100);
    
            //TODO::业务判断+秒传判断
    
            Map<String, Object> result = uploadService.initMultiPartUpload(path, filename, partCount, contentType);
    
            return new ResponseEntity<>(result, HttpStatus.OK);
        }
    
        /**
         * 合并分片文件，完成上传
         *
         * @param requestParam 用户参数
         * @return /
         */
        @PutMapping("/multipart/complete")
        public ResponseEntity<Object> completeMultiPartUpload(
                @RequestBody JSONObject requestParam
        ) {
        
            // 文件名完整路径
            String objectName = requestParam.getStr("objectName");
            //初始化返回的uploadId
            String uploadId = requestParam.getStr("uploadId");
            Assert.notNull(objectName, "objectName must not be null");
            Assert.notNull(uploadId, "uploadId must not be null");
            boolean result = uploadService.mergeMultipartUpload(objectName, uploadId);
    
            return new ResponseEntity<>(ImmutableMap.of("success", result), HttpStatus.OK);
        }
    }

### 第四步：文件切片 ###

把result.png图片以500M大小切片，重命名为result.png.后缀

split -b 500m result.png result.png.

![在这里插入图片描述][597a933e61b8477d8d19365fa3875da4.png]

### 第五步：示例截图 ###

这里测试为把result.png切为2片，返回了uploadId 和两个minio文件上传地址  
![在这里插入图片描述][a5cd49ee94e24c9f95996867cea90975.png]

分片上传的地址上传文件，请求方式为put  
![在这里插入图片描述][b59068809e8b4952a52971217f0e4378.png]

![在这里插入图片描述][00a5bcb41fa4416893c9e4a954c43dc0.png]  
合并文件，完成上传  
![在这里插入图片描述][b56b58f3e79540f2b5f28b133ed7ea20.png]  
检查minio是否成功  
![在这里插入图片描述][bd4991a72da24b3c9170e7b42425ea84.png]

[fcac4b9c37b54d34b61ec57000db3a67.png]: https://img-blog.csdnimg.cn/fcac4b9c37b54d34b61ec57000db3a67.png
[597a933e61b8477d8d19365fa3875da4.png]: https://img-blog.csdnimg.cn/597a933e61b8477d8d19365fa3875da4.png
[a5cd49ee94e24c9f95996867cea90975.png]: https://img-blog.csdnimg.cn/a5cd49ee94e24c9f95996867cea90975.png
[b59068809e8b4952a52971217f0e4378.png]: https://img-blog.csdnimg.cn/b59068809e8b4952a52971217f0e4378.png
[00a5bcb41fa4416893c9e4a954c43dc0.png]: https://img-blog.csdnimg.cn/00a5bcb41fa4416893c9e4a954c43dc0.png
[b56b58f3e79540f2b5f28b133ed7ea20.png]: https://img-blog.csdnimg.cn/b56b58f3e79540f2b5f28b133ed7ea20.png
[bd4991a72da24b3c9170e7b42425ea84.png]: https://img-blog.csdnimg.cn/bd4991a72da24b3c9170e7b42425ea84.png