大文件秒传，分片上传，断点续传

大文件分片上传

一功能描述

1.文件通过web端分片多线程上传到服务端，然后web端发起分片合并，完成大文件分片上传功能

2.上传过的大文件，实现秒传

3.上传过程中，服务异常退出，实现断点续传

二流程图

在这里插入图片描述

三代码运行展示

1.分片上传

在这里插入图片描述

2.秒传

在这里插入图片描述

3.断点续传

在这里插入图片描述

四代码结构

在这里插入图片描述

五技术栈

1.springboot
2.aws存储
3.mysql

六代码片段

 @ApiOperation(value = "大文件上传")@PostMapping(value = "/big/upload",consumes = MediaType.MULTIPART_FORM_DATA_VALUE)public RestResp<String> uploadChunk(HttpServletRequest request,BigFileChunkFo bigFileChunkFo, MultipartFile file) {log.debug("分片上传参数:{}",bigFileChunkFo);String result = fileManager.uploadChunk(request,bigFileChunkFo,file);return RestResp.success(result);}@ApiOperation(value = "检查是否上传")@PostMapping("/big/checkMd5")public RestResp<BigFileCheckDto> checkMd5(HttpServletRequest request,@RequestBody BigFileCheckFo fileCheckFo) {BigFileCheckDto bigFileCheckDto = fileManager.checkMd5(request,fileCheckFo);return RestResp.success(bigFileCheckDto);}@ApiOperation(value = "大文件合并")@PostMapping("/big/merge")public RestResp<String> merge(HttpServletRequest request,@RequestBody BigFileMergeFo bigFileMergeFo) {log.debug("文件合并:{}",bigFileMergeFo);String result = fileManager.merge(request,bigFileMergeFo);return RestResp.success(result);}

    @Overridepublic String copyFile(String bucketName, String sourceFileKey, String targetFileKey) throws Exception {log.info("bucketName:{},sourceFileKey:{},targetFileKey:{}",bucketName,sourceFileKey,targetFileKey);CopyObjectRequest copyObjectRequest = new CopyObjectRequest();copyObjectRequest.setSourceBucketName(bucketName);copyObjectRequest.setDestinationBucketName(bucketName);copyObjectRequest.setSourceKey(sourceFileKey);copyObjectRequest.setDestinationKey(targetFileKey);getClient().copyObject(copyObjectRequest);return this.getObjectPrefixUrl(bucketName) +  targetFileKey;
}/*** 上传分片** @param bigFileChunkFo* @return* @throws Exception*/@Overridepublic PartETag uploadChunk(BigFileChunkFo bigFileChunkFo, BigFile bigFile, MultipartFile multipartFile) throws Exception {//桶名String bucketName = bigFileChunkFo.getBucketName();//当前分片int chunkNum = bigFileChunkFo.getChunkNum();//当前分片大小long chunkSize = bigFileChunkFo.getChunkSize();// 上传ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(IOUtils.toByteArray(multipartFile.getInputStream()));UploadPartRequest uploadRequest = new UploadPartRequest().withBucketName(bucketName).withKey(bigFile.getFileKey()).withUploadId(bigFile.getUploadId()).withInputStream(byteArrayInputStream).withPartNumber(chunkNum).withPartSize(chunkSize);UploadPartResult uploadResult = getClient().uploadPart(uploadRequest);return uploadResult.getPartETag();}/*** 获取上传任务id** @param bigFileCheckFo* @param fileKey* @return*/@Overridepublic String getUploadId(BigFileCheckFo bigFileCheckFo, String fileKey) {String bucketName = bigFileCheckFo.getBucketName();InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, fileKey);log.info("initRequest:{}", initRequest);InitiateMultipartUploadResult initResponse = getClient().initiateMultipartUpload(initRequest);return initResponse.getUploadId();}/*** 大文件合并** @param bigFile* @param bigFileChunkList* @return* @throws Exception*/@Overridepublic String merge(BigFile bigFile, List<BigFileChunk> bigFileChunkList) throws Exception {String bucketName = bigFile.getBucketName();List<PartETag> partETagList = bigFileChunkList.stream().map(bigFileChunk -> new PartETag(bigFileChunk.getChunkNum(), bigFileChunk.getETag())).collect(Collectors.toList());CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, bigFile.getFileKey(),bigFile.getUploadId(), partETagList);log.info("compRequest:{}", compRequest);getClient().completeMultipartUpload(compRequest);return this.getObjectPrefixUrl(bucketName) + bigFile.getFileKey();}

七表设计

CREATE TABLE `hfle_big_file` (`id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT '主键id',`md5` varchar(128) COLLATE utf8mb4_general_ci NOT NULL COMMENT '文件MD5',`status` int NOT NULL DEFAULT '1' COMMENT '上传状态,1:上传中;2:上传完成',`access_key` varchar(64) COLLATE utf8mb4_general_ci NOT NULL COMMENT 'accessKey',`chunk_count` bigint DEFAULT NULL COMMENT '分片总数',`file_name` varchar(240) COLLATE utf8mb4_general_ci NOT NULL COMMENT '文件名称',`file_size` bigint DEFAULT NULL COMMENT '文件大小',`bucket_name` varchar(64) COLLATE utf8mb4_general_ci NOT NULL COMMENT '上传桶',`file_type` varchar(128) COLLATE utf8mb4_general_ci NOT NULL COMMENT '文件类型',`file_key` varchar(128) COLLATE utf8mb4_general_ci NOT NULL COMMENT '文件唯一值',`url` varchar(256) COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '上传地址',`upload_id` varchar(128) COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT 's3上传任务id',`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',`creator` bigint NOT NULL DEFAULT '-1' COMMENT '创建人',`modifier` bigint NOT NULL DEFAULT '-1' COMMENT '最后更新人',`modified_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间',PRIMARY KEY (`id`) USING BTREE,UNIQUE KEY `uniq_md5_access_key` (`access_key`, `md5`, `bucket_name`, `file_key`)
) ENGINE = InnoDB AUTO_INCREMENT = 47 DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_general_ci COMMENT = '大文件上传记录表';CREATE TABLE `hfle_big_chunck` (`id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT '主键id',`big_file_id` bigint DEFAULT NULL COMMENT '大文件id',`chunk_num` bigint DEFAULT NULL COMMENT '当前分片',`e_tag` varchar(64) COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT 's3上传专用',`chunk_size` bigint DEFAULT NULL COMMENT '分片大小',`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',`creator` bigint NOT NULL DEFAULT '-1' COMMENT '创建人',`modifier` bigint NOT NULL DEFAULT '-1' COMMENT '最后更新人',`modified_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间',PRIMARY KEY (`id`) USING BTREE,UNIQUE KEY `uniq_chunk_num` (`big_file_id`, `chunk_num`)
) ENGINE = InnoDB AUTO_INCREMENT = 1542 DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_general_ci COMMENT = '大文件上传分片记录表'