ak/file
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

aliyunOSS分片上传

Browse Source
This commit is contained in:
马超 2020-11-23 00:00:30 +08:00
parent f970ef326d
commit 9d14174791
4 changed files with 126 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
file-common/src/main/java/com/qiwenshare/common/upload/Uploader.java
View File

@ -2,12 +2,15 @@ package com.qiwenshare.common.upload;
import com.qiwenshare.common.domain.UploadFile;
import com.qiwenshare.common.util.PathUtil;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.multipart.support.StandardMultipartHttpServletRequest;
import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.text.SimpleDateFormat;
@ -69,6 +72,27 @@ public abstract class Uploader {
}
public synchronized boolean checkUploadStatus(UploadFile param, File confFile) throws IOException {
//File confFile = new File(savePath, timeStampName + ".conf");
RandomAccessFile confAccessFile = new RandomAccessFile(confFile, "rw");
//设置文件长度
confAccessFile.setLength(param.getTotalChunks());
//设置起始偏移量
confAccessFile.seek(param.getChunkNumber() - 1);
//将指定的一个字节写入文件中 127，
confAccessFile.write(Byte.MAX_VALUE);
byte[] completeStatusList = FileUtils.readFileToByteArray(confFile);
confAccessFile.close();//不关闭会造成无法占用
//创建conf文件文件长度为总分片数，每上传一个分块即向conf文件中写入一个127，那么没上传的位置就是默认的0,已上传的就是127
for (int i = 0; i < completeStatusList.length; i++) {
if (completeStatusList[i] != Byte.MAX_VALUE) {
return false;
}
}
confFile.delete();
return true;
}
protected String getFileName(String fileName){
return fileName.substring(0, fileName.lastIndexOf("."));
}

176
file-common/src/main/java/com/qiwenshare/common/upload/product/AliyunOSSUploader.java
View File

@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSON;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSClientBuilder;
import com.aliyun.oss.model.*;
import com.aliyuncs.utils.StringUtils;
import com.qiwenshare.common.domain.AliyunOSS;
import com.qiwenshare.common.domain.UploadFile;
import com.qiwenshare.common.upload.Uploader;
@ -31,12 +32,19 @@ public class AliyunOSSUploader extends Uploader {
private static final Logger logger = LoggerFactory.getLogger(AliyunOSSUploader.class);
private UploadFile uploadFile;
public static ExecutorService executorService = Executors.newFixedThreadPool(1);
private String endpoint;
private String accessKeyId;
private String accessKeySecret;
private String bucketName;
// partETags是PartETag的集合。PartETag由分片的ETag和分片号组成。
public static Map<String, List<PartETag>> partETagsMap = new HashMap<String, List<PartETag>>();
public static Map<String, UploadFileInfo> uploadPartRequestMap = new HashMap<>();
public static Map<String, OSS> ossMap = new HashMap<>();
public static Map<String, String> timeStampNameMap = new HashMap<>();
public AliyunOSSUploader() {
}
@ -49,8 +57,14 @@ public class AliyunOSSUploader extends Uploader {
public List<UploadFile> upload(HttpServletRequest httpServletRequest) {
logger.info("开始上传upload");
List<UploadFile> saveUploadFileList = new ArrayList<UploadFile>();
List<UploadFile> saveUploadFileList = new ArrayList<>();
this.request = (StandardMultipartHttpServletRequest) httpServletRequest;
AliyunOSS aliyunOSS = (AliyunOSS) request.getAttribute("oss");
endpoint = aliyunOSS.getEndpoint();
accessKeyId = aliyunOSS.getAccessKeyId();
accessKeySecret = aliyunOSS.getAccessKeySecret();
bucketName = aliyunOSS.getBucketName();
boolean isMultipart = ServletFileUpload.isMultipartContent(this.request);
if (!isMultipart) {
UploadFile uploadFile = new UploadFile();
@ -85,15 +99,9 @@ public class AliyunOSSUploader extends Uploader {
}
private List<UploadFile> doUpload(String savePath, Iterator<String> iter) throws IOException {
AliyunOSS aliyunOSS = (AliyunOSS) request.getAttribute("oss");
OSS ossClient = getClient();
String endpoint = aliyunOSS.getEndpoint();
String accessKeyId = aliyunOSS.getAccessKeyId();
String accessKeySecret = aliyunOSS.getAccessKeySecret();
String bucketName = aliyunOSS.getBucketName();
OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret);
List<UploadFile> saveUploadFileList = new ArrayList<UploadFile>();
List<UploadFile> saveUploadFileList = new ArrayList<>();
try {
MultipartFile multipartfile = this.request.getFile(iter.next());
@ -112,7 +120,6 @@ public class AliyunOSSUploader extends Uploader {
String confFilePath = savePath + FILE_SEPARATOR + uploadFile.getIdentifier() + "." + "conf";
File confFile = new File(PathUtil.getStaticPath() + FILE_SEPARATOR + confFilePath);
synchronized (AliyunOSSUploader.class) {
if (uploadPartRequestMap.get(uploadFile.getIdentifier()) == null) {
InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, ossFilePath.substring(1));
@ -146,7 +153,7 @@ public class AliyunOSSUploader extends Uploader {
UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
synchronized (AliyunOSSUploader.class) {
logger.info("上传结果：" + JSON.toJSONString(uploadPartResult));
if (partETagsMap.get(uploadFile.getIdentifier()) == null) {
List<PartETag> partETags = new ArrayList<PartETag>();
partETags.add(uploadPartResult.getPartETag());
@ -159,25 +166,8 @@ public class AliyunOSSUploader extends Uploader {
boolean isComplete = checkUploadStatus(uploadFile, confFile);
if (isComplete) {
logger.info("分片上传完成");
List<PartETag> partETags = partETagsMap.get(uploadFile.getIdentifier());
List<PartETag> partETagsSort = new ArrayList<>();
for (int i = 0; i < partETags.size(); i++) {
for (int index = 1; index <= partETags.size(); index++) {
if (i + 1 == partETags.get(index - 1).getPartNumber()) {
partETagsSort.add(partETags.get(index - 1));
}
}
completeMultipartUpload();
}
CompleteMultipartUploadRequest completeMultipartUploadRequest =
new CompleteMultipartUploadRequest(bucketName,
ossFilePath.substring(1),
uploadFileInfo.getUploadId(),
partETagsSort);
logger.info("----:" + JSON.toJSONString(partETagsSort));
// 完成上传。
CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
logger.info("----:" + JSON.toJSONString(completeMultipartUploadRequest));
uploadFile.setUrl(ossFilePath);
uploadFile.setSuccess(1);
uploadFile.setMessage("上传成功");
@ -188,34 +178,6 @@ public class AliyunOSSUploader extends Uploader {
} catch (Exception e) {
logger.error("上传出错：" + e);
// 列举已上传的分片，其中uploadId来自于InitiateMultipartUpload返回的结果。
ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, uploadPartRequestMap.get(uploadFile.getIdentifier()).getKey(), uploadPartRequestMap.get(uploadFile.getIdentifier()).getUploadId());
// 设置uploadId。
//listPartsRequest.setUploadId(uploadId);
// 设置分页时每一页中分片数量为100个。默认列举1000个分片。
listPartsRequest.setMaxParts(100);
// 指定List的起始位置。只有分片号大于此参数值的分片会被列举。
// listPartsRequest.setPartNumberMarker(1);
PartListing partListing = ossClient.listParts(listPartsRequest);
for (PartSummary part : partListing.getParts()) {
logger.info("分片号："+part.getPartNumber() + ", 分片数据大小: "+
part.getSize() + "，分片的ETag:"+part.getETag()
+ "， 分片最后修改时间："+ part.getLastModified());
// 获取分片号。
part.getPartNumber();
// 获取分片数据大小。
part.getSize();
// 获取分片的ETag。
part.getETag();
// 获取分片的最后修改时间。
part.getLastModified();
}
AbortMultipartUploadRequest abortMultipartUploadRequest =
new AbortMultipartUploadRequest(bucketName, uploadPartRequestMap.get(uploadFile.getIdentifier()).getKey(), uploadPartRequestMap.get(uploadFile.getIdentifier()).getUploadId());
ossClient.abortMultipartUpload(abortMultipartUploadRequest);
} finally {
ossClient.shutdown();
}
uploadFile.setIsOSS(1);
@ -225,25 +187,87 @@ public class AliyunOSSUploader extends Uploader {
return saveUploadFileList;
}
public synchronized boolean checkUploadStatus(UploadFile param, File confFile) throws IOException {
//File confFile = new File(savePath, timeStampName + ".conf");
RandomAccessFile confAccessFile = new RandomAccessFile(confFile, "rw");
//设置文件长度
confAccessFile.setLength(param.getTotalChunks());
//设置起始偏移量
confAccessFile.seek(param.getChunkNumber() - 1);
//将指定的一个字节写入文件中 127，
confAccessFile.write(Byte.MAX_VALUE);
byte[] completeStatusList = FileUtils.readFileToByteArray(confFile);
confAccessFile.close();//不关闭会造成无法占用
//创建conf文件文件长度为总分片数，每上传一个分块即向conf文件中写入一个127，那么没上传的位置就是默认的0,已上传的就是127
for (int i = 0; i < completeStatusList.length; i++) {
if (completeStatusList[i] != Byte.MAX_VALUE) {
return false;
}
/**
* 将文件分块进行升序排序并执行文件上传。
*/
protected void completeMultipartUpload() {
List<PartETag> partETags = partETagsMap.get(uploadFile.getIdentifier());
Collections.sort(partETags, Comparator.comparingInt(PartETag::getPartNumber));
UploadFileInfo uploadFileInfo = uploadPartRequestMap.get(uploadFile.getIdentifier());
CompleteMultipartUploadRequest completeMultipartUploadRequest =
new CompleteMultipartUploadRequest(bucketName,
uploadFileInfo.getKey(),
uploadFileInfo.getUploadId(),
partETags);
logger.info("----:" + JSON.toJSONString(partETags));
// 完成上传。
CompleteMultipartUploadResult completeMultipartUploadResult = getClient().completeMultipartUpload(completeMultipartUploadRequest);
logger.info("----:" + JSON.toJSONString(completeMultipartUploadRequest));
getClient().shutdown();
partETagsMap.remove(uploadFile.getIdentifier());
uploadPartRequestMap.remove(uploadFile.getIdentifier());
ossMap.remove(uploadFile.getIdentifier());
//
}
private void listFile() {
// 列举已上传的分片，其中uploadId来自于InitiateMultipartUpload返回的结果。
ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, uploadPartRequestMap.get(uploadFile.getIdentifier()).getKey(), uploadPartRequestMap.get(uploadFile.getIdentifier()).getUploadId());
// 设置uploadId。
//listPartsRequest.setUploadId(uploadId);
// 设置分页时每一页中分片数量为100个。默认列举1000个分片。
listPartsRequest.setMaxParts(100);
// 指定List的起始位置。只有分片号大于此参数值的分片会被列举。
// listPartsRequest.setPartNumberMarker(1);
PartListing partListing = getClient().listParts(listPartsRequest);
for (PartSummary part : partListing.getParts()) {
logger.info("分片号："+part.getPartNumber() + ", 分片数据大小: "+
part.getSize() + "，分片的ETag:"+part.getETag()
+ "， 分片最后修改时间："+ part.getLastModified());
// 获取分片号。
part.getPartNumber();
// 获取分片数据大小。
part.getSize();
// 获取分片的ETag。
part.getETag();
// 获取分片的最后修改时间。
part.getLastModified();
}
confFile.delete();
return true;
}
/**
* 取消上传
*/
private void cancelUpload() {
AbortMultipartUploadRequest abortMultipartUploadRequest =
new AbortMultipartUploadRequest(bucketName, uploadPartRequestMap.get(uploadFile.getIdentifier()).getKey(), uploadPartRequestMap.get(uploadFile.getIdentifier()).getUploadId());
getClient().abortMultipartUpload(abortMultipartUploadRequest);
}
protected synchronized String getTimeStampName(){
String timeStampName;
if (StringUtils.isEmpty(timeStampNameMap.get(uploadFile.getIdentifier()))) {
timeStampName = super.getTimeStampName();
timeStampNameMap.put(uploadFile.getIdentifier(), timeStampName);
} else {
timeStampName = timeStampNameMap.get(uploadFile.getIdentifier());
}
return timeStampName;
}
private synchronized OSS getClient() {
OSS ossClient = null;
if (ossMap.get(uploadFile.getIdentifier()) == null) {
ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret);
ossMap.put(uploadFile.getIdentifier(), ossClient);
} else {
ossClient = ossMap.get(uploadFile.getIdentifier());
}
return ossClient;
}
@Data

23
file-common/src/main/java/com/qiwenshare/common/upload/product/ChunkUploader.java
View File

@ -77,27 +77,6 @@ public class ChunkUploader extends Uploader {
}
public boolean checkUploadStatus(UploadFile param, File confFile) throws IOException {
//File confFile = new File(savePath, timeStampName + ".conf");
RandomAccessFile confAccessFile = new RandomAccessFile(confFile, "rw");
//设置文件长度
confAccessFile.setLength(param.getTotalChunks());
//设置起始偏移量
confAccessFile.seek(param.getChunkNumber() - 1);
//将指定的一个字节写入文件中 127，
confAccessFile.write(Byte.MAX_VALUE);
byte[] completeStatusList = FileUtils.readFileToByteArray(confFile);
confAccessFile.close();//不关闭会造成无法占用
//创建conf文件文件长度为总分片数，每上传一个分块即向conf文件中写入一个127，那么没上传的位置就是默认的0,已上传的就是127
for (int i = 0; i < completeStatusList.length; i++) {
if (completeStatusList[i] != Byte.MAX_VALUE) {
return false;
}
}
confFile.delete();
return true;
}
private List<UploadFile> doUpload(String savePath, Iterator<String> iter) throws IOException, NotSameFileExpection {
List<UploadFile> saveUploadFileList = new ArrayList<UploadFile>();
//UploadFile uploadFile = new UploadFile();
@ -168,7 +147,7 @@ public class ChunkUploader extends Uploader {
uploadFile.setSuccess(0);
uploadFile.setMessage("未完成");
}
uploadFile.setFileSize(uploadFile.getCurrentChunkSize());
uploadFile.setFileSize(uploadFile.getTotalSize());
saveUploadFileList.add(uploadFile);
return saveUploadFileList;

16
file-common/src/main/java/com/qiwenshare/common/util/JjwtUtil.java
View File

@ -1,6 +1,5 @@
package com.qiwenshare.common.util;
import com.alibaba.fastjson.JSON;
import io.jsonwebtoken.Claims;
import io.jsonwebtoken.JwtBuilder;
import io.jsonwebtoken.Jwts;
@ -9,7 +8,6 @@ import org.apache.commons.net.util.Base64;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
@ -24,7 +22,7 @@ public class JjwtUtil {
public static final String JWT_SECRET = "jiamimiwen";
// 过期时间，单位毫秒
public static final int EXPIRE_TIME = 60 * 60 * 1000; // 一个小时
public static final int EXPIRE_TIME = 60 * 60 * 1000 * 24 * 7; // 一个星期
// public static final long EXPIRE_TIME = 7 * 24 * 3600 * 1000; // 一个星期
// 由字符串生成加密key
@ -90,17 +88,5 @@ public class JjwtUtil {
return claims;
}
public static void main(String[] args) {
Claims claims = null;
try {
claims = parseJWT("eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ7XCJhZGRyQXJlYVwiOlwi6ZuB5aGU5Yy6XCIsXCJhZGRyQ2l0eVwiOlwi6KW_5a6J5biCXCIsXCJhZGRyUHJvdmluY2VcIjpcIumZleilv-ecgVwiLFwiYmVnaW5Db3VudFwiOjAsXCJiaXJ0aGRheVwiOlwiMTk5NC0wNS0wNlwiLFwiY3VycmVudFBhZ2VcIjowLFwiZWRpdG9yVHlwZVwiOjAsXCJlbWFpbFwiOlwiMTE2MjcxNDQ4M0BxcS5jb21cIixcImVzc2F5Q291bnRcIjowLFwiaW1hZ2VVcmxcIjpcIi91cGxvYWQvMjAyMDA3MTUvODA5NjE1OTQ4MjEzODkwNTkuanBnXCIsXCJpbmR1c3RyeVwiOlwi6K6h566X5py66KGM5LiaXCIsXCJpbnRyb1wiOlwi6ZSZ5oqK6ZmI6YaL5b2T5oiQ5aKo77yM5YaZ5bC95Y2K55Sf6YO95piv6YW444CCXCIsXCJsYXN0TG9naW5UaW1lXCI6XCIyMDIwLTEwLTI3IDE5OjAxOjU0XCIsXCJub3RSZWFkQ291bnRcIjowLFwicGFnZUNvdW50XCI6MCxcInBhc3N3b3JkXCI6XCI3NjcyYmE0NDc4OTY5NTlmNmU1NDQ1NTcxOGY2ZWU2ZlwiLFwic2FsdFwiOlwiMzY5MjQ1NzgxMDIyMDg3NlwiLFwic2V4XCI6XCLnlLdcIixcInRlbGVwaG9uZVwiOlwiMTg4MjkyOTE4MTdcIixcInVzZXJJZFwiOjIsXCJ1c2VybmFtZVwiOlwiTUFDXCJ9IiwiYXVkIjoicWl3ZW4iLCJwYXNzd29yZCI6IjAxMDIwMyIsImlzcyI6InFpd2Vuc2hhcmUiLCJleHAiOjE2MDM4MDA0MDcsImlhdCI6MTYwMzc5NjgwNywianRpIjoiNzYzZTE0NDMtYmZjYS00MDkzLTg3NGYtMzlmMzQ1ODRiZDkyIiwidXNlcm5hbWUiOiJhZG1pbiJ9.Gv_QlXZEiT_-mbI2Iz6oa2SqgBXwVzsS_xK8RhvF7oY");
} catch (Exception e) {
e.printStackTrace();
}
String subject = claims.getSubject();
System.out.println(subject);
}
}