From 9d141747911a051a13d5286ee8c3fbea6d9b398e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E8=B6=85?= <1162714483@qq.com> Date: Mon, 23 Nov 2020 00:00:30 +0800 Subject: [PATCH] =?UTF-8?q?aliyunOSS=E5=88=86=E7=89=87=E4=B8=8A=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../qiwenshare/common/upload/Uploader.java | 24 +++ .../upload/product/AliyunOSSUploader.java | 176 ++++++++++-------- .../common/upload/product/ChunkUploader.java | 23 +-- .../com/qiwenshare/common/util/JjwtUtil.java | 16 +- 4 files changed, 126 insertions(+), 113 deletions(-) diff --git a/file-common/src/main/java/com/qiwenshare/common/upload/Uploader.java b/file-common/src/main/java/com/qiwenshare/common/upload/Uploader.java index 24314da..4044352 100644 --- a/file-common/src/main/java/com/qiwenshare/common/upload/Uploader.java +++ b/file-common/src/main/java/com/qiwenshare/common/upload/Uploader.java @@ -2,12 +2,15 @@ package com.qiwenshare.common.upload; import com.qiwenshare.common.domain.UploadFile; import com.qiwenshare.common.util.PathUtil; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.multipart.support.StandardMultipartHttpServletRequest; import javax.servlet.http.HttpServletRequest; import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.text.SimpleDateFormat; @@ -69,6 +72,27 @@ public abstract class Uploader { } + public synchronized boolean checkUploadStatus(UploadFile param, File confFile) throws IOException { + //File confFile = new File(savePath, timeStampName + ".conf"); + RandomAccessFile confAccessFile = new RandomAccessFile(confFile, "rw"); + //设置文件长度 + confAccessFile.setLength(param.getTotalChunks()); + //设置起始偏移量 + confAccessFile.seek(param.getChunkNumber() - 1); + //将指定的一个字节写入文件中 127， + confAccessFile.write(Byte.MAX_VALUE); + byte[] completeStatusList = FileUtils.readFileToByteArray(confFile); + confAccessFile.close();//不关闭会造成无法占用 + //创建conf文件文件长度为总分片数，每上传一个分块即向conf文件中写入一个127，那么没上传的位置就是默认的0,已上传的就是127 + for (int i = 0; i < completeStatusList.length; i++) { + if (completeStatusList[i] != Byte.MAX_VALUE) { + return false; + } + } + confFile.delete(); + return true; + } + protected String getFileName(String fileName){ return fileName.substring(0, fileName.lastIndexOf(".")); } diff --git a/file-common/src/main/java/com/qiwenshare/common/upload/product/AliyunOSSUploader.java b/file-common/src/main/java/com/qiwenshare/common/upload/product/AliyunOSSUploader.java index 0666369..a866a22 100644 --- a/file-common/src/main/java/com/qiwenshare/common/upload/product/AliyunOSSUploader.java +++ b/file-common/src/main/java/com/qiwenshare/common/upload/product/AliyunOSSUploader.java @@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSON; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClientBuilder; import com.aliyun.oss.model.*; +import com.aliyuncs.utils.StringUtils; import com.qiwenshare.common.domain.AliyunOSS; import com.qiwenshare.common.domain.UploadFile; import com.qiwenshare.common.upload.Uploader; @@ -31,12 +32,19 @@ public class AliyunOSSUploader extends Uploader { private static final Logger logger = LoggerFactory.getLogger(AliyunOSSUploader.class); private UploadFile uploadFile; - public static ExecutorService executorService = Executors.newFixedThreadPool(1); + private String endpoint; + private String accessKeyId; + private String accessKeySecret; + private String bucketName; // partETags是PartETag的集合。PartETag由分片的ETag和分片号组成。 public static Map> partETagsMap = new HashMap>(); public static Map uploadPartRequestMap = new HashMap<>(); + public static Map ossMap = new HashMap<>(); + + public static Map timeStampNameMap = new HashMap<>(); + public AliyunOSSUploader() { } @@ -49,8 +57,14 @@ public class AliyunOSSUploader extends Uploader { public List upload(HttpServletRequest httpServletRequest) { logger.info("开始上传upload"); - List saveUploadFileList = new ArrayList(); + List saveUploadFileList = new ArrayList<>(); this.request = (StandardMultipartHttpServletRequest) httpServletRequest; + AliyunOSS aliyunOSS = (AliyunOSS) request.getAttribute("oss"); + + endpoint = aliyunOSS.getEndpoint(); + accessKeyId = aliyunOSS.getAccessKeyId(); + accessKeySecret = aliyunOSS.getAccessKeySecret(); + bucketName = aliyunOSS.getBucketName(); boolean isMultipart = ServletFileUpload.isMultipartContent(this.request); if (!isMultipart) { UploadFile uploadFile = new UploadFile(); @@ -85,15 +99,9 @@ public class AliyunOSSUploader extends Uploader { } private List doUpload(String savePath, Iterator iter) throws IOException { - AliyunOSS aliyunOSS = (AliyunOSS) request.getAttribute("oss"); + OSS ossClient = getClient(); - String endpoint = aliyunOSS.getEndpoint(); - String accessKeyId = aliyunOSS.getAccessKeyId(); - String accessKeySecret = aliyunOSS.getAccessKeySecret(); - String bucketName = aliyunOSS.getBucketName(); - - OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); - List saveUploadFileList = new ArrayList(); + List saveUploadFileList = new ArrayList<>(); try { MultipartFile multipartfile = this.request.getFile(iter.next()); @@ -112,7 +120,6 @@ public class AliyunOSSUploader extends Uploader { String confFilePath = savePath + FILE_SEPARATOR + uploadFile.getIdentifier() + "." + "conf"; File confFile = new File(PathUtil.getStaticPath() + FILE_SEPARATOR + confFilePath); - synchronized (AliyunOSSUploader.class) { if (uploadPartRequestMap.get(uploadFile.getIdentifier()) == null) { InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, ossFilePath.substring(1)); @@ -146,7 +153,7 @@ public class AliyunOSSUploader extends Uploader { UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest); synchronized (AliyunOSSUploader.class) { - + logger.info("上传结果：" + JSON.toJSONString(uploadPartResult)); if (partETagsMap.get(uploadFile.getIdentifier()) == null) { List partETags = new ArrayList(); partETags.add(uploadPartResult.getPartETag()); @@ -159,25 +166,8 @@ public class AliyunOSSUploader extends Uploader { boolean isComplete = checkUploadStatus(uploadFile, confFile); if (isComplete) { logger.info("分片上传完成"); - List partETags = partETagsMap.get(uploadFile.getIdentifier()); - List partETagsSort = new ArrayList<>(); - for (int i = 0; i < partETags.size(); i++) { - for (int index = 1; index <= partETags.size(); index++) { - if (i + 1 == partETags.get(index - 1).getPartNumber()) { - partETagsSort.add(partETags.get(index - 1)); - } - } + completeMultipartUpload(); - } - CompleteMultipartUploadRequest completeMultipartUploadRequest = - new CompleteMultipartUploadRequest(bucketName, - ossFilePath.substring(1), - uploadFileInfo.getUploadId(), - partETagsSort); - logger.info("----:" + JSON.toJSONString(partETagsSort)); - // 完成上传。 - CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest); - logger.info("----:" + JSON.toJSONString(completeMultipartUploadRequest)); uploadFile.setUrl(ossFilePath); uploadFile.setSuccess(1); uploadFile.setMessage("上传成功"); @@ -188,34 +178,6 @@ public class AliyunOSSUploader extends Uploader { } catch (Exception e) { logger.error("上传出错：" + e); - // 列举已上传的分片，其中uploadId来自于InitiateMultipartUpload返回的结果。 - ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, uploadPartRequestMap.get(uploadFile.getIdentifier()).getKey(), uploadPartRequestMap.get(uploadFile.getIdentifier()).getUploadId()); - // 设置uploadId。 - //listPartsRequest.setUploadId(uploadId); - // 设置分页时每一页中分片数量为100个。默认列举1000个分片。 - listPartsRequest.setMaxParts(100); - // 指定List的起始位置。只有分片号大于此参数值的分片会被列举。 -// listPartsRequest.setPartNumberMarker(1); - PartListing partListing = ossClient.listParts(listPartsRequest); - - for (PartSummary part : partListing.getParts()) { - logger.info("分片号："+part.getPartNumber() + ", 分片数据大小: "+ - part.getSize() + "，分片的ETag:"+part.getETag() - + "， 分片最后修改时间："+ part.getLastModified()); - // 获取分片号。 - part.getPartNumber(); - // 获取分片数据大小。 - part.getSize(); - // 获取分片的ETag。 - part.getETag(); - // 获取分片的最后修改时间。 - part.getLastModified(); - } - AbortMultipartUploadRequest abortMultipartUploadRequest = - new AbortMultipartUploadRequest(bucketName, uploadPartRequestMap.get(uploadFile.getIdentifier()).getKey(), uploadPartRequestMap.get(uploadFile.getIdentifier()).getUploadId()); - ossClient.abortMultipartUpload(abortMultipartUploadRequest); - } finally { - ossClient.shutdown(); } uploadFile.setIsOSS(1); @@ -225,25 +187,87 @@ public class AliyunOSSUploader extends Uploader { return saveUploadFileList; } - public synchronized boolean checkUploadStatus(UploadFile param, File confFile) throws IOException { - //File confFile = new File(savePath, timeStampName + ".conf"); - RandomAccessFile confAccessFile = new RandomAccessFile(confFile, "rw"); - //设置文件长度 - confAccessFile.setLength(param.getTotalChunks()); - //设置起始偏移量 - confAccessFile.seek(param.getChunkNumber() - 1); - //将指定的一个字节写入文件中 127， - confAccessFile.write(Byte.MAX_VALUE); - byte[] completeStatusList = FileUtils.readFileToByteArray(confFile); - confAccessFile.close();//不关闭会造成无法占用 - //创建conf文件文件长度为总分片数，每上传一个分块即向conf文件中写入一个127，那么没上传的位置就是默认的0,已上传的就是127 - for (int i = 0; i < completeStatusList.length; i++) { - if (completeStatusList[i] != Byte.MAX_VALUE) { - return false; - } + /** + * 将文件分块进行升序排序并执行文件上传。 + */ + protected void completeMultipartUpload() { + + List partETags = partETagsMap.get(uploadFile.getIdentifier()); + Collections.sort(partETags, Comparator.comparingInt(PartETag::getPartNumber)); + UploadFileInfo uploadFileInfo = uploadPartRequestMap.get(uploadFile.getIdentifier()); + CompleteMultipartUploadRequest completeMultipartUploadRequest = + new CompleteMultipartUploadRequest(bucketName, + uploadFileInfo.getKey(), + uploadFileInfo.getUploadId(), + partETags); + logger.info("----:" + JSON.toJSONString(partETags)); + // 完成上传。 + CompleteMultipartUploadResult completeMultipartUploadResult = getClient().completeMultipartUpload(completeMultipartUploadRequest); + logger.info("----:" + JSON.toJSONString(completeMultipartUploadRequest)); + getClient().shutdown(); + partETagsMap.remove(uploadFile.getIdentifier()); + uploadPartRequestMap.remove(uploadFile.getIdentifier()); + ossMap.remove(uploadFile.getIdentifier()); +// + } + + private void listFile() { + // 列举已上传的分片，其中uploadId来自于InitiateMultipartUpload返回的结果。 + ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, uploadPartRequestMap.get(uploadFile.getIdentifier()).getKey(), uploadPartRequestMap.get(uploadFile.getIdentifier()).getUploadId()); + // 设置uploadId。 + //listPartsRequest.setUploadId(uploadId); + // 设置分页时每一页中分片数量为100个。默认列举1000个分片。 + listPartsRequest.setMaxParts(100); + // 指定List的起始位置。只有分片号大于此参数值的分片会被列举。 +// listPartsRequest.setPartNumberMarker(1); + PartListing partListing = getClient().listParts(listPartsRequest); + + for (PartSummary part : partListing.getParts()) { + logger.info("分片号："+part.getPartNumber() + ", 分片数据大小: "+ + part.getSize() + "，分片的ETag:"+part.getETag() + + "， 分片最后修改时间："+ part.getLastModified()); + // 获取分片号。 + part.getPartNumber(); + // 获取分片数据大小。 + part.getSize(); + // 获取分片的ETag。 + part.getETag(); + // 获取分片的最后修改时间。 + part.getLastModified(); } - confFile.delete(); - return true; + + } + + /** + * 取消上传 + */ + private void cancelUpload() { + AbortMultipartUploadRequest abortMultipartUploadRequest = + new AbortMultipartUploadRequest(bucketName, uploadPartRequestMap.get(uploadFile.getIdentifier()).getKey(), uploadPartRequestMap.get(uploadFile.getIdentifier()).getUploadId()); + getClient().abortMultipartUpload(abortMultipartUploadRequest); + } + + protected synchronized String getTimeStampName(){ + String timeStampName; + + if (StringUtils.isEmpty(timeStampNameMap.get(uploadFile.getIdentifier()))) { + timeStampName = super.getTimeStampName(); + timeStampNameMap.put(uploadFile.getIdentifier(), timeStampName); + } else { + timeStampName = timeStampNameMap.get(uploadFile.getIdentifier()); + } + return timeStampName; + } + + private synchronized OSS getClient() { + OSS ossClient = null; + if (ossMap.get(uploadFile.getIdentifier()) == null) { + ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); + ossMap.put(uploadFile.getIdentifier(), ossClient); + } else { + ossClient = ossMap.get(uploadFile.getIdentifier()); + } + return ossClient; } @Data diff --git a/file-common/src/main/java/com/qiwenshare/common/upload/product/ChunkUploader.java b/file-common/src/main/java/com/qiwenshare/common/upload/product/ChunkUploader.java index 50b0efa..13bb6e8 100644 --- a/file-common/src/main/java/com/qiwenshare/common/upload/product/ChunkUploader.java +++ b/file-common/src/main/java/com/qiwenshare/common/upload/product/ChunkUploader.java @@ -77,27 +77,6 @@ public class ChunkUploader extends Uploader { } - public boolean checkUploadStatus(UploadFile param, File confFile) throws IOException { - //File confFile = new File(savePath, timeStampName + ".conf"); - RandomAccessFile confAccessFile = new RandomAccessFile(confFile, "rw"); - //设置文件长度 - confAccessFile.setLength(param.getTotalChunks()); - //设置起始偏移量 - confAccessFile.seek(param.getChunkNumber() - 1); - //将指定的一个字节写入文件中 127， - confAccessFile.write(Byte.MAX_VALUE); - byte[] completeStatusList = FileUtils.readFileToByteArray(confFile); - confAccessFile.close();//不关闭会造成无法占用 - //创建conf文件文件长度为总分片数，每上传一个分块即向conf文件中写入一个127，那么没上传的位置就是默认的0,已上传的就是127 - for (int i = 0; i < completeStatusList.length; i++) { - if (completeStatusList[i] != Byte.MAX_VALUE) { - return false; - } - } - confFile.delete(); - return true; - } - private List doUpload(String savePath, Iterator iter) throws IOException, NotSameFileExpection { List saveUploadFileList = new ArrayList(); //UploadFile uploadFile = new UploadFile(); @@ -168,7 +147,7 @@ public class ChunkUploader extends Uploader { uploadFile.setSuccess(0); uploadFile.setMessage("未完成"); } - uploadFile.setFileSize(uploadFile.getCurrentChunkSize()); + uploadFile.setFileSize(uploadFile.getTotalSize()); saveUploadFileList.add(uploadFile); return saveUploadFileList; diff --git a/file-common/src/main/java/com/qiwenshare/common/util/JjwtUtil.java b/file-common/src/main/java/com/qiwenshare/common/util/JjwtUtil.java index ef32233..bbfdafa 100644 --- a/file-common/src/main/java/com/qiwenshare/common/util/JjwtUtil.java +++ b/file-common/src/main/java/com/qiwenshare/common/util/JjwtUtil.java @@ -1,6 +1,5 @@ package com.qiwenshare.common.util; -import com.alibaba.fastjson.JSON; import io.jsonwebtoken.Claims; import io.jsonwebtoken.JwtBuilder; import io.jsonwebtoken.Jwts; @@ -9,7 +8,6 @@ import org.apache.commons.net.util.Base64; import javax.crypto.SecretKey; import javax.crypto.spec.SecretKeySpec; -import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -24,7 +22,7 @@ public class JjwtUtil { public static final String JWT_SECRET = "jiamimiwen"; // 过期时间，单位毫秒 - public static final int EXPIRE_TIME = 60 * 60 * 1000; // 一个小时 + public static final int EXPIRE_TIME = 60 * 60 * 1000 * 24 * 7; // 一个星期 // public static final long EXPIRE_TIME = 7 * 24 * 3600 * 1000; // 一个星期 // 由字符串生成加密key @@ -89,18 +87,6 @@ public class JjwtUtil { .parseClaimsJws(jwt).getBody(); // 设置需要解析的jwt return claims; } - - public static void main(String[] args) { - Claims claims = null; - try { - claims = parseJWT("eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ7XCJhZGRyQXJlYVwiOlwi6ZuB5aGU5Yy6XCIsXCJhZGRyQ2l0eVwiOlwi6KW_5a6J5biCXCIsXCJhZGRyUHJvdmluY2VcIjpcIumZleilv-ecgVwiLFwiYmVnaW5Db3VudFwiOjAsXCJiaXJ0aGRheVwiOlwiMTk5NC0wNS0wNlwiLFwiY3VycmVudFBhZ2VcIjowLFwiZWRpdG9yVHlwZVwiOjAsXCJlbWFpbFwiOlwiMTE2MjcxNDQ4M0BxcS5jb21cIixcImVzc2F5Q291bnRcIjowLFwiaW1hZ2VVcmxcIjpcIi91cGxvYWQvMjAyMDA3MTUvODA5NjE1OTQ4MjEzODkwNTkuanBnXCIsXCJpbmR1c3RyeVwiOlwi6K6h566X5py66KGM5LiaXCIsXCJpbnRyb1wiOlwi6ZSZ5oqK6ZmI6YaL5b2T5oiQ5aKo77yM5YaZ5bC95Y2K55Sf6YO95piv6YW444CCXCIsXCJsYXN0TG9naW5UaW1lXCI6XCIyMDIwLTEwLTI3IDE5OjAxOjU0XCIsXCJub3RSZWFkQ291bnRcIjowLFwicGFnZUNvdW50XCI6MCxcInBhc3N3b3JkXCI6XCI3NjcyYmE0NDc4OTY5NTlmNmU1NDQ1NTcxOGY2ZWU2ZlwiLFwic2FsdFwiOlwiMzY5MjQ1NzgxMDIyMDg3NlwiLFwic2V4XCI6XCLnlLdcIixcInRlbGVwaG9uZVwiOlwiMTg4MjkyOTE4MTdcIixcInVzZXJJZFwiOjIsXCJ1c2VybmFtZVwiOlwiTUFDXCJ9IiwiYXVkIjoicWl3ZW4iLCJwYXNzd29yZCI6IjAxMDIwMyIsImlzcyI6InFpd2Vuc2hhcmUiLCJleHAiOjE2MDM4MDA0MDcsImlhdCI6MTYwMzc5NjgwNywianRpIjoiNzYzZTE0NDMtYmZjYS00MDkzLTg3NGYtMzlmMzQ1ODRiZDkyIiwidXNlcm5hbWUiOiJhZG1pbiJ9.Gv_QlXZEiT_-mbI2Iz6oa2SqgBXwVzsS_xK8RhvF7oY"); - } catch (Exception e) { - e.printStackTrace(); - } - String subject = claims.getSubject(); - System.out.println(subject); - } - } \ No newline at end of file