ak/file
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fdfs上传代码优化

Browse Source
This commit is contained in:
马超 2021-03-19 23:10:45 +08:00
parent e1abac566c
commit 48cdbb8e0f
2 changed files with 69 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
file-common/src/main/java/com/qiwenshare/common/upload/product/FastDFSUploader.java
View File

@ -10,17 +10,16 @@ import com.qiwenshare.common.util.PathUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.support.StandardMultipartHttpServletRequest;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.util.*;
@Slf4j
public class FastDFSUploader extends Uploader {
public static Object lock = new Object();
AppendFileStorageClient defaultAppendFileStorageClient;
UploadFile uploadFile;
@ -28,16 +27,12 @@ public class FastDFSUploader extends Uploader {
private static Map<String, Integer> CURRENT_UPLOAD_CHUNK_NUMBER = new HashMap<>();
private static Map<String, Long> UPLOADED_SIZE = new HashMap<>();
private static Map<String, String> STORE_PATH = new HashMap<>();
private static Map<String, Boolean> LOCK_MAP = new HashMap<>();
private static Map<String, Object> LOCK_MAP = new HashMap<>();
public FastDFSUploader() {
}
public FastDFSUploader(UploadFile uploadFile) {
this.uploadFile = uploadFile;
}
public FastDFSUploader(UploadFile uploadFile, AppendFileStorageClient defaultAppendFileStorageClient) {
this.uploadFile = uploadFile;
this.defaultAppendFileStorageClient = defaultAppendFileStorageClient;
@ -76,13 +71,17 @@ public class FastDFSUploader extends Uploader {
}
private List<UploadFile> doUpload(String savePath, Iterator<String> iter){
private List<UploadFile> doUpload(String savePath, Iterator<String> iter) {
List<UploadFile> saveUploadFileList = new ArrayList<>();
try {
MultipartFile multipartfile = this.request.getFile(iter.next());
synchronized (lock) {
if (LOCK_MAP.get(uploadFile.getIdentifier()) == null) {
LOCK_MAP.put(uploadFile.getIdentifier(), new Object());
}
}
uploadFileChunk(multipartfile);
String timeStampName = getTimeStampName();
@ -100,11 +99,10 @@ public class FastDFSUploader extends Uploader {
File confFile = new File(PathUtil.getStaticPath() + FILE_SEPARATOR + confFilePath);
boolean isComplete = checkUploadStatus(uploadFile, confFile);
if (isComplete) {
log.info("分片上传完成");
LOCK_MAP.remove(uploadFile.getIdentifier());
uploadFile.setUrl(STORE_PATH.get(uploadFile.getIdentifier()));
uploadFile.setSuccess(1);
uploadFile.setMessage("上传成功");
@ -126,40 +124,23 @@ public class FastDFSUploader extends Uploader {
public void uploadFileChunk(MultipartFile multipartFile) {
// 存储在fastdfs不带组的路径
// String noGroupPath = "";
log.info("当前文件的Md5:{}", uploadFile.getIdentifier());
synchronized (LOCK_MAP.get(uploadFile.getIdentifier())) {
// 存储在fastdfs不带组的路径
// 真正的拥有者
boolean currOwner = false;
try {
Boolean lock = LOCK_MAP.get(uploadFile.getIdentifier());
if (lock != null && lock) {
throw new UploadGeneralException("请求块锁失败");
}
LOCK_MAP.put(uploadFile.getIdentifier(), true);
// 写入锁的当前拥有者
currOwner = true;
// redis中记录当前应该传第几块(从0开始)
Integer currentChunkInRedis = CURRENT_UPLOAD_CHUNK_NUMBER.get(uploadFile.getIdentifier());
log.info("当前文件的Md5:{}", uploadFile.getIdentifier());
log.info("当前块的大小:{}", uploadFile.getCurrentChunkSize());
if (currentChunkInRedis == null) {
currentChunkInRedis = 1;
if (CURRENT_UPLOAD_CHUNK_NUMBER.get(uploadFile.getIdentifier()) == null) {
CURRENT_UPLOAD_CHUNK_NUMBER.put(uploadFile.getIdentifier(), 1);
}
//此段代码保证顺序，如果满足条件则返回失败
if (uploadFile.getChunkNumber() < currentChunkInRedis) {
log.info("当前文件块已上传");
throw new UploadGeneralException("当前文件块已上传");
// return false;
} else if (uploadFile.getChunkNumber() > currentChunkInRedis) {
log.info("当前文件块需要等待上传,稍后请重试");
throw new UploadGeneralException("当前文件块需要等待上传,稍后请重试");
// return false;
while (uploadFile.getChunkNumber() != CURRENT_UPLOAD_CHUNK_NUMBER.get(uploadFile.getIdentifier())) {
try {
LOCK_MAP.get(uploadFile.getIdentifier()).wait();
} catch (InterruptedException e) {
log.error("--------InterruptedException-------");
e.printStackTrace();
}
}
log.info("***********开始上传第{}块**********", uploadFile.getChunkNumber());
@ -180,13 +161,11 @@ public class FastDFSUploader extends Uploader {
CURRENT_UPLOAD_CHUNK_NUMBER.put(uploadFile.getIdentifier(), uploadFile.getChunkNumber());
log.info("获取远程文件路径出错");
throw new UploadGeneralException("获取远程文件路径出错");
// return false;
}
} catch (Exception e) {
CURRENT_UPLOAD_CHUNK_NUMBER.put(uploadFile.getIdentifier(), uploadFile.getChunkNumber());
log.error("初次上传远程文件出错", e);
throw new UploadGeneralException("初次上传远程文件出错", e);
// return false;
}
STORE_PATH.put(uploadFile.getIdentifier(), storePath.getPath());
@ -199,7 +178,6 @@ public class FastDFSUploader extends Uploader {
log.error("无法获取已上传服务器文件地址");
CURRENT_UPLOAD_CHUNK_NUMBER.put(uploadFile.getIdentifier(), uploadFile.getChunkNumber());
throw new UploadGeneralException("无法获取已上传服务器文件地址");
// return false;
}
try {
Long alreadySize = UPLOADED_SIZE.get(uploadFile.getIdentifier());
@ -213,20 +191,16 @@ public class FastDFSUploader extends Uploader {
CURRENT_UPLOAD_CHUNK_NUMBER.put(uploadFile.getIdentifier(), uploadFile.getChunkNumber());
log.error("更新远程文件出错", e);
throw new UploadGeneralException("更新远程文件出错", e);
// return false;
}
}
} catch (Exception e) {
log.error("上传文件错误", e);
throw new UploadGeneralException("上传文件错误", e);
// return false;
}
} finally {
// 锁的当前拥有者才能释放块上传锁
if (currOwner) {
LOCK_MAP.put(uploadFile.getIdentifier(), false);
}
log.info("***********第{}块上传成功**********", uploadFile.getChunkNumber());
LOCK_MAP.get(uploadFile.getIdentifier()).notifyAll();
}
log.info("***********第{}块上传成功**********", uploadFile.getChunkNumber());
}
}

81
file-web/src/main/java/com/qiwenshare/file/controller/FiletransferController.java
View File

@ -136,7 +136,7 @@ public class FiletransferController {
@Operation(summary = "下载文件", description = "下载文件接口", tags = {"filetransfer"})
@MyLog(operation = "下载文件", module = CURRENT_MODULE)
@RequestMapping(value = "/downloadfile", method = RequestMethod.GET)
public String downloadFile(HttpServletResponse response, DownloadFileDTO downloadFileDTO) {
public void downloadFile(HttpServletResponse response, DownloadFileDTO downloadFileDTO) {
UserFile userFile = userFileService.getById(downloadFileDTO.getUserFileId());
String fileName = userFile.getFileName() + "." + userFile.getExtendName();
@ -148,23 +148,36 @@ public class FiletransferController {
response.setContentType("application/force-download");// 设置强制下载不打开
response.addHeader("Content-Disposition", "attachment;fileName=" + fileName);// 设置文件名
byte[] buffer = new byte[1024];
BufferedInputStream bis = null;
FileBean fileBean = fileService.getById(userFile.getFileId());
if (fileBean.getIsOSS() != null && fileBean.getIsOSS() == 1) {
aliyunDownload(response, buffer, fileBean);
} else {
localFileDownload(response, buffer, fileBean);
}
}
private void localFileDownload(HttpServletResponse response, byte[] buffer, FileBean fileBean) {
BufferedInputStream bis = null;
//设置文件路径
File file = FileOperation.newFile(PathUtil.getStaticPath() + fileBean.getFileUrl());
if (file.exists()) {
FileInputStream fis = null;
AliyunOSSDownload aliyunOSSDownload= new AliyunOSSDownload();
OSS ossClient = aliyunOSSDownload.createOSSClient(qiwenFileConfig.getAliyun().getOss());
OSSObject ossObject = ossClient.getObject(qiwenFileConfig.getAliyun().getOss().getBucketName(), fileBean.getTimeStampName());
InputStream inputStream = ossObject.getObjectContent();
try {
bis = new BufferedInputStream(inputStream);
fis = new FileInputStream(file);
bis = new BufferedInputStream(fis);
OutputStream os = response.getOutputStream();
int i = bis.read(buffer);
while (i != -1) {
os.write(buffer, 0, i);
i = bis.read(buffer);
}
} catch (IOException e) {
} catch (Exception e) {
e.printStackTrace();
} finally {
if (bis != null) {
@ -174,43 +187,37 @@ public class FiletransferController {
e.printStackTrace();
}
}
}
ossClient.shutdown();
} else {
//设置文件路径
File file = FileOperation.newFile(PathUtil.getStaticPath() + fileBean.getFileUrl());
if (file.exists()) {
FileInputStream fis = null;
}
}
private void aliyunDownload(HttpServletResponse response, byte[] buffer, FileBean fileBean) {
BufferedInputStream bis = null;
AliyunOSSDownload aliyunOSSDownload= new AliyunOSSDownload();
OSS ossClient = aliyunOSSDownload.createOSSClient(qiwenFileConfig.getAliyun().getOss());
OSSObject ossObject = ossClient.getObject(qiwenFileConfig.getAliyun().getOss().getBucketName(), fileBean.getTimeStampName());
InputStream inputStream = ossObject.getObjectContent();
try {
bis = new BufferedInputStream(inputStream);
OutputStream os = response.getOutputStream();
int i = bis.read(buffer);
while (i != -1) {
os.write(buffer, 0, i);
i = bis.read(buffer);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (bis != null) {
try {
fis = new FileInputStream(file);
bis = new BufferedInputStream(fis);
OutputStream os = response.getOutputStream();
int i = bis.read(buffer);
while (i != -1) {
os.write(buffer, 0, i);
i = bis.read(buffer);
}
} catch (Exception e) {
bis.close();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (bis != null) {
try {
bis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
return null;
ossClient.shutdown();
}
@Operation(summary = "获取存储信息", description = "获取存储信息", tags = {"filetransfer"})