ak/file
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fdfs上传代码优化

Browse Source
This commit is contained in:
马超 2021-03-19 23:10:45 +08:00
parent e1abac566c
commit 48cdbb8e0f
2 changed files with 69 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
file-common/src/main/java/com/qiwenshare/common/upload/product/FastDFSUploader.java
View File

@ -10,17 +10,16 @@ import com.qiwenshare.common.util.PathUtil;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.fileupload.disk.DiskFileItemFactory; import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.support.StandardMultipartHttpServletRequest; import org.springframework.web.multipart.support.StandardMultipartHttpServletRequest;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import java.io.File; import java.io.File;
import java.util.*; import java.util.*;
@Slf4j @Slf4j
public class FastDFSUploader extends Uploader { public class FastDFSUploader extends Uploader {
public static Object lock = new Object();
AppendFileStorageClient defaultAppendFileStorageClient; AppendFileStorageClient defaultAppendFileStorageClient;
UploadFile uploadFile; UploadFile uploadFile;
@ -28,16 +27,12 @@ public class FastDFSUploader extends Uploader {
private static Map<String, Integer> CURRENT_UPLOAD_CHUNK_NUMBER = new HashMap<>(); private static Map<String, Integer> CURRENT_UPLOAD_CHUNK_NUMBER = new HashMap<>();
private static Map<String, Long> UPLOADED_SIZE = new HashMap<>(); private static Map<String, Long> UPLOADED_SIZE = new HashMap<>();
private static Map<String, String> STORE_PATH = new HashMap<>(); private static Map<String, String> STORE_PATH = new HashMap<>();
private static Map<String, Boolean> LOCK_MAP = new HashMap<>(); private static Map<String, Object> LOCK_MAP = new HashMap<>();
public FastDFSUploader() { public FastDFSUploader() {
} }
public FastDFSUploader(UploadFile uploadFile) {
this.uploadFile = uploadFile;
}
public FastDFSUploader(UploadFile uploadFile, AppendFileStorageClient defaultAppendFileStorageClient) { public FastDFSUploader(UploadFile uploadFile, AppendFileStorageClient defaultAppendFileStorageClient) {
this.uploadFile = uploadFile; this.uploadFile = uploadFile;
this.defaultAppendFileStorageClient = defaultAppendFileStorageClient; this.defaultAppendFileStorageClient = defaultAppendFileStorageClient;
@ -76,13 +71,17 @@ public class FastDFSUploader extends Uploader {
} }
private List<UploadFile> doUpload(String savePath, Iterator<String> iter) {
private List<UploadFile> doUpload(String savePath, Iterator<String> iter){
List<UploadFile> saveUploadFileList = new ArrayList<>(); List<UploadFile> saveUploadFileList = new ArrayList<>();
try { try {
MultipartFile multipartfile = this.request.getFile(iter.next()); MultipartFile multipartfile = this.request.getFile(iter.next());
synchronized (lock) {
if (LOCK_MAP.get(uploadFile.getIdentifier()) == null) {
LOCK_MAP.put(uploadFile.getIdentifier(), new Object());
}
}
uploadFileChunk(multipartfile); uploadFileChunk(multipartfile);
String timeStampName = getTimeStampName(); String timeStampName = getTimeStampName();
@ -100,11 +99,10 @@ public class FastDFSUploader extends Uploader {
File confFile = new File(PathUtil.getStaticPath() + FILE_SEPARATOR + confFilePath); File confFile = new File(PathUtil.getStaticPath() + FILE_SEPARATOR + confFilePath);
boolean isComplete = checkUploadStatus(uploadFile, confFile); boolean isComplete = checkUploadStatus(uploadFile, confFile);
if (isComplete) { if (isComplete) {
log.info("分片上传完成"); log.info("分片上传完成");
LOCK_MAP.remove(uploadFile.getIdentifier());
uploadFile.setUrl(STORE_PATH.get(uploadFile.getIdentifier())); uploadFile.setUrl(STORE_PATH.get(uploadFile.getIdentifier()));
uploadFile.setSuccess(1); uploadFile.setSuccess(1);
uploadFile.setMessage("上传成功"); uploadFile.setMessage("上传成功");
@ -126,40 +124,23 @@ public class FastDFSUploader extends Uploader {
public void uploadFileChunk(MultipartFile multipartFile) { public void uploadFileChunk(MultipartFile multipartFile) {
// 存储在fastdfs不带组的路径 synchronized (LOCK_MAP.get(uploadFile.getIdentifier())) {
// String noGroupPath = ""; // 存储在fastdfs不带组的路径
log.info("当前文件的Md5:{}", uploadFile.getIdentifier());
// 真正的拥有者 log.info("当前文件的Md5:{}", uploadFile.getIdentifier());
boolean currOwner = false;
try {
Boolean lock = LOCK_MAP.get(uploadFile.getIdentifier());
if (lock != null && lock) {
throw new UploadGeneralException("请求块锁失败");
}
LOCK_MAP.put(uploadFile.getIdentifier(), true);
// 写入锁的当前拥有者
currOwner = true;
// redis中记录当前应该传第几块(从0开始)
Integer currentChunkInRedis = CURRENT_UPLOAD_CHUNK_NUMBER.get(uploadFile.getIdentifier());
log.info("当前块的大小:{}", uploadFile.getCurrentChunkSize()); log.info("当前块的大小:{}", uploadFile.getCurrentChunkSize());
if (currentChunkInRedis == null) { if (CURRENT_UPLOAD_CHUNK_NUMBER.get(uploadFile.getIdentifier()) == null) {
currentChunkInRedis = 1; CURRENT_UPLOAD_CHUNK_NUMBER.put(uploadFile.getIdentifier(), 1);
} }
//此段代码保证顺序，如果满足条件则返回失败 while (uploadFile.getChunkNumber() != CURRENT_UPLOAD_CHUNK_NUMBER.get(uploadFile.getIdentifier())) {
if (uploadFile.getChunkNumber() < currentChunkInRedis) { try {
log.info("当前文件块已上传"); LOCK_MAP.get(uploadFile.getIdentifier()).wait();
throw new UploadGeneralException("当前文件块已上传"); } catch (InterruptedException e) {
// return false; log.error("--------InterruptedException-------");
} else if (uploadFile.getChunkNumber() > currentChunkInRedis) { e.printStackTrace();
log.info("当前文件块需要等待上传,稍后请重试"); }
throw new UploadGeneralException("当前文件块需要等待上传,稍后请重试");
// return false;
} }
log.info("***********开始上传第{}块**********", uploadFile.getChunkNumber()); log.info("***********开始上传第{}块**********", uploadFile.getChunkNumber());
@ -180,13 +161,11 @@ public class FastDFSUploader extends Uploader {
CURRENT_UPLOAD_CHUNK_NUMBER.put(uploadFile.getIdentifier(), uploadFile.getChunkNumber()); CURRENT_UPLOAD_CHUNK_NUMBER.put(uploadFile.getIdentifier(), uploadFile.getChunkNumber());
log.info("获取远程文件路径出错"); log.info("获取远程文件路径出错");
throw new UploadGeneralException("获取远程文件路径出错"); throw new UploadGeneralException("获取远程文件路径出错");
// return false;
} }
} catch (Exception e) { } catch (Exception e) {
CURRENT_UPLOAD_CHUNK_NUMBER.put(uploadFile.getIdentifier(), uploadFile.getChunkNumber()); CURRENT_UPLOAD_CHUNK_NUMBER.put(uploadFile.getIdentifier(), uploadFile.getChunkNumber());
log.error("初次上传远程文件出错", e); log.error("初次上传远程文件出错", e);
throw new UploadGeneralException("初次上传远程文件出错", e); throw new UploadGeneralException("初次上传远程文件出错", e);
// return false;
} }
STORE_PATH.put(uploadFile.getIdentifier(), storePath.getPath()); STORE_PATH.put(uploadFile.getIdentifier(), storePath.getPath());
@ -199,7 +178,6 @@ public class FastDFSUploader extends Uploader {
log.error("无法获取已上传服务器文件地址"); log.error("无法获取已上传服务器文件地址");
CURRENT_UPLOAD_CHUNK_NUMBER.put(uploadFile.getIdentifier(), uploadFile.getChunkNumber()); CURRENT_UPLOAD_CHUNK_NUMBER.put(uploadFile.getIdentifier(), uploadFile.getChunkNumber());
throw new UploadGeneralException("无法获取已上传服务器文件地址"); throw new UploadGeneralException("无法获取已上传服务器文件地址");
// return false;
} }
try { try {
Long alreadySize = UPLOADED_SIZE.get(uploadFile.getIdentifier()); Long alreadySize = UPLOADED_SIZE.get(uploadFile.getIdentifier());
@ -213,20 +191,16 @@ public class FastDFSUploader extends Uploader {
CURRENT_UPLOAD_CHUNK_NUMBER.put(uploadFile.getIdentifier(), uploadFile.getChunkNumber()); CURRENT_UPLOAD_CHUNK_NUMBER.put(uploadFile.getIdentifier(), uploadFile.getChunkNumber());
log.error("更新远程文件出错", e); log.error("更新远程文件出错", e);
throw new UploadGeneralException("更新远程文件出错", e); throw new UploadGeneralException("更新远程文件出错", e);
// return false;
} }
} }
} catch (Exception e) { } catch (Exception e) {
log.error("上传文件错误", e); log.error("上传文件错误", e);
throw new UploadGeneralException("上传文件错误", e); throw new UploadGeneralException("上传文件错误", e);
// return false;
}
} finally {
// 锁的当前拥有者才能释放块上传锁
if (currOwner) {
LOCK_MAP.put(uploadFile.getIdentifier(), false);
} }
log.info("***********第{}块上传成功**********", uploadFile.getChunkNumber());
LOCK_MAP.get(uploadFile.getIdentifier()).notifyAll();
} }
log.info("***********第{}块上传成功**********", uploadFile.getChunkNumber());
} }
} }

81
file-web/src/main/java/com/qiwenshare/file/controller/FiletransferController.java
View File

@ -136,7 +136,7 @@ public class FiletransferController {
@Operation(summary = "下载文件", description = "下载文件接口", tags = {"filetransfer"}) @Operation(summary = "下载文件", description = "下载文件接口", tags = {"filetransfer"})
@MyLog(operation = "下载文件", module = CURRENT_MODULE) @MyLog(operation = "下载文件", module = CURRENT_MODULE)
@RequestMapping(value = "/downloadfile", method = RequestMethod.GET) @RequestMapping(value = "/downloadfile", method = RequestMethod.GET)
public String downloadFile(HttpServletResponse response, DownloadFileDTO downloadFileDTO) { public void downloadFile(HttpServletResponse response, DownloadFileDTO downloadFileDTO) {
UserFile userFile = userFileService.getById(downloadFileDTO.getUserFileId()); UserFile userFile = userFileService.getById(downloadFileDTO.getUserFileId());
String fileName = userFile.getFileName() + "." + userFile.getExtendName(); String fileName = userFile.getFileName() + "." + userFile.getExtendName();
@ -148,23 +148,36 @@ public class FiletransferController {
response.setContentType("application/force-download");// 设置强制下载不打开 response.setContentType("application/force-download");// 设置强制下载不打开
response.addHeader("Content-Disposition", "attachment;fileName=" + fileName);// 设置文件名 response.addHeader("Content-Disposition", "attachment;fileName=" + fileName);// 设置文件名
byte[] buffer = new byte[1024]; byte[] buffer = new byte[1024];
BufferedInputStream bis = null;
FileBean fileBean = fileService.getById(userFile.getFileId()); FileBean fileBean = fileService.getById(userFile.getFileId());
if (fileBean.getIsOSS() != null && fileBean.getIsOSS() == 1) { if (fileBean.getIsOSS() != null && fileBean.getIsOSS() == 1) {
aliyunDownload(response, buffer, fileBean);
} else {
localFileDownload(response, buffer, fileBean);
}
}
private void localFileDownload(HttpServletResponse response, byte[] buffer, FileBean fileBean) {
BufferedInputStream bis = null;
//设置文件路径
File file = FileOperation.newFile(PathUtil.getStaticPath() + fileBean.getFileUrl());
if (file.exists()) {
FileInputStream fis = null;
AliyunOSSDownload aliyunOSSDownload= new AliyunOSSDownload();
OSS ossClient = aliyunOSSDownload.createOSSClient(qiwenFileConfig.getAliyun().getOss());
OSSObject ossObject = ossClient.getObject(qiwenFileConfig.getAliyun().getOss().getBucketName(), fileBean.getTimeStampName());
InputStream inputStream = ossObject.getObjectContent();
try { try {
bis = new BufferedInputStream(inputStream); fis = new FileInputStream(file);
bis = new BufferedInputStream(fis);
OutputStream os = response.getOutputStream(); OutputStream os = response.getOutputStream();
int i = bis.read(buffer); int i = bis.read(buffer);
while (i != -1) { while (i != -1) {
os.write(buffer, 0, i); os.write(buffer, 0, i);
i = bis.read(buffer); i = bis.read(buffer);
} }
} catch (IOException e) {
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} finally { } finally {
if (bis != null) { if (bis != null) {
@ -174,43 +187,37 @@ public class FiletransferController {
e.printStackTrace(); e.printStackTrace();
} }
} }
} }
ossClient.shutdown(); }
} else { }
//设置文件路径
File file = FileOperation.newFile(PathUtil.getStaticPath() + fileBean.getFileUrl());
if (file.exists()) {
FileInputStream fis = null;
private void aliyunDownload(HttpServletResponse response, byte[] buffer, FileBean fileBean) {
BufferedInputStream bis = null;
AliyunOSSDownload aliyunOSSDownload= new AliyunOSSDownload();
OSS ossClient = aliyunOSSDownload.createOSSClient(qiwenFileConfig.getAliyun().getOss());
OSSObject ossObject = ossClient.getObject(qiwenFileConfig.getAliyun().getOss().getBucketName(), fileBean.getTimeStampName());
InputStream inputStream = ossObject.getObjectContent();
try {
bis = new BufferedInputStream(inputStream);
OutputStream os = response.getOutputStream();
int i = bis.read(buffer);
while (i != -1) {
os.write(buffer, 0, i);
i = bis.read(buffer);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (bis != null) {
try { try {
fis = new FileInputStream(file); bis.close();
bis = new BufferedInputStream(fis); } catch (IOException e) {
OutputStream os = response.getOutputStream();
int i = bis.read(buffer);
while (i != -1) {
os.write(buffer, 0, i);
i = bis.read(buffer);
}
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} finally {
if (bis != null) {
try {
bis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
} }
} }
} }
return null; ossClient.shutdown();
} }
@Operation(summary = "获取存储信息", description = "获取存储信息", tags = {"filetransfer"}) @Operation(summary = "获取存储信息", description = "获取存储信息", tags = {"filetransfer"})