新增文件搜索功能

2021-03-20 17:05:55 +08:00 · 2021-03-20 17:05:55 +08:00 · 98a4ebe8f1
commit 98a4ebe8f1
parent 48cdbb8e0f
18 changed files with 279 additions and 37 deletions
--- a/file-common/pom.xml
+++ b/file-common/pom.xml
@ -50,6 +50,13 @@
            <version>1.26.2</version>
        </dependency>

+        <!--redis-->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-redis</artifactId>
+        </dependency>
+
+
    </dependencies>


--- a/file-common/src/main/java/com/qiwenshare/common/config/RedisAutoConfiguration.java
+++ b/file-common/src/main/java/com/qiwenshare/common/config/RedisAutoConfiguration.java
@ -0,0 +1,37 @@
+package com.qiwenshare.common.config;
+
+
+import org.springframework.cache.annotation.CachingConfigurerSupport;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.cache.interceptor.KeyGenerator;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@EnableCaching
+public class RedisAutoConfiguration extends CachingConfigurerSupport {
+
+    @Bean
+    public KeyGenerator keyGenerator() {
+        return (target, method, params) -> {
+            StringBuilder sb = new StringBuilder();
+            sb.append(target.getClass().getName());
+            sb.append(method.getName());
+            for (Object obj : params) {
+                sb.append(obj.toString());
+            }
+            return sb.toString();
+        };
+    }
+
+//    @Bean
+//    public CacheManager cacheManager(RedisConnectionFactory redisConnectionFactory) {
+//        RedisCacheConfiguration redisCacheConfiguration = RedisCacheConfiguration.defaultCacheConfig()
+//                .entryTtl(Duration.ofHours(1)); // 设置缓存有效期一小时
+//        return RedisCacheManager
+//                .builder(RedisCacheWriter.nonLockingRedisCacheWriter(redisConnectionFactory))
+//                .cacheDefaults(redisCacheConfiguration).build();
+//    }
+
+}
+
--- a/file-web/pom.xml
+++ b/file-web/pom.xml
@ -73,12 +73,10 @@
            <version>4.13</version>
            <scope>test</scope>
        </dependency>
-
-<!--        <dependency>-->
-<!--            <groupId>com.qiwenshare</groupId>-->
-<!--            <artifactId>file-office</artifactId>-->
-<!--            <version>1.0.0-SNAPSHOT</version>-->
-<!--        </dependency>-->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
+        </dependency>

    </dependencies>

--- a/file-web/src/main/java/com/qiwenshare/file/api/IElasticSearchService.java
+++ b/file-web/src/main/java/com/qiwenshare/file/api/IElasticSearchService.java
@ -0,0 +1,8 @@
+package com.qiwenshare.file.api;
+
+import com.qiwenshare.file.config.es.FileSearch;
+import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
+
+public interface IElasticSearchService extends ElasticsearchRepository<FileSearch,Long> {
+
+}
--- a/file-web/src/main/java/com/qiwenshare/file/config/es/FileSearch.java
+++ b/file-web/src/main/java/com/qiwenshare/file/config/es/FileSearch.java
@ -0,0 +1,49 @@
+package com.qiwenshare.file.config.es;
+
+import lombok.Data;
+import org.springframework.data.annotation.Id;
+import org.springframework.data.elasticsearch.annotations.Document;
+import org.springframework.data.elasticsearch.annotations.Field;
+import org.springframework.data.elasticsearch.annotations.FieldType;
+
+@Data
+@Document(indexName = "filesearch", shards = 1, replicas = 0)
+public class FileSearch {
+
+    @Id
+    private long fileSearchId;
+    @Field(type = FieldType.Long)
+    private Long fileId;
+    @Field(type = FieldType.Text, analyzer = "ik_max_word")
+    private String fileName;
+    @Field(type = FieldType.Keyword)
+    private String timeStampName;
+    @Field(type = FieldType.Keyword)
+    private String fileUrl;
+    @Field(type = FieldType.Long)
+    private Long fileSize;
+    @Field(type = FieldType.Integer)
+    private Integer isOSS;
+    @Field(type = FieldType.Integer)
+    private Integer pointCount;
+    @Field(type = FieldType.Keyword)
+    private String identifier;
+    @Field(type = FieldType.Long)
+    private Long userFileId;
+    @Field(type = FieldType.Long)
+    private Long userId;
+    @Field(type = FieldType.Keyword)
+    private String filePath;
+    @Field(type = FieldType.Keyword)
+    private String extendName;
+    @Field(type = FieldType.Integer)
+    private Integer isDir;
+    @Field(type = FieldType.Keyword)
+    private String uploadTime;
+    @Field(type = FieldType.Integer)
+    private Integer deleteFlag;
+    @Field(type = FieldType.Keyword)
+    private String deleteTime;
+    @Field(type = FieldType.Keyword)
+    private String deleteBatchNum;
+}
--- a/file-web/src/main/java/com/qiwenshare/file/controller/FileController.java
+++ b/file-web/src/main/java/com/qiwenshare/file/controller/FileController.java
@ -1,6 +1,5 @@
 package com.qiwenshare.file.controller;

-import cn.hutool.core.bean.BeanUtil;
 import com.alibaba.fastjson.JSON;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
@ -16,13 +15,23 @@ import com.qiwenshare.file.api.IRecoveryFileService;
 import com.qiwenshare.file.api.IUserFileService;
 import com.qiwenshare.file.api.IUserService;
 import com.qiwenshare.file.config.QiwenFileConfig;
+import com.qiwenshare.file.config.es.FileSearch;
 import com.qiwenshare.file.domain.*;
 import com.qiwenshare.file.dto.*;
+import com.qiwenshare.file.dto.file.*;
 import com.qiwenshare.file.vo.file.FileListVo;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.digest.DigestUtils;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
+import org.springframework.data.elasticsearch.core.SearchHits;
+import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
 import org.springframework.web.bind.annotation.*;

 import javax.annotation.Resource;
@ -54,7 +63,8 @@ public class FileController {

    public static final String CURRENT_MODULE = "文件接口";

-
+    @Autowired
+    private ElasticsearchRestTemplate elasticsearchRestTemplate;
    public static long treeid = 0;


@ -86,12 +96,48 @@ public class FileController {
        return RestResult.success();
    }

+    @Operation(summary = "文件搜索", description = "文件搜索", tags = {"file"})
+    @GetMapping(value = "/search")
+    @MyLog(operation = "文件搜索", module = CURRENT_MODULE)
+    @ResponseBody
+    public RestResult<SearchHits<FileSearch>> searchFile(SearchFileDTO searchFileDTO) {
+        NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder();
+        HighlightBuilder.Field allHighLight = new HighlightBuilder.Field("*").preTags("<span class='keyword'>")
+                .postTags("</span>");
+
+        queryBuilder.withHighlightFields(allHighLight);
+
+        //设置分页
+        int currentPage = (int)searchFileDTO.getCurrentPage() - 1;
+        int pageCount = (int)(searchFileDTO.getPageCount() == 0 ? 10 : searchFileDTO.getPageCount());
+        String order = searchFileDTO.getOrder();
+        Sort.Direction direction = null;
+        if (searchFileDTO.getDirection() == null) {
+            direction = Sort.Direction.DESC;
+        } else if ("asc".equals(searchFileDTO.getDirection())) {
+            direction = Sort.Direction.ASC;
+        } else if ("desc".equals(searchFileDTO.getDirection())) {
+            direction = Sort.Direction.DESC;
+        } else {
+            direction = Sort.Direction.DESC;
+        }
+        if (order == null) {
+            queryBuilder.withPageable(PageRequest.of(currentPage, pageCount));
+        } else {
+            queryBuilder.withPageable(PageRequest.of(currentPage, pageCount, Sort.by(direction, order)));
+        }
+
+        queryBuilder.withQuery(QueryBuilders.matchQuery("fileName", searchFileDTO.getFileName()));
+        SearchHits<FileSearch> search = elasticsearchRestTemplate.search(queryBuilder.build(), FileSearch.class);
+
+        return RestResult.success().data(search);
+    }
+
    @Operation(summary = "文件重命名", description = "文件重命名", tags = {"file"})
    @RequestMapping(value = "/renamefile", method = RequestMethod.POST)
    @MyLog(operation = "文件重命名", module = CURRENT_MODULE)
    @ResponseBody
    public RestResult<String> renameFile(@RequestBody RenameFileDTO renameFileDto, @RequestHeader("token") String token) {
-        RestResult<String> restResult = new RestResult<>();
        if (!operationCheck(token).getSuccess()){
            return operationCheck(token);
        }
--- a/file-web/src/main/java/com/qiwenshare/file/controller/FiletransferController.java
+++ b/file-web/src/main/java/com/qiwenshare/file/controller/FiletransferController.java
@ -2,6 +2,9 @@ package com.qiwenshare.file.controller;

 import com.aliyun.oss.OSS;
 import com.aliyun.oss.model.OSSObject;
+import com.github.tobato.fastdfs.proto.storage.DownloadByteArray;
+import com.github.tobato.fastdfs.service.AppendFileStorageClient;
+import com.github.tobato.fastdfs.service.FastFileStorageClient;
 import com.qiwenshare.common.cbb.DateUtil;
 import com.qiwenshare.common.operation.FileOperation;
 import com.qiwenshare.common.oss.AliyunOSSDownload;
@ -27,9 +30,11 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;

 import javax.annotation.Resource;
+import javax.servlet.ServletOutputStream;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import java.io.*;
+import java.net.URLEncoder;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@ -53,6 +58,8 @@ public class FiletransferController {
    IUserService userService;
    @Resource
    IUserFileService userFileService;
+    @Autowired
+    private FastFileStorageClient fastFileStorageClient;
    public static final String CURRENT_MODULE = "文件传输接口";

    @Operation(summary = "极速上传", description = "校验文件MD5判断文件是否存在，如果存在直接上传成功并返回skipUpload=true，如果不存在返回skipUpload=false需要再次调用该接口的POST方法", tags = {"filetransfer"})
@ -147,19 +154,20 @@ public class FiletransferController {
        }
        response.setContentType("application/force-download");// 设置强制下载不打开
        response.addHeader("Content-Disposition", "attachment;fileName=" + fileName);// 设置文件名
-        byte[] buffer = new byte[1024];
+

        FileBean fileBean = fileService.getById(userFile.getFileId());
        if (fileBean.getIsOSS() != null && fileBean.getIsOSS() == 1) {
-            aliyunDownload(response, buffer, fileBean);
+            aliyunDownload(response, fileBean);
        } else {
-            localFileDownload(response, buffer, fileBean);
+            localFileDownload(response, fileBean);
        }

    }

-    private void localFileDownload(HttpServletResponse response, byte[] buffer, FileBean fileBean) {
+    private void localFileDownload(HttpServletResponse response, FileBean fileBean) {
        BufferedInputStream bis = null;
+        byte[] buffer = new byte[1024];
        //设置文件路径
        File file = FileOperation.newFile(PathUtil.getStaticPath() + fileBean.getFileUrl());
        if (file.exists()) {
@ -191,8 +199,9 @@ public class FiletransferController {
        }
    }

-    private void aliyunDownload(HttpServletResponse response, byte[] buffer, FileBean fileBean) {
+    private void aliyunDownload(HttpServletResponse response, FileBean fileBean) {
        BufferedInputStream bis = null;
+        byte[] buffer = new byte[1024];
        AliyunOSSDownload aliyunOSSDownload= new AliyunOSSDownload();
        OSS ossClient = aliyunOSSDownload.createOSSClient(qiwenFileConfig.getAliyun().getOss());
        OSSObject ossObject = ossClient.getObject(qiwenFileConfig.getAliyun().getOss().getBucketName(), fileBean.getTimeStampName());
@ -220,6 +229,33 @@ public class FiletransferController {
        ossClient.shutdown();
    }

+
+    public void fastFDSDownload(HttpServletResponse response, FileBean fileBean) throws IOException {
+        String group = fileBean.getFileUrl().substring(0, fileBean.getFileUrl().indexOf("/"));
+        String path = fileBean.getFileUrl().substring(fileBean.getFileUrl().indexOf("/") + 1);
+        DownloadByteArray downloadByteArray = new DownloadByteArray();
+        byte[] bytes = fastFileStorageClient.downloadFile(group, path, downloadByteArray);
+
+        // 这里只是为了整合fastdfs，所以写死了文件格式。需要在上传的时候保存文件名。下载的时候使用对应的格式
+        response.setHeader("Content-disposition", "attachment;filename=" + URLEncoder.encode("sb.xlsx", "UTF-8"));
+        response.setCharacterEncoding("UTF-8");
+        ServletOutputStream outputStream = null;
+        try {
+            outputStream = response.getOutputStream();
+            outputStream.write(bytes);
+        } catch (IOException e) {
+            e.printStackTrace();
+        } finally {
+            try {
+                outputStream.flush();
+                outputStream.close();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+    }
+
+
    @Operation(summary = "获取存储信息", description = "获取存储信息", tags = {"filetransfer"})
    @RequestMapping(value = "/getstorage", method = RequestMethod.GET)
    @ResponseBody
--- a/file-web/src/main/java/com/qiwenshare/file/controller/TaskController.java
+++ b/file-web/src/main/java/com/qiwenshare/file/controller/TaskController.java
@ -1,13 +1,19 @@
 package com.qiwenshare.file.controller;

+import cn.hutool.core.bean.BeanUtil;
 import com.alibaba.fastjson.JSON;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.qiwenshare.file.api.IElasticSearchService;
+import com.qiwenshare.file.config.es.FileSearch;
 import com.qiwenshare.file.domain.FileBean;
 import com.qiwenshare.file.domain.UserFile;
 import com.qiwenshare.file.service.FileService;
 import com.qiwenshare.file.service.UserFileService;
 import com.qiwenshare.file.service.UserService;
+import com.qiwenshare.file.vo.file.FileListVo;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.catalina.User;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Controller;

@ -22,6 +28,10 @@ public class TaskController {

    @Resource
    FileService fileService;
+    @Resource
+    UserFileService userFileService;
+    @Autowired
+    private IElasticSearchService elasticSearchService;

    @Scheduled(cron = "0 0/1 * * * ?")
    public void deleteFile() {
@ -36,7 +46,28 @@ public class TaskController {
            fileService.deleteLocalFile(fileBean);
            fileService.removeById(fileBean.getFileId());
        }
-//        fileService.remove(fileBeanLambdaQueryWrapper);

    }
+
+    @Scheduled(fixedRate = 1000 * 60 * 60 * 24)
+    public void updateElasticSearch() {
+
+        UserFile userFile = new UserFile();
+        try {
+            elasticSearchService.deleteAll();
+        } catch (Exception e) {
+            log.error("删除ES失败:" + e);
+        }
+        try {
+            List<FileListVo> userfiles = userFileService.userFileList(userFile, 0L, 999999L);
+            for (FileListVo fileListVo : userfiles) {
+                log.info(JSON.toJSONString(fileListVo));
+                FileSearch fileSearch = new FileSearch();
+                BeanUtil.copyProperties(fileListVo, fileSearch);
+                elasticSearchService.save(fileSearch);
+            }
+        } catch (Exception e) {
+            log.error("更新ES失败:" + e);
+        }
+    }
 }
--- a/file-web/src/main/java/com/qiwenshare/file/controller/UserController.java
+++ b/file-web/src/main/java/com/qiwenshare/file/controller/UserController.java
@ -9,7 +9,7 @@ import com.qiwenshare.file.anno.MyLog;
 import com.qiwenshare.file.api.IUserService;
 import com.qiwenshare.file.config.QiwenFileConfig;
 import com.qiwenshare.file.domain.UserBean;
-import com.qiwenshare.file.dto.RegisterDTO;
+import com.qiwenshare.file.dto.user.RegisterDTO;
 import com.qiwenshare.file.vo.user.UserLoginVo;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
--- a/file-web/src/main/java/com/qiwenshare/file/dto/RegisterDTO.java
+++ b/file-web/src/main/java/com/qiwenshare/file/dto/RegisterDTO.java
@ -1,15 +0,0 @@
-package com.qiwenshare.file.dto;
-
-import io.swagger.v3.oas.annotations.media.Schema;
-import lombok.Data;
-
-@Data
-@Schema(name = "用户注册DTO",required = true)
-public class RegisterDTO {
-    @Schema(description = "用户名")
-    private String username;
-    @Schema(description = "手机号")
-    private String telephone;
-    @Schema(description = "密码")
-    private String password;
-}
--- a/file-web/src/main/java/com/qiwenshare/file/dto/file/BatchDeleteFileDTO.java
+++ b/file-web/src/main/java/com/qiwenshare/file/dto/file/BatchDeleteFileDTO.java
@ -1,4 +1,4 @@
-package com.qiwenshare.file.dto;
+package com.qiwenshare.file.dto.file;

 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Data;
--- a/file-web/src/main/java/com/qiwenshare/file/dto/file/CreateFileDTO.java
+++ b/file-web/src/main/java/com/qiwenshare/file/dto/file/CreateFileDTO.java
@ -1,4 +1,4 @@
-package com.qiwenshare.file.dto;
+package com.qiwenshare.file.dto.file;

 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Data;
--- a/file-web/src/main/java/com/qiwenshare/file/dto/file/DeleteFileDTO.java
+++ b/file-web/src/main/java/com/qiwenshare/file/dto/file/DeleteFileDTO.java
@ -1,4 +1,4 @@
-package com.qiwenshare.file.dto;
+package com.qiwenshare.file.dto.file;

 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Data;
--- a/file-web/src/main/java/com/qiwenshare/file/dto/file/RenameFileDTO.java
+++ b/file-web/src/main/java/com/qiwenshare/file/dto/file/RenameFileDTO.java
@ -1,4 +1,4 @@
-package com.qiwenshare.file.dto;
+package com.qiwenshare.file.dto.file;

 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Data;
--- a/file-web/src/main/java/com/qiwenshare/file/dto/file/SearchFileDTO.java
+++ b/file-web/src/main/java/com/qiwenshare/file/dto/file/SearchFileDTO.java
@ -0,0 +1,12 @@
+package com.qiwenshare.file.dto.file;
+
+import lombok.Data;
+
+@Data
+public class SearchFileDTO {
+    private String fileName;
+    private long currentPage;
+    private long pageCount;
+    private String order;
+    private String direction;
+}
--- a/file-web/src/main/java/com/qiwenshare/file/dto/file/UnzipFileDTO.java
+++ b/file-web/src/main/java/com/qiwenshare/file/dto/file/UnzipFileDTO.java
@ -1,4 +1,4 @@
-package com.qiwenshare.file.dto;
+package com.qiwenshare.file.dto.file;

 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Data;
--- a/file-web/src/main/java/com/qiwenshare/file/dto/user/RegisterDTO.java
+++ b/file-web/src/main/java/com/qiwenshare/file/dto/user/RegisterDTO.java
@ -1,6 +1,15 @@
 package com.qiwenshare.file.dto.user;

+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.Data;

+@Data
+@Schema(name = "用户注册DTO",required = true)
 public class RegisterDTO {
-
+    @Schema(description = "用户名")
+    private String username;
+    @Schema(description = "手机号")
+    private String telephone;
+    @Schema(description = "密码")
+    private String password;
 }
--- a/file-web/src/main/resources/config/application.properties
+++ b/file-web/src/main/resources/config/application.properties
@ -71,3 +71,27 @@ fdfs.thumb-image.width=150
 fdfs.thumb-image.height=150
 fdfs.tracker-list=121.89.222.103:22122 

+
+# Redis数据库索引（默认为0）
+spring.redis.database=0  
+# Redis服务器地址
+spring.redis.host=123.56.9.174
+# Redis服务器连接端口
+spring.redis.port=6379
+# Redis服务器连接密码（默认为空）
+spring.redis.password=ma123456
+# 连接池最大连接数（使用负值表示没有限制） 默认 8
+spring.redis.lettuce.pool.max-active=8
+# 连接池最大阻塞等待时间（使用负值表示没有限制） 默认 -1
+spring.redis.lettuce.pool.max-wait=10000
+# 连接池中的最大空闲连接 默认 8
+spring.redis.lettuce.pool.max-idle=30
+# 连接池中的最小空闲连接 默认 0
+spring.redis.lettuce.pool.min-idle=10
+#连接超时时间（毫秒）
+spring.redis.timeout=5000
+
+
+spring.data.elasticsearch.client.reactive.endpoints=127.0.0.1:9200
+spring.elasticsearch.rest.uris=127.0.0.1:9200
+