Gitlink
/
microservices
0
1
Fork 12
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(制品库功能开发): 产品库增加打包下载产品库功能 

文件微服务打包文件逻辑：
1. 增加输入参数：压缩文件名（不包含文件后缀）；修改输入参数打包结构的值fileSource为文件来源（文件标识或文件下载地址）
2. 由于打包过程中需要将下载或复制所有文件，并且需要压缩文件夹耗时较长，可能导致远程调用接口超时，所以调整为异步打包，内部调用打包接口时直接返回压缩文件标识，异步处理打包逻辑
3. 通过判断fileSource字段值是否为8位（文件标识长度为8位）来识别拷贝文件或从提供的url下载文件
This commit is contained in:
OTTO 2024-08-21 09:31:47 +08:00
parent fa3a852c94
commit 0923b92d85
12 changed files with 223 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
microservices-api/microservices-api-system/src/main/java/com/microservices/system/api/RemoteFileService.java
View File

@ -83,7 +83,8 @@ public interface RemoteFileService {
/**
* 打包文件
*
* @param packagedStructure 打包结构<文件包内路径,文件标识>
* @param packagedStructure 打包结构<文件包内路径,文件来源（文件标识或文件下载地址）>
* @param zipFileName 压缩文件名（不包含文件后缀）
* @param type 微服务类型
* @param hierarchy 层次结构
* @param source 鉴权
@ -92,6 +93,7 @@ public interface RemoteFileService {
@PostMapping(value = "/common/packagedFile")
R<String> packagedFile(
@RequestBody HashMap<String, String> packagedStructure,
@RequestParam("zipFileName") String zipFileName,
@RequestParam("type") String type,
@RequestParam("hierarchy") String hierarchy,
@RequestHeader(SecurityConstants.FROM_SOURCE) String source);

17
microservices-api/microservices-api-system/src/main/java/com/microservices/system/api/domain/SysFileInfo.java
View File

@ -1,10 +1,13 @@
package com.microservices.system.api.domain;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.microservices.common.core.utils.file.FileUtils;
import com.microservices.common.core.web.domain.BaseEntity;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.io.File;
/**
* 文件管理对象 sys_file_info
*
@ -103,6 +106,20 @@ public class SysFileInfo extends BaseEntity {
@ApiModelProperty(value = "是否部分匹配")
private Integer isPartialMatch;
public SysFileInfo() {
}
public SysFileInfo(String fileIdentifier, File file) {
this.setFileIdentifier(fileIdentifier);
this.setFilePath(file.getAbsolutePath());
this.setFileSizeInfo(FileUtils.calcFileSize(file.length()));
this.setFileOriginName(file.getName());
this.setDelFlag("N");
this.setFileOriginName(file.getName());
int separatorIndex = file.getName().lastIndexOf(".");
this.setFileSuffix(file.getName().substring(separatorIndex + 1).toLowerCase());
}
public SimpleGitlinkFileInfo toSimpleGitlinkFileInfo(){
SimpleGitlinkFileInfo simpleGitlinkFileInfo=new SimpleGitlinkFileInfo();
simpleGitlinkFileInfo.setId(this.getFileId());

2
microservices-api/microservices-api-system/src/main/java/com/microservices/system/api/factory/RemoteFileFallbackFactory.java
View File

@ -62,7 +62,7 @@ public class RemoteFileFallbackFactory implements FallbackFactory<RemoteFileServ
}
@Override
public R<String> packagedFile(HashMap<String, String> packagedStructure, String type, String hierarchy, String source) {
public R<String> packagedFile(HashMap<String, String> packagedStructure, String zipFileName, String type, String hierarchy, String source) {
return R.fail("打包文件失败:" + throwable.getMessage());
}
};

2
microservices-modules/microservices-modules-pms/src/main/java/com/microservices/pms/utils/ThreadPoolExecutorWrap.java → microservices-common/microservices-common-core/src/main/java/com/microservices/common/core/threadPool/ThreadPoolExecutorWrap.java
View File

@ -1,4 +1,4 @@
package com.microservices.pms.utils;
package com.microservices.common.core.threadPool;
import com.microservices.common.core.context.SecurityContextHolder;
import org.slf4j.Logger;

17
microservices-common/microservices-common-core/src/main/java/com/microservices/common/core/utils/file/FileUtils.java
View File

@ -14,6 +14,7 @@ import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
/**
* 文件处理工具类
@ -238,12 +239,13 @@ public class FileUtils
public static MultipartFile createMultipartFile(InputStream inputStream, String fileName) throws IOException {
FileItemFactory factory = new DiskFileItemFactory(16, null);
String textFieldName = "file";
FileItem item = factory.createItem(textFieldName, MediaType.MULTIPART_FORM_DATA_VALUE, true, fileName);
int bytesRead = 0;
byte[] buffer = new byte[10 * 1024 * 1024];
byte[] buffer = new byte[4096];
//使用输出流输出输入流的字节
try (OutputStream os = item.getOutputStream()) {
while ((bytesRead = inputStream.read(buffer, 0, 8192)) != -1) {
while ((bytesRead = inputStream.read(buffer)) != -1) {
os.write(buffer, 0, bytesRead);
}
inputStream.close();
@ -254,4 +256,15 @@ public class FileUtils
}
return new CommonsMultipartFile(item);
}
public static String calcFileSize(long fileSize) {
// 计算文件大小信息
String fileSizeInfo = "0kB";
if (fileSize != 0) {
String[] unitNames = new String[]{"B", "kB", "MB", "GB", "TB", "EB"};
int digitGroups = Math.min(unitNames.length - 1, (int) (Math.log10(fileSize) / Math.log10(1024)));
fileSizeInfo = new DecimalFormat("#,##0.##").format(fileSize / Math.pow(1024, digitGroups)) + " " + unitNames[digitGroups];
}
return fileSizeInfo;
}
}

6
microservices-modules/microservices-modules-file/src/main/java/com/microservices/file/controller/CommonController.java
View File

@ -72,7 +72,8 @@ public class CommonController extends BaseController {
/**
* 打包文件
*
* @param packagedStructure 打包结构<文件包内路径,文件标识>
* @param packagedStructure 打包结构<文件包内路径,文件来源（文件标识或文件下载地址）>
* @param zipFileName 压缩文件名（不包含文件后缀）
* @param type 微服务类型
* @param hierarchy 层次结构
* @return 压缩包文件标识
@ -81,9 +82,10 @@ public class CommonController extends BaseController {
@InnerAuth
public R<String> packagedFile(
@RequestBody HashMap<String, String> packagedStructure
, @RequestParam("zipFileName") String zipFileName
, @RequestParam("type") String type
, @RequestParam("hierarchy") String hierarchy) {
return R.ok(sysFileInfoService.packagedFile(packagedStructure, type, hierarchy));
return R.ok(sysFileInfoService.packagedFile(packagedStructure, zipFileName, type, hierarchy));
}
/**

14
microservices-modules/microservices-modules-file/src/main/java/com/microservices/file/service/ISysFileInfoAsyncService.java Normal file
View File

@ -0,0 +1,14 @@
package com.microservices.file.service;
import java.util.HashMap;
/**
* 文件管理异步Service接口
*/
public interface ISysFileInfoAsyncService {
/**
* 异步执行打包逻辑
*/
void asyncPackagedFile(String fileIdentifier, HashMap<String, String> packagedStructure, String zipFileName, String type, String hierarchy);
}

7
microservices-modules/microservices-modules-file/src/main/java/com/microservices/file/service/ISysFileInfoService.java
View File

@ -164,8 +164,11 @@ public interface ISysFileInfoService {
void getFileEntityByIdentifier(HttpServletResponse response, String fileIdentifier);
/**
* @param packagedStructure 打包结构<文件包内路径,文件标识>
* @param packagedStructure 打包结构<文件包内路径,文件来源（文件标识或文件下载地址）>
* @param zipFileName 压缩文件名（不包含文件后缀）
* @param type 微服务类型
* @param hierarchy 层次结构
* @return 压缩包文件标识
*/
String packagedFile(HashMap<String, String> packagedStructure, String type, String hierarchy);
String packagedFile(HashMap<String, String> packagedStructure, String zipFileName, String type, String hierarchy);
}

105
microservices-modules/microservices-modules-file/src/main/java/com/microservices/file/service/impl/SysFileInfoAsyncServiceImpl.java Normal file
View File

@ -0,0 +1,105 @@
package com.microservices.file.service.impl;
import com.microservices.common.core.exception.ServiceException;
import com.microservices.common.core.utils.DateUtils;
import com.microservices.common.core.utils.ServletUtils;
import com.microservices.file.service.ISysFileInfoAsyncService;
import com.microservices.file.service.ISysFileInfoService;
import com.microservices.file.utils.CustomExecutorFactory;
import com.microservices.file.utils.ZipUtils;
import com.microservices.system.api.domain.SysFileInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
/**
* @author OTTO
*/
@Service
public class SysFileInfoAsyncServiceImpl implements ISysFileInfoAsyncService {
private static final Logger logger = LoggerFactory.getLogger(SysFileInfoAsyncServiceImpl.class);
@Lazy
@Autowired
private ISysFileInfoService sysFileInfoService;
/**
* 上传文件存储在本地的根路径
*/
@Value("${file.path}")
private String localFilePath;
@Override
public void asyncPackagedFile(String fileIdentifier, HashMap<String, String> packagedStructure, String zipFileName, String type, String hierarchy) {
CustomExecutorFactory.threadPoolExecutor.execute(() -> {
Path localPath = Paths.get(localFilePath);
// 输出目录
Path packagedPath = localPath
.resolve(type)
.resolve(hierarchy)
.resolve(DateUtils.datePath());
// 构建打包的临时工作目录
Path tempPackagedPath = packagedPath
.resolve(zipFileName);
try {
try {
// 将所有文件根据层级结构拷贝到临时目录
packagedStructure.forEach((packagedFilePathStr, fileSource) -> {
String packagedDirPathStr = Paths.get(ServletUtils.urlDecode(packagedFilePathStr)).getParent().toString();
try {
// 创建文件对应目录
Path packagedDirPath = tempPackagedPath.resolve(packagedDirPathStr);
Files.createDirectories(packagedDirPath);
// 文件标识长度为8位
if (fileSource.length() == 8) {
SysFileInfo fileInfo = sysFileInfoService.selectSysFileInfoByFileIdentifier(fileSource);
String filePath = localFilePath + fileInfo.getFilePath();
File file = new File(filePath);
if (!file.exists()) {
throw new IOException("该文件不存在");
}
// 拷贝文件到打包目录
FileInputStream fileInputStream = new FileInputStream(file);
Files.copy(fileInputStream, packagedDirPath.resolve(fileInfo.getFileOriginName()));
} else {
String encodeFileName = fileSource.substring(fileSource.lastIndexOf('/') + 1);
String fileName = ServletUtils.urlDecode(ServletUtils.urlDecode(encodeFileName));
Path packagedFilePath = packagedDirPath.resolve(fileName);
org.apache.commons.io.FileUtils.copyURLToFile(new URL(fileSource), packagedFilePath.toFile());
}
} catch (IOException e) {
logger.error("打包文件时出现异常:{0}", e);
throw new ServiceException("打包文件时出现异常！");
}
});
Path zipFilePath = packagedPath.resolve(zipFileName + ".zip");
ZipUtils.toZip(tempPackagedPath.toString(), Files.newOutputStream(zipFilePath), true);
File zipFile = zipFilePath.toFile();
if (!zipFile.exists()) {
throw new ServiceException("打包文件时出现异常：文件压缩失败");
}
SysFileInfo sysFileInfo = new SysFileInfo(fileIdentifier, zipFile);
sysFileInfoService.insertSysFileInfo(sysFileInfo);
} finally {
//删除临时目录
org.apache.commons.io.FileUtils.deleteDirectory(tempPackagedPath.toFile());
}
} catch (IOException e) {
logger.error("打包文件时出现异常:{}", e.getMessage());
throw new ServiceException("打包文件时出现异常！");
}
});
}
}

102
microservices-modules/microservices-modules-file/src/main/java/com/microservices/file/service/impl/SysFileInfoServiceImpl.java
View File

@ -7,7 +7,6 @@ import com.j256.simplemagic.ContentInfoUtil;
import com.microservices.common.core.constant.Constants;
import com.microservices.common.core.exception.ServiceException;
import com.microservices.common.core.utils.DateUtils;
import com.microservices.common.core.utils.ServletUtils;
import com.microservices.common.core.utils.StringUtils;
import com.microservices.common.core.utils.file.FileTypeUtils;
import com.microservices.common.core.utils.file.FileUtils;
@ -17,11 +16,10 @@ import com.microservices.common.httpClient.domain.GitLinkRequestUrl;
import com.microservices.common.httpClient.util.GitLinkRequestHelper;
import com.microservices.common.security.utils.SecurityUtils;
import com.microservices.file.mapper.SysFileInfoMapper;
import com.microservices.file.service.ISysFileInfoAsyncService;
import com.microservices.file.service.ISysFileInfoService;
import com.microservices.file.utils.FileUploadUtils;
import com.microservices.file.utils.ZipUtils;
import com.microservices.system.api.domain.SysFileInfo;
import com.microservices.system.api.model.LoginUser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -36,10 +34,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.DecimalFormat;
import java.util.*;
import java.util.regex.Pattern;
@ -56,6 +51,8 @@ public class SysFileInfoServiceImpl implements ISysFileInfoService {
private SysFileInfoMapper sysFileInfoMapper;
@Autowired
private GitLinkRequestHelper gitLinkRequestHelper;
@Autowired
private ISysFileInfoAsyncService fileInfoAsyncService;
/**
* 上传文件存储在本地的根路径
*/
@ -132,6 +129,7 @@ public class SysFileInfoServiceImpl implements ISysFileInfoService {
if (StringUtils.isEmpty(sysFileInfo.getFileSuffix())) {
sysFileInfo.setFileSuffix("other");
}
sysFileInfo.setCreateBy(SecurityUtils.getUsername());
sysFileInfo.setCreateTime(DateUtils.getNowDate());
return sysFileInfoMapper.insertSysFileInfo(sysFileInfo);
}
@ -394,15 +392,8 @@ public class SysFileInfoServiceImpl implements ISysFileInfoService {
break;
}
case "pms": {
boolean identifierIsExist = true;
while (identifierIsExist) {
// 项目管理下需生成唯一标识，防止用户通过id递增访问
sysFileInfo.setFileIdentifier(IdUtils.fastShortUUID());
SysFileInfo sysFileInfoByIdentifier = sysFileInfoMapper.selectSysFileInfoByIdentifier(sysFileInfo.getFileIdentifier());
if (sysFileInfoByIdentifier == null) {
identifierIsExist = false;
}
}
// 项目管理下需生成唯一标识，防止用户通过id递增访问
sysFileInfo.setFileIdentifier(genFileIdentifier());
baseFilePath += "/" + fileType + "/" + hierarchy + "/";
isCommonFileType = false;
break;
@ -452,17 +443,7 @@ public class SysFileInfoServiceImpl implements ISysFileInfoService {
sysFileInfo.setFileObjectName(newFileName);
sysFileInfo.setFileSuffix(newFileName.substring(separatorIndex + 1).toLowerCase());
LoginUser loginUser = SecurityUtils.getLoginUser();
// 计算文件大小信息
long size = file.getSize();
String fileSizeInfo = "0kB";
if (size != 0) {
String[] unitNames = new String[]{"B", "kB", "MB", "GB", "TB", "EB"};
int digitGroups = Math.min(unitNames.length - 1, (int) (Math.log10(size) / Math.log10(1024)));
fileSizeInfo = new DecimalFormat("#,##0.##").format(size / Math.pow(1024, digitGroups)) + " " + unitNames[digitGroups];
}
sysFileInfo.setFileSizeInfo(fileSizeInfo);
sysFileInfo.setFileSizeInfo(FileUtils.calcFileSize(file.getSize()));
// 获取文件内容信息
try {
ContentInfoUtil util = new ContentInfoUtil();
@ -480,11 +461,6 @@ public class SysFileInfoServiceImpl implements ISysFileInfoService {
}
sysFileInfo.setFileOriginName(file.getOriginalFilename());
sysFileInfo.setDelFlag("N");
if (loginUser == null) {
logger.error("无法获取到当前登录用户信息");
} else {
sysFileInfo.setCreateBy(loginUser.getUsername());
}
insertSysFileInfo(sysFileInfo);
fileId = sysFileInfo.getFileId();
//当上传路径为默认路径时，标识支持通过静态资源访问
@ -503,52 +479,10 @@ public class SysFileInfoServiceImpl implements ISysFileInfoService {
}
@Override
public String packagedFile(HashMap<String, String> packagedStructure, String type, String hierarchy) {
Path localPath = Paths.get(localFilePath);
// 输出目录
Path packagedPath = localPath
.resolve(type)
.resolve(hierarchy)
.resolve(DateUtils.datePath());
// 构建打包的临时工作目录
String tempPackagedDirName = "tempPackagedDir-" + DateUtils.dateTimeNow();
Path tempPackagedPath = packagedPath
.resolve(tempPackagedDirName);
try {
try {
// 将所有文件根据层级结构拷贝到临时目录
packagedStructure.forEach((packagedDirPathStr, fileIdentifier) -> {
packagedDirPathStr = ServletUtils.urlDecode(packagedDirPathStr);
SysFileInfo fileInfo = selectSysFileInfoByFileIdentifier(fileIdentifier);
try {
String filePath = localFilePath + fileInfo.getFilePath();
File file = new File(filePath);
if (!file.exists()) {
throw new IOException("该文件不存在");
}
// 创建文件对应目录
Path packagedDirPath = tempPackagedPath.resolve(packagedDirPathStr);
Files.createDirectories(packagedDirPath);
// 拷贝文件到打包目录
FileInputStream fileInputStream = new FileInputStream(file);
Files.copy(fileInputStream, packagedDirPath.resolve(fileInfo.getFileOriginName()));
} catch (IOException e) {
logger.error("打包文件时出现异常:{0}", e);
throw new ServiceException("打包文件时出现异常！");
}
});
Path zipFilePath = packagedPath.resolve("product.zip");
ZipUtils.toZip(tempPackagedPath.toString(), Files.newOutputStream(zipFilePath), true);
} finally {
//删除临时目录
org.apache.commons.io.FileUtils.deleteDirectory(tempPackagedPath.toFile());
}
} catch (IOException e) {
logger.error("打包文件时出现异常:{}", e.getMessage());
throw new ServiceException("打包文件时出现异常！");
}
return "";
public String packagedFile(HashMap<String, String> packagedStructure, String zipFileName, String type, String hierarchy) {
String fileIdentifier = genFileIdentifier();
fileInfoAsyncService.asyncPackagedFile(fileIdentifier, packagedStructure, zipFileName, type, hierarchy);
return fileIdentifier;
}
private void fileDownload(SysFileInfo sysFileInfo, HttpServletResponse response) {
@ -609,4 +543,18 @@ public class SysFileInfoServiceImpl implements ISysFileInfoService {
}
return hierarchy;
}
private String genFileIdentifier() {
String fileIdentifier = null;
boolean identifierIsExist = true;
while (identifierIsExist) {
// 项目管理下需生成唯一标识，防止用户通过id递增访问
fileIdentifier = IdUtils.fastShortUUID();
SysFileInfo sysFileInfoByIdentifier = sysFileInfoMapper.selectSysFileInfoByIdentifier(fileIdentifier);
if (sysFileInfoByIdentifier == null) {
identifierIsExist = false;
}
}
return fileIdentifier;
}
}

32
microservices-modules/microservices-modules-file/src/main/java/com/microservices/file/utils/CustomExecutorFactory.java Normal file
View File

@ -0,0 +1,32 @@
package com.microservices.file.utils;
import com.microservices.common.core.exception.ServiceException;
import com.microservices.common.core.threadPool.ThreadPoolExecutorWrap;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
public class CustomExecutorFactory {
/**
* 创建线程池
* 存在并发处理的情况，设置核心线程为2，设置有界队列长度为100，最大线程数为10
* 当超出队列已满且达到最大线程数时抛出异常
* Ncpu=CPU数量
* Ucpu=目标CPU的使用率，0<=Ucpu<=1
* W/C=任务等待时间与任务计算时间的比率
* Nthreads =Ncpu*Ucpu*(1+W/C)
*/
public static ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutorWrap(
2
, 10
, 10
, TimeUnit.SECONDS
, new ArrayBlockingQueue<>(100)
, Executors.defaultThreadFactory()
, (r, executor) -> {
throw new ServiceException("目前处理的人太多了，请稍后再试");
}
);
}

1
microservices-modules/microservices-modules-pms/src/main/java/com/microservices/pms/utils/CustomExecutorFactory.java
View File

@ -1,6 +1,7 @@
package com.microservices.pms.utils;
import com.microservices.common.core.exception.ServiceException;
import com.microservices.common.core.threadPool.ThreadPoolExecutorWrap;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Executors;