Gitlink
/
microservices
0
1
Fork 12
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(AI资源广场): 模型和数据集资源相关功能开发 

新增解析已上传的模型/数据集ZIP包分析接口：
使用commons-compress组件可以在不解压压缩包的前提下，对超大压缩包通过读取 Central Directory方式快速获取压缩包内，同时直接从根目录直接提取README.md文件
This commit is contained in:
otto 2026-05-18 09:17:17 +08:00
parent ea17f00fd8
commit 084042891d
9 changed files with 271 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
microservices-api/microservices-api-system/src/main/java/com/microservices/system/api/RemoteFileService.java
View File

@ -52,6 +52,10 @@ public interface RemoteFileService {
@GetMapping("/open/getFile/{fileId}")
public R<SysFileInfo> getFile(@PathVariable("fileId") Long fileId);
@GetMapping("/common/getFileDetail/{fileId}")
public R<SysFileInfo> getFileDetail(@PathVariable("fileId") Long fileId,
@RequestHeader(SecurityConstants.FROM_SOURCE) String source);
/**
* 根据文件Id获取文件信息
*

5
microservices-api/microservices-api-system/src/main/java/com/microservices/system/api/factory/RemoteFileFallbackFactory.java
View File

@ -46,6 +46,11 @@ public class RemoteFileFallbackFactory implements FallbackFactory<RemoteFileServ
return R.fail("根据文件Id获取文件信息失败:" + throwable.getMessage());
}
@Override
public R<SysFileInfo> getFileDetail(Long fileId, String source) {
return R.fail("根据文件id获取文件详细信息失败:" + throwable.getMessage());
}
@Override
public R<SysFileInfo> getFileByFileIdentifier(String fileIdentifier, String source) {
return R.fail("根据文件标识获取文件信息失败:" + throwable.getMessage());

12
microservices-modules/microservices-modules-file/src/main/java/com/microservices/file/controller/CommonController.java
View File

@ -235,4 +235,16 @@ public class CommonController extends BaseController {
R<List<String>> uploadFileToForge(@PathVariable("fileIdentifiers") String fileIdentifiers) {
return R.ok(sysFileInfoService.uploadFileToForge(fileIdentifiers));
}
/**
* 根据文件Id获取文件信息
*/
@InnerAuth
@GetMapping("/getFileDetail/{fileId}")
@ApiOperation(value = "根据文件Id获取文件信息")
public R<SysFileInfo> getFileDetail(@PathVariable("fileId") Long fileId) {
SysFileInfo sysFileInfo = sysFileInfoService.selectSysFileInfoByFileId(fileId);
sysFileInfo.setFilePath(localFilePath + sysFileInfo.getFilePath());
return R.ok(sysFileInfo);
}
}

6
microservices-modules/microservices-modules-zone/pom.xml
View File

@ -133,6 +133,12 @@
<version>3.6.2</version>
<scope>compile</scope>
</dependency>
<!-- Apache Commons Compress - 解决 ZIP 文件中文文件名 GBK 编码导致的 MALFORMED 问题 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.26.0</version>
</dependency>
</dependencies>
<build>

22
microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/common/controller/FrontController.java
View File

@ -287,6 +287,28 @@ public class FrontController extends BaseController {
return success(zoneResourceService.updateAiSkillResource(updateVo));
}
// ==================== 模型/数据集资源 - ZIP解析 ====================
/**
* 解析已上传的模型ZIP包
*/
@RequiresPermissions("zone:front:model:add")
@PostMapping("/model/parse")
@ApiOperation("解析已上传的模型ZIP包")
public GenericsAjaxResult<AiUploadedResourceParseResultVo> parseModelZip(@RequestParam Long fileId) {
return genericsSuccess(aiSkillParseService.parseUploadedZip(fileId));
}
/**
* 解析已上传的数据集ZIP包
*/
@RequiresPermissions("zone:front:dataset:add")
@PostMapping("/dataset/parse")
@ApiOperation("解析已上传的数据集ZIP包")
public GenericsAjaxResult<AiUploadedResourceParseResultVo> parseDatasetZip(@RequestParam Long fileId) {
return genericsSuccess(aiSkillParseService.parseUploadedZip(fileId));
}
/**
* 查询我的资源详情
*/

20
microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/controller/AiResourceController.java
View File

@ -66,4 +66,24 @@ public class AiResourceController extends BaseController {
public AjaxResult editSkill(@RequestBody @Validated AiSkillUpdateVo updateVo) {
return success(zoneResourceService.updateAiSkillResource(updateVo));
}
/**
* 解析已上传的模型ZIP包
*/
// @RequiresPermissions("ai:resource:model:add")
@PostMapping("/model/parse")
@ApiOperation("解析已上传的模型ZIP包")
public GenericsAjaxResult<AiUploadedResourceParseResultVo> parseModelZip(@RequestParam Long fileId) {
return genericsSuccess(aiSkillParseService.parseUploadedZip(fileId));
}
/**
* 解析已上传的数据集ZIP包
*/
// @RequiresPermissions("ai:resource:dataset:add")
@PostMapping("/dataset/parse")
@ApiOperation("解析已上传的数据集ZIP包")
public GenericsAjaxResult<AiUploadedResourceParseResultVo> parseDatasetZip(@RequestParam Long fileId) {
return genericsSuccess(aiSkillParseService.parseUploadedZip(fileId));
}
}

20
microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/domain/vo/AiUploadedResourceParseResultVo.java Normal file
View File

@ -0,0 +1,20 @@
package com.microservices.zone.resource.domain.vo;
import lombok.Data;
/**
* 已上传资源ZIP解析结果
*/
@Data
public class AiUploadedResourceParseResultVo {
/**
* README.md 文件内容，不存在时为 null
*/
private String readmeContent;
/**
* 目录树 JSON 字符串
*/
private String fileTreeJson;
}

12
microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/service/IAiSkillParseService.java
View File

@ -1,10 +1,11 @@
package com.microservices.zone.resource.service;
import com.microservices.zone.resource.domain.vo.AiSkillParseResultVo;
import com.microservices.zone.resource.domain.vo.AiUploadedResourceParseResultVo;
import org.springframework.web.multipart.MultipartFile;
/**
* Skill ZIP解析服务接口
* AI资源ZIP解析服务接口
*/
public interface IAiSkillParseService {
@ -18,4 +19,13 @@ public interface IAiSkillParseService {
* @return 解析结果
*/
AiSkillParseResultVo parseSkillZip(MultipartFile zipFile);
/**
* 解析已上传的资源ZIP包（模型/数据集）
* 基于已通过分片上传到服务器的文件，解析目录树和 README.md
*
* @param fileId 已上传文件的ID
* @return 解析结果
*/
AiUploadedResourceParseResultVo parseUploadedZip(Long fileId);
}

180
microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/service/impl/AiSkillParseServiceImpl.java
View File

@ -1,6 +1,5 @@
package com.microservices.zone.resource.service.impl;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.microservices.common.core.constant.SecurityConstants;
@ -8,21 +7,27 @@ import com.microservices.common.core.exception.ServiceException;
import com.microservices.common.core.utils.StringUtils;
import com.microservices.system.api.RemoteFileService;
import com.microservices.system.api.domain.SysFile;
import com.microservices.system.api.domain.SysFileInfo;
import com.microservices.system.api.utils.FeignUtils;
import com.microservices.zone.resource.domain.vo.AiSkillParseResultVo;
import com.microservices.zone.resource.domain.vo.AiUploadedResourceParseResultVo;
import com.microservices.zone.resource.service.IAiSkillParseService;
import com.microservices.zone.utils.ZoneConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.compress.archivers.zip.ZipFile;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
/**
* Skill ZIP解析服务实现
@ -32,6 +37,7 @@ public class AiSkillParseServiceImpl implements IAiSkillParseService {
private static final Logger logger = LoggerFactory.getLogger(AiSkillParseServiceImpl.class);
private static final String SKILL_MD_FILE = "SKILL.md";
private static final String README_MD_FILE = "README.md";
private final RemoteFileService remoteFileService;
@ -52,12 +58,12 @@ public class AiSkillParseServiceImpl implements IAiSkillParseService {
throw new ServiceException("仅支持ZIP格式文件");
}
try (ZipInputStream zis = new ZipInputStream(zipFile.getInputStream(), StandardCharsets.UTF_8)) {
ZipEntry entry;
try (ZipArchiveInputStream zais = new ZipArchiveInputStream(zipFile.getInputStream(), "GBK", true)) {
ZipArchiveEntry entry;
String skillMdContent = null;
List<FileTreeNode> fileTreeNodes = new ArrayList<>();
while ((entry = zis.getNextEntry()) != null) {
while ((entry = zais.getNextZipEntry()) != null) {
String entryName = entry.getName();
if (entry.isDirectory()) {
continue;
@ -69,7 +75,7 @@ public class AiSkillParseServiceImpl implements IAiSkillParseService {
// 提取 SKILL.md（在任意层级下查找）
String fileName = new File(entryName).getName();
if (SKILL_MD_FILE.equalsIgnoreCase(fileName)) {
skillMdContent = readEntryContent(zis);
skillMdContent = readEntryContent(zais);
}
// 收集文件信息构建目录树
fileTreeNodes.add(new FileTreeNode(entryName, entry.getSize()));
@ -118,16 +124,172 @@ public class AiSkillParseServiceImpl implements IAiSkillParseService {
}
}
private String readEntryContent(ZipInputStream zis) throws IOException {
private String readEntryContent(ZipArchiveInputStream zais) throws IOException {
StringBuilder sb = new StringBuilder();
byte[] buffer = new byte[4096];
int len;
while ((len = zis.read(buffer)) > 0) {
while ((len = zais.read(buffer)) > 0) {
sb.append(new String(buffer, 0, len, StandardCharsets.UTF_8));
}
return sb.toString();
}
@Override
public AiUploadedResourceParseResultVo parseUploadedZip(Long fileId) {
if (fileId == null) {
throw new ServiceException("文件ID不能为空");
}
// 1. 获取文件信息
SysFileInfo sysFileInfo = FeignUtils.getReturnData(remoteFileService.getFileDetail(fileId, SecurityConstants.INNER));
if (sysFileInfo == null) {
throw new ServiceException("文件不存在");
}
// 2. 校验文件格式
String fileSuffix = sysFileInfo.getFileSuffix();
if (!"zip".equalsIgnoreCase(fileSuffix)) {
throw new ServiceException("仅支持ZIP格式文件");
}
// 3. 构建绝对路径
String absolutePath = sysFileInfo.getFilePath();
File zipFile = new File(absolutePath);
if (!zipFile.exists()) {
throw new ServiceException("ZIP文件不存在：" + sysFileInfo.getFilePath());
}
// 4. 使用 ZipFile 解析（读取 Central Directory，支持随机访问，大文件秒级响应）
try (ZipFile zf = new ZipFile(zipFile, "GBK", true)) {
String readmeContent = null;
List<FileTreeNode> fileTreeNodes = new ArrayList<>();
Enumeration<ZipArchiveEntry> entries = zf.getEntries();
while (entries.hasMoreElements()) {
ZipArchiveEntry entry = entries.nextElement();
String entryName = entry.getName();
if (entry.isDirectory()) {
continue;
}
if (entryName.contains("..")) {
throw new ServiceException("ZIP包中包含非法路径");
}
fileTreeNodes.add(new FileTreeNode(entryName, entry.getSize()));
}
// 5. 查找并读取 README.md
readmeContent = findAndReadReadmeMd(zf, fileTreeNodes);
// 6. 构建目录树 JSON
JSONObject fileTreeJson = buildFileTreeJsonFast(fileTreeNodes);
AiUploadedResourceParseResultVo result = new AiUploadedResourceParseResultVo();
result.setReadmeContent(readmeContent);
result.setFileTreeJson(fileTreeJson.toJSONString());
return result;
} catch (Exception e) {
logger.error("ZIP包解析失败: fileId={}, path={}", fileId, absolutePath, e);
throw new ServiceException("ZIP包解析失败：" + e.getMessage());
}
}
private String findAndReadReadmeMd(ZipFile zf, List<FileTreeNode> nodes) throws IOException {
// 优先查找根目录 README.md
ZipArchiveEntry readmeEntry = zf.getEntry(README_MD_FILE);
if (readmeEntry != null && !readmeEntry.isDirectory()) {
return readZipEntryContent(zf, readmeEntry);
}
// 其次查找顶层唯一文件夹下的 README.md
String topFolder = findSingleTopFolder(nodes);
if (topFolder != null) {
readmeEntry = zf.getEntry(topFolder + "/" + README_MD_FILE);
if (readmeEntry != null && !readmeEntry.isDirectory()) {
return readZipEntryContent(zf, readmeEntry);
}
}
return null;
}
private String findSingleTopFolder(List<FileTreeNode> nodes) {
Set<String> topFolders = new HashSet<>();
for (FileTreeNode node : nodes) {
int slashIndex = node.path.indexOf('/');
if (slashIndex > 0) {
topFolders.add(node.path.substring(0, slashIndex));
}
}
return topFolders.size() == 1 ? topFolders.iterator().next() : null;
}
private String readZipEntryContent(ZipFile zf, ZipArchiveEntry entry) throws IOException {
try (InputStream is = zf.getInputStream(entry)) {
StringBuilder sb = new StringBuilder();
byte[] buffer = new byte[4096];
int len;
while ((len = is.read(buffer)) > 0) {
sb.append(new String(buffer, 0, len, StandardCharsets.UTF_8));
}
return sb.toString();
}
}
private JSONObject buildFileTreeJsonFast(List<FileTreeNode> nodes) {
JSONObject root = new JSONObject();
root.put("name", "root");
root.put("type", "folder");
JSONArray rootChildren = new JSONArray();
root.put("children", rootChildren);
Map<String, JSONObject> folderCache = new LinkedHashMap<>();
for (FileTreeNode node : nodes) {
String[] parts = node.path.split("/");
JSONArray currentChildren = rootChildren;
StringBuilder pathBuilder = new StringBuilder();
for (int i = 0; i < parts.length; i++) {
String part = parts[i];
if (StringUtils.isEmpty(part)) {
continue;
}
boolean isFile = (i == parts.length - 1);
if (isFile) {
JSONObject fileNode = new JSONObject();
fileNode.put("name", part);
fileNode.put("type", "file");
fileNode.put("size", node.size);
currentChildren.add(fileNode);
} else {
pathBuilder.append(part);
String cacheKey = pathBuilder.toString();
pathBuilder.append('/');
JSONObject existingFolder = folderCache.get(cacheKey);
if (existingFolder == null) {
existingFolder = new JSONObject();
existingFolder.put("name", part);
existingFolder.put("type", "folder");
existingFolder.put("children", new JSONArray());
currentChildren.add(existingFolder);
folderCache.put(cacheKey, existingFolder);
}
currentChildren = existingFolder.getJSONArray("children");
}
}
}
// 如果根目录只有一个子文件夹，直接返回该子文件夹
JSONArray children = root.getJSONArray("children");
if (children.size() == 1 && "folder".equals(children.getJSONObject(0).get("type"))) {
return children.getJSONObject(0);
}
return root;
}
private JSONObject buildFileTreeJson(List<FileTreeNode> nodes) {
// 构建树形结构的目录
JSONObject root = new JSONObject();