From 084042891df84d94a5cd45c03aac2e8da9b363db Mon Sep 17 00:00:00 2001 From: otto <731554297@qq.com> Date: Mon, 18 May 2026 09:17:17 +0800 Subject: [PATCH] =?UTF-8?q?feat(AI=E8=B5=84=E6=BA=90=E5=B9=BF=E5=9C=BA):?= =?UTF-8?q?=20=E6=A8=A1=E5=9E=8B=E5=92=8C=E6=95=B0=E6=8D=AE=E9=9B=86?= =?UTF-8?q?=E8=B5=84=E6=BA=90=E7=9B=B8=E5=85=B3=E5=8A=9F=E8=83=BD=E5=BC=80?= =?UTF-8?q?=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增解析已上传的模型/数据集ZIP包分析接口： 使用commons-compress组件可以在不解压压缩包的前提下，对超大压缩包通过读取 Central Directory方式快速获取压缩包内，同时直接从根目录直接提取README.md文件 --- .../system/api/RemoteFileService.java | 4 + .../factory/RemoteFileFallbackFactory.java | 5 + .../file/controller/CommonController.java | 12 ++ .../microservices-modules-zone/pom.xml | 6 + .../common/controller/FrontController.java | 22 +++ .../controller/AiResourceController.java | 20 ++ .../vo/AiUploadedResourceParseResultVo.java | 20 ++ .../service/IAiSkillParseService.java | 12 +- .../service/impl/AiSkillParseServiceImpl.java | 180 +++++++++++++++++- 9 files changed, 271 insertions(+), 10 deletions(-) create mode 100644 microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/domain/vo/AiUploadedResourceParseResultVo.java diff --git a/microservices-api/microservices-api-system/src/main/java/com/microservices/system/api/RemoteFileService.java b/microservices-api/microservices-api-system/src/main/java/com/microservices/system/api/RemoteFileService.java index 67052ee82..dcbfa0924 100644 --- a/microservices-api/microservices-api-system/src/main/java/com/microservices/system/api/RemoteFileService.java +++ b/microservices-api/microservices-api-system/src/main/java/com/microservices/system/api/RemoteFileService.java @@ -52,6 +52,10 @@ public interface RemoteFileService { @GetMapping("/open/getFile/{fileId}") public R getFile(@PathVariable("fileId") Long fileId); + @GetMapping("/common/getFileDetail/{fileId}") + public R getFileDetail(@PathVariable("fileId") Long fileId, + @RequestHeader(SecurityConstants.FROM_SOURCE) String source); + /** * 根据文件Id获取文件信息 * diff --git a/microservices-api/microservices-api-system/src/main/java/com/microservices/system/api/factory/RemoteFileFallbackFactory.java b/microservices-api/microservices-api-system/src/main/java/com/microservices/system/api/factory/RemoteFileFallbackFactory.java index 1bf85bbfe..1572c3b60 100644 --- a/microservices-api/microservices-api-system/src/main/java/com/microservices/system/api/factory/RemoteFileFallbackFactory.java +++ b/microservices-api/microservices-api-system/src/main/java/com/microservices/system/api/factory/RemoteFileFallbackFactory.java @@ -46,6 +46,11 @@ public class RemoteFileFallbackFactory implements FallbackFactory getFileDetail(Long fileId, String source) { + return R.fail("根据文件id获取文件详细信息失败:" + throwable.getMessage()); + } + @Override public R getFileByFileIdentifier(String fileIdentifier, String source) { return R.fail("根据文件标识获取文件信息失败:" + throwable.getMessage()); diff --git a/microservices-modules/microservices-modules-file/src/main/java/com/microservices/file/controller/CommonController.java b/microservices-modules/microservices-modules-file/src/main/java/com/microservices/file/controller/CommonController.java index f28d6c32a..dc25eb5d0 100644 --- a/microservices-modules/microservices-modules-file/src/main/java/com/microservices/file/controller/CommonController.java +++ b/microservices-modules/microservices-modules-file/src/main/java/com/microservices/file/controller/CommonController.java @@ -235,4 +235,16 @@ public class CommonController extends BaseController { R> uploadFileToForge(@PathVariable("fileIdentifiers") String fileIdentifiers) { return R.ok(sysFileInfoService.uploadFileToForge(fileIdentifiers)); } + + /** + * 根据文件Id获取文件信息 + */ + @InnerAuth + @GetMapping("/getFileDetail/{fileId}") + @ApiOperation(value = "根据文件Id获取文件信息") + public R getFileDetail(@PathVariable("fileId") Long fileId) { + SysFileInfo sysFileInfo = sysFileInfoService.selectSysFileInfoByFileId(fileId); + sysFileInfo.setFilePath(localFilePath + sysFileInfo.getFilePath()); + return R.ok(sysFileInfo); + } } diff --git a/microservices-modules/microservices-modules-zone/pom.xml b/microservices-modules/microservices-modules-zone/pom.xml index b92340783..36e833409 100644 --- a/microservices-modules/microservices-modules-zone/pom.xml +++ b/microservices-modules/microservices-modules-zone/pom.xml @@ -133,6 +133,12 @@ 3.6.2 compile + + + org.apache.commons + commons-compress + 1.26.0 + diff --git a/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/common/controller/FrontController.java b/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/common/controller/FrontController.java index 136936863..34f5a5147 100644 --- a/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/common/controller/FrontController.java +++ b/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/common/controller/FrontController.java @@ -287,6 +287,28 @@ public class FrontController extends BaseController { return success(zoneResourceService.updateAiSkillResource(updateVo)); } + // ==================== 模型/数据集资源 - ZIP解析 ==================== + + /** + * 解析已上传的模型ZIP包 + */ + @RequiresPermissions("zone:front:model:add") + @PostMapping("/model/parse") + @ApiOperation("解析已上传的模型ZIP包") + public GenericsAjaxResult parseModelZip(@RequestParam Long fileId) { + return genericsSuccess(aiSkillParseService.parseUploadedZip(fileId)); + } + + /** + * 解析已上传的数据集ZIP包 + */ + @RequiresPermissions("zone:front:dataset:add") + @PostMapping("/dataset/parse") + @ApiOperation("解析已上传的数据集ZIP包") + public GenericsAjaxResult parseDatasetZip(@RequestParam Long fileId) { + return genericsSuccess(aiSkillParseService.parseUploadedZip(fileId)); + } + /** * 查询我的资源详情 */ diff --git a/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/controller/AiResourceController.java b/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/controller/AiResourceController.java index e976f22f0..c84b78241 100644 --- a/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/controller/AiResourceController.java +++ b/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/controller/AiResourceController.java @@ -66,4 +66,24 @@ public class AiResourceController extends BaseController { public AjaxResult editSkill(@RequestBody @Validated AiSkillUpdateVo updateVo) { return success(zoneResourceService.updateAiSkillResource(updateVo)); } + + /** + * 解析已上传的模型ZIP包 + */ +// @RequiresPermissions("ai:resource:model:add") + @PostMapping("/model/parse") + @ApiOperation("解析已上传的模型ZIP包") + public GenericsAjaxResult parseModelZip(@RequestParam Long fileId) { + return genericsSuccess(aiSkillParseService.parseUploadedZip(fileId)); + } + + /** + * 解析已上传的数据集ZIP包 + */ +// @RequiresPermissions("ai:resource:dataset:add") + @PostMapping("/dataset/parse") + @ApiOperation("解析已上传的数据集ZIP包") + public GenericsAjaxResult parseDatasetZip(@RequestParam Long fileId) { + return genericsSuccess(aiSkillParseService.parseUploadedZip(fileId)); + } } diff --git a/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/domain/vo/AiUploadedResourceParseResultVo.java b/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/domain/vo/AiUploadedResourceParseResultVo.java new file mode 100644 index 000000000..dc03b9412 --- /dev/null +++ b/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/domain/vo/AiUploadedResourceParseResultVo.java @@ -0,0 +1,20 @@ +package com.microservices.zone.resource.domain.vo; + +import lombok.Data; + +/** + * 已上传资源ZIP解析结果 + */ +@Data +public class AiUploadedResourceParseResultVo { + + /** + * README.md 文件内容，不存在时为 null + */ + private String readmeContent; + + /** + * 目录树 JSON 字符串 + */ + private String fileTreeJson; +} diff --git a/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/service/IAiSkillParseService.java b/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/service/IAiSkillParseService.java index 76b0e4b91..c3b7162a3 100644 --- a/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/service/IAiSkillParseService.java +++ b/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/service/IAiSkillParseService.java @@ -1,10 +1,11 @@ package com.microservices.zone.resource.service; import com.microservices.zone.resource.domain.vo.AiSkillParseResultVo; +import com.microservices.zone.resource.domain.vo.AiUploadedResourceParseResultVo; import org.springframework.web.multipart.MultipartFile; /** - * Skill ZIP解析服务接口 + * AI资源ZIP解析服务接口 */ public interface IAiSkillParseService { @@ -18,4 +19,13 @@ public interface IAiSkillParseService { * @return 解析结果 */ AiSkillParseResultVo parseSkillZip(MultipartFile zipFile); + + /** + * 解析已上传的资源ZIP包（模型/数据集） + * 基于已通过分片上传到服务器的文件，解析目录树和 README.md + * + * @param fileId 已上传文件的ID + * @return 解析结果 + */ + AiUploadedResourceParseResultVo parseUploadedZip(Long fileId); } diff --git a/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/service/impl/AiSkillParseServiceImpl.java b/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/service/impl/AiSkillParseServiceImpl.java index e58e70ab3..2f782e01e 100644 --- a/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/service/impl/AiSkillParseServiceImpl.java +++ b/microservices-modules/microservices-modules-zone/src/main/java/com/microservices/zone/resource/service/impl/AiSkillParseServiceImpl.java @@ -1,6 +1,5 @@ package com.microservices.zone.resource.service.impl; -import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONArray; import com.alibaba.fastjson2.JSONObject; import com.microservices.common.core.constant.SecurityConstants; @@ -8,21 +7,27 @@ import com.microservices.common.core.exception.ServiceException; import com.microservices.common.core.utils.StringUtils; import com.microservices.system.api.RemoteFileService; import com.microservices.system.api.domain.SysFile; +import com.microservices.system.api.domain.SysFileInfo; import com.microservices.system.api.utils.FeignUtils; import com.microservices.zone.resource.domain.vo.AiSkillParseResultVo; +import com.microservices.zone.resource.domain.vo.AiUploadedResourceParseResultVo; import com.microservices.zone.resource.service.IAiSkillParseService; import com.microservices.zone.utils.ZoneConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipFile; + import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.*; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; /** * Skill ZIP解析服务实现 @@ -32,6 +37,7 @@ public class AiSkillParseServiceImpl implements IAiSkillParseService { private static final Logger logger = LoggerFactory.getLogger(AiSkillParseServiceImpl.class); private static final String SKILL_MD_FILE = "SKILL.md"; + private static final String README_MD_FILE = "README.md"; private final RemoteFileService remoteFileService; @@ -52,12 +58,12 @@ public class AiSkillParseServiceImpl implements IAiSkillParseService { throw new ServiceException("仅支持ZIP格式文件"); } - try (ZipInputStream zis = new ZipInputStream(zipFile.getInputStream(), StandardCharsets.UTF_8)) { - ZipEntry entry; + try (ZipArchiveInputStream zais = new ZipArchiveInputStream(zipFile.getInputStream(), "GBK", true)) { + ZipArchiveEntry entry; String skillMdContent = null; List fileTreeNodes = new ArrayList<>(); - while ((entry = zis.getNextEntry()) != null) { + while ((entry = zais.getNextZipEntry()) != null) { String entryName = entry.getName(); if (entry.isDirectory()) { continue; @@ -69,7 +75,7 @@ public class AiSkillParseServiceImpl implements IAiSkillParseService { // 提取 SKILL.md（在任意层级下查找） String fileName = new File(entryName).getName(); if (SKILL_MD_FILE.equalsIgnoreCase(fileName)) { - skillMdContent = readEntryContent(zis); + skillMdContent = readEntryContent(zais); } // 收集文件信息构建目录树 fileTreeNodes.add(new FileTreeNode(entryName, entry.getSize())); @@ -118,16 +124,172 @@ public class AiSkillParseServiceImpl implements IAiSkillParseService { } } - private String readEntryContent(ZipInputStream zis) throws IOException { + private String readEntryContent(ZipArchiveInputStream zais) throws IOException { StringBuilder sb = new StringBuilder(); byte[] buffer = new byte[4096]; int len; - while ((len = zis.read(buffer)) > 0) { + while ((len = zais.read(buffer)) > 0) { sb.append(new String(buffer, 0, len, StandardCharsets.UTF_8)); } return sb.toString(); } + @Override + public AiUploadedResourceParseResultVo parseUploadedZip(Long fileId) { + if (fileId == null) { + throw new ServiceException("文件ID不能为空"); + } + // 1. 获取文件信息 + SysFileInfo sysFileInfo = FeignUtils.getReturnData(remoteFileService.getFileDetail(fileId, SecurityConstants.INNER)); + if (sysFileInfo == null) { + throw new ServiceException("文件不存在"); + } + + // 2. 校验文件格式 + String fileSuffix = sysFileInfo.getFileSuffix(); + if (!"zip".equalsIgnoreCase(fileSuffix)) { + throw new ServiceException("仅支持ZIP格式文件"); + } + + // 3. 构建绝对路径 + String absolutePath = sysFileInfo.getFilePath(); + File zipFile = new File(absolutePath); + if (!zipFile.exists()) { + throw new ServiceException("ZIP文件不存在：" + sysFileInfo.getFilePath()); + } + + // 4. 使用 ZipFile 解析（读取 Central Directory，支持随机访问，大文件秒级响应） + try (ZipFile zf = new ZipFile(zipFile, "GBK", true)) { + String readmeContent = null; + List fileTreeNodes = new ArrayList<>(); + + Enumeration entries = zf.getEntries(); + while (entries.hasMoreElements()) { + ZipArchiveEntry entry = entries.nextElement(); + String entryName = entry.getName(); + if (entry.isDirectory()) { + continue; + } + if (entryName.contains("..")) { + throw new ServiceException("ZIP包中包含非法路径"); + } + + fileTreeNodes.add(new FileTreeNode(entryName, entry.getSize())); + } + + // 5. 查找并读取 README.md + readmeContent = findAndReadReadmeMd(zf, fileTreeNodes); + + // 6. 构建目录树 JSON + JSONObject fileTreeJson = buildFileTreeJsonFast(fileTreeNodes); + + AiUploadedResourceParseResultVo result = new AiUploadedResourceParseResultVo(); + result.setReadmeContent(readmeContent); + result.setFileTreeJson(fileTreeJson.toJSONString()); + return result; + + } catch (Exception e) { + logger.error("ZIP包解析失败: fileId={}, path={}", fileId, absolutePath, e); + throw new ServiceException("ZIP包解析失败：" + e.getMessage()); + } + } + + private String findAndReadReadmeMd(ZipFile zf, List nodes) throws IOException { + // 优先查找根目录 README.md + ZipArchiveEntry readmeEntry = zf.getEntry(README_MD_FILE); + if (readmeEntry != null && !readmeEntry.isDirectory()) { + return readZipEntryContent(zf, readmeEntry); + } + + // 其次查找顶层唯一文件夹下的 README.md + String topFolder = findSingleTopFolder(nodes); + if (topFolder != null) { + readmeEntry = zf.getEntry(topFolder + "/" + README_MD_FILE); + if (readmeEntry != null && !readmeEntry.isDirectory()) { + return readZipEntryContent(zf, readmeEntry); + } + } + + return null; + } + + private String findSingleTopFolder(List nodes) { + Set topFolders = new HashSet<>(); + for (FileTreeNode node : nodes) { + int slashIndex = node.path.indexOf('/'); + if (slashIndex > 0) { + topFolders.add(node.path.substring(0, slashIndex)); + } + } + return topFolders.size() == 1 ? topFolders.iterator().next() : null; + } + + private String readZipEntryContent(ZipFile zf, ZipArchiveEntry entry) throws IOException { + try (InputStream is = zf.getInputStream(entry)) { + StringBuilder sb = new StringBuilder(); + byte[] buffer = new byte[4096]; + int len; + while ((len = is.read(buffer)) > 0) { + sb.append(new String(buffer, 0, len, StandardCharsets.UTF_8)); + } + return sb.toString(); + } + } + + private JSONObject buildFileTreeJsonFast(List nodes) { + JSONObject root = new JSONObject(); + root.put("name", "root"); + root.put("type", "folder"); + JSONArray rootChildren = new JSONArray(); + root.put("children", rootChildren); + + Map folderCache = new LinkedHashMap<>(); + + for (FileTreeNode node : nodes) { + String[] parts = node.path.split("/"); + JSONArray currentChildren = rootChildren; + StringBuilder pathBuilder = new StringBuilder(); + + for (int i = 0; i < parts.length; i++) { + String part = parts[i]; + if (StringUtils.isEmpty(part)) { + continue; + } + boolean isFile = (i == parts.length - 1); + + if (isFile) { + JSONObject fileNode = new JSONObject(); + fileNode.put("name", part); + fileNode.put("type", "file"); + fileNode.put("size", node.size); + currentChildren.add(fileNode); + } else { + pathBuilder.append(part); + String cacheKey = pathBuilder.toString(); + pathBuilder.append('/'); + + JSONObject existingFolder = folderCache.get(cacheKey); + if (existingFolder == null) { + existingFolder = new JSONObject(); + existingFolder.put("name", part); + existingFolder.put("type", "folder"); + existingFolder.put("children", new JSONArray()); + currentChildren.add(existingFolder); + folderCache.put(cacheKey, existingFolder); + } + currentChildren = existingFolder.getJSONArray("children"); + } + } + } + + // 如果根目录只有一个子文件夹，直接返回该子文件夹 + JSONArray children = root.getJSONArray("children"); + if (children.size() == 1 && "folder".equals(children.getJSONObject(0).get("type"))) { + return children.getJSONObject(0); + } + return root; + } + private JSONObject buildFileTreeJson(List nodes) { // 构建树形结构的目录 JSONObject root = new JSONObject();