From e10c756af13995e3adb3de0b12959487d92d6157 Mon Sep 17 00:00:00 2001 From: OTTO <731554297@qq.com> Date: Thu, 1 Aug 2024 10:35:15 +0800 Subject: [PATCH] =?UTF-8?q?feat(=E5=88=B6=E5=93=81=E5=BA=93=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=E5=BC=80=E5=8F=91):=20=E4=BA=A7=E5=93=81=E5=BA=93Dock?= =?UTF-8?q?er=E5=88=B6=E5=93=81=E7=A7=BB=E5=85=A5=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 解析docker库的映射端口，通过docker镜像名以及镜像版本拼接docker包文件名 2. 基于Nexus Ip地址、docker库映射端口，docker镜像名、docker镜像版本生成docker镜像拉取地址 3. 调用python脚本根据docker镜像拉取地址已经docker包文件名获取docker镜像包文件流 4. 通过Raw上传接口将docker镜像包文件流上传至产品库 --- .../IPmsProductLibraryAsyncService.java | 10 ++ .../PmsProductLibraryAsyncServiceImpl.java | 94 ++++++++++--------- 2 files changed, 62 insertions(+), 42 deletions(-) diff --git a/microservices-modules/microservices-modules-pms/src/main/java/com/microservices/pms/productLibrary/service/IPmsProductLibraryAsyncService.java b/microservices-modules/microservices-modules-pms/src/main/java/com/microservices/pms/productLibrary/service/IPmsProductLibraryAsyncService.java index fd5764d62..a0852720d 100644 --- a/microservices-modules/microservices-modules-pms/src/main/java/com/microservices/pms/productLibrary/service/IPmsProductLibraryAsyncService.java +++ b/microservices-modules/microservices-modules-pms/src/main/java/com/microservices/pms/productLibrary/service/IPmsProductLibraryAsyncService.java @@ -4,6 +4,7 @@ package com.microservices.pms.productLibrary.service; import com.microservices.pms.productLibrary.domain.PmsProductLibraryComponent; import com.microservices.pms.productLibrary.domain.PmsProductLibraryRepositories; import com.microservices.pms.productLibrary.domain.vo.NexusAssetResultDataVo; +import com.microservices.pms.productLibrary.domain.vo.NexusComponentResultDataVo; import java.util.List; @@ -35,4 +36,13 @@ public interface IPmsProductLibraryAsyncService { * @param snapshotRepo 快照库 */ void syncRepoToSnapshotRepo(PmsProductLibraryRepositories repo, PmsProductLibraryRepositories snapshotRepo); + + /** + * 异步将Docker制品库制品移入到产品库中 + * + * @param productRepo 产品库 + * @param repo 制品库 + * @param nexusComponentResultDataVoList Docker制品列表 + */ + void asyncAddDockerComponentToProductRepo(PmsProductLibraryRepositories productRepo, PmsProductLibraryRepositories repo, List nexusComponentResultDataVoList); } diff --git a/microservices-modules/microservices-modules-pms/src/main/java/com/microservices/pms/productLibrary/service/impl/PmsProductLibraryAsyncServiceImpl.java b/microservices-modules/microservices-modules-pms/src/main/java/com/microservices/pms/productLibrary/service/impl/PmsProductLibraryAsyncServiceImpl.java index 8a17940e2..54acc68a9 100644 --- a/microservices-modules/microservices-modules-pms/src/main/java/com/microservices/pms/productLibrary/service/impl/PmsProductLibraryAsyncServiceImpl.java +++ b/microservices-modules/microservices-modules-pms/src/main/java/com/microservices/pms/productLibrary/service/impl/PmsProductLibraryAsyncServiceImpl.java @@ -12,6 +12,7 @@ import com.microservices.pms.productLibrary.service.IPmsProductLibraryRepositori import com.microservices.pms.utils.CustomExecutorFactory; import com.microservices.pms.utils.NexusRequestHelper; import com.microservices.pms.utils.NexusRequestUrl; +import com.microservices.pms.utils.PythonUtil; import com.microservices.system.api.domain.SysFileBytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -45,10 +47,7 @@ public class PmsProductLibraryAsyncServiceImpl implements IPmsProductLibraryAsyn boolean isContinuationGet = true; NexusComponentAndAssetListVo assetListVo = new NexusComponentAndAssetListVo<>(); while (isContinuationGet) { - assetListVo = nexusRequestHelper.doRequestToJavaObject( - NexusRequestUrl.GET_ASSETS(repo.getNexusRepositoriesName(), assetListVo.getContinuationToken()), - NexusComponentAndAssetListVo.class, - NexusAssetVo.class); + assetListVo = nexusRequestHelper.doRequestToJavaObject(NexusRequestUrl.GET_ASSETS(repo.getNexusRepositoriesName(), assetListVo.getContinuationToken()), NexusComponentAndAssetListVo.class, NexusAssetVo.class); List assetList = assetListVo.getItems(); if (pmsProductLibraryComponents.stream().allMatch(x -> StringUtils.isNotEmpty(x.getIdentifier()))) { break; @@ -79,24 +78,15 @@ public class PmsProductLibraryAsyncServiceImpl implements IPmsProductLibraryAsyn nexusAssetResultDataVo.setVersion(""); } if (StringUtils.isNotEmpty(nexusAssetResultDataVo.getFileIdentifier())) { - List assetRawInputVoList = new ArrayList<>(); - AssetRawInputVo assetRawInputVo = new AssetRawInputVo(); - assetRawInputVo.setFilename(nexusAssetResultDataVo.getFileName()); assetRawInputVo.setFileIdentifier(nexusAssetResultDataVo.getFileIdentifier()); - assetRawInputVoList.add(assetRawInputVo); - insertRawComponentToProductRepo(productRepo, repo, nexusAssetResultDataVo, assetRawInputVoList); } else { - List assetRawInputVoList = new ArrayList<>(); - AssetRawInputVo assetRawInputVo = new AssetRawInputVo(); - assetRawInputVo.setFilename(nexusAssetResultDataVo.getFileName()); assetRawInputVo.setFileIdentifier(""); - String nexusFileUrl = - EscapeUtil.removeExtraSlashOfUrl(repo.getUrl() + "/" + nexusAssetResultDataVo.getName()); + String nexusFileUrl = EscapeUtil.removeExtraSlashOfUrl(repo.getUrl() + "/" + nexusAssetResultDataVo.getName()); SysFileBytes fileBytes = nexusRequestHelper.getFileBytes(nexusFileUrl); assetRawInputVo.setFileBytes(fileBytes); - assetRawInputVoList.add(assetRawInputVo); - insertRawComponentToProductRepo(productRepo, repo, nexusAssetResultDataVo, assetRawInputVoList); } + assetRawInputVoList.add(assetRawInputVo); + insertRawComponentToProductRepo(productRepo, repo, assetRawInputVoList, nexusAssetResultDataVo.getVersion(), customAttribute, nexusAssetResultDataVo.getDirectory()); } } finally { pmsProductLibraryRepositoriesService.unlockRepository(repo.getPmsEnterpriseIdentifier(), repo.getId()); @@ -140,9 +130,43 @@ public class PmsProductLibraryAsyncServiceImpl implements IPmsProductLibraryAsyn }); } + @Override + public void asyncAddDockerComponentToProductRepo(PmsProductLibraryRepositories productRepo, PmsProductLibraryRepositories repo, List nexusComponentResultDataVoList) { + CustomExecutorFactory.threadPoolExecutor.execute(() -> { + pmsProductLibraryRepositoriesService.lockRepository(repo.getPmsEnterpriseIdentifier(), repo.getId()); + try { + // 解析docker库的映射端口 + RepoAttributesVo dockerRepoAttributes = repo.getRepoAttributes(); + Integer dockerHttpPort = dockerRepoAttributes.getDocker().getHttpPort(); + for (NexusComponentResultDataVo nexusComponentResultDataVo : nexusComponentResultDataVoList) { + List assetRawInputVoList = new ArrayList<>(); + AssetRawInputVo assetRawInputVo = new AssetRawInputVo(); + String dockerImgName = nexusComponentResultDataVo.getName().replaceAll("/", "_").replaceAll("\\\\", "_"); + String dockerImgFileName = String.format("%s_%s.tar", dockerImgName, nexusComponentResultDataVo.getVersion()); + assetRawInputVo.setFilename(dockerImgFileName); + assetRawInputVo.setFileIdentifier(""); + String dockerImgUrl = EscapeUtil.removeExtraSlashOfUrl(String.format("%s:%d/%s:%s", nexusRequestHelper.getNexusIp(), dockerHttpPort, nexusComponentResultDataVo.getName(), nexusComponentResultDataVo.getVersion())); + try { + byte[] fileBytes = PythonUtil.pullDockerImage(dockerImgUrl, dockerImgFileName); + SysFileBytes sysFileBytes = new SysFileBytes(); + sysFileBytes.setFileBytes(fileBytes); + sysFileBytes.setFileName(dockerImgFileName); + assetRawInputVo.setFileBytes(sysFileBytes); + assetRawInputVo.setAssetIdentifier(nexusComponentResultDataVo.getId()); + assetRawInputVoList.add(assetRawInputVo); + insertRawComponentToProductRepo(productRepo, repo, assetRawInputVoList, nexusComponentResultDataVo.getVersion(), null, ""); + } catch (IOException e) { + log.error("拉取Docker镜像失败：{}", e.getMessage()); + } + } + } finally { + pmsProductLibraryRepositoriesService.unlockRepository(repo.getPmsEnterpriseIdentifier(), repo.getId()); + } + }); + } + private void syncMavenRepoToSnapshotRepo(PmsProductLibraryRepositories repo, PmsProductLibraryRepositories snapshotRepo) { - List pmsProductLibraryComponentList = - pmsProductLibraryComponentService.selectPmsProductLibraryComponentListByDatabase(repo.getId()); + List pmsProductLibraryComponentList = pmsProductLibraryComponentService.selectPmsProductLibraryComponentListByDatabase(repo.getId()); for (PmsProductLibraryComponent pmsProductLibraryComponent : pmsProductLibraryComponentList) { ComponentMavenInputVo componentMavenInputVo = pmsProductLibraryComponent.toComponentMavenInputVo(snapshotRepo); List assetMavenInputVoList = getAssetMavenInputVoList(repo, pmsProductLibraryComponent, componentMavenInputVo); @@ -164,8 +188,7 @@ public class PmsProductLibraryAsyncServiceImpl implements IPmsProductLibraryAsyn componentInputVo.setSnapshotName(snapshotRepo.getSnapshotName()); componentInputVo.setEnterpriseIdentifier(snapshotRepo.getPmsEnterpriseIdentifier()); componentInputVo.setFileIdentifier(""); - String nexusFileUrl = - EscapeUtil.removeExtraSlashOfUrl(repo.getUrl() + "/" + asset.getPath()); + String nexusFileUrl = EscapeUtil.removeExtraSlashOfUrl(repo.getUrl() + "/" + asset.getPath()); SysFileBytes fileBytes = nexusRequestHelper.getFileBytes(nexusFileUrl); if (fileBytes == null) { continue; @@ -186,8 +209,7 @@ public class PmsProductLibraryAsyncServiceImpl implements IPmsProductLibraryAsyn componentInputVo.setSnapshotName(snapshotRepo.getSnapshotName()); componentInputVo.setEnterpriseIdentifier(snapshotRepo.getPmsEnterpriseIdentifier()); componentInputVo.setFileIdentifier(""); - String nexusFileUrl = - EscapeUtil.removeExtraSlashOfUrl(repo.getUrl() + "/" + asset.getPath()); + String nexusFileUrl = EscapeUtil.removeExtraSlashOfUrl(repo.getUrl() + "/" + asset.getPath()); SysFileBytes fileBytes = nexusRequestHelper.getFileBytes(nexusFileUrl); if (fileBytes == null) { continue; @@ -198,8 +220,7 @@ public class PmsProductLibraryAsyncServiceImpl implements IPmsProductLibraryAsyn } private void syncRawRepoToSnapshotRepo(PmsProductLibraryRepositories repo, PmsProductLibraryRepositories snapshotRepo) { - List pmsProductLibraryComponentList = - pmsProductLibraryComponentService.selectPmsProductLibraryComponentListByDatabase(repo.getId()); + List pmsProductLibraryComponentList = pmsProductLibraryComponentService.selectPmsProductLibraryComponentListByDatabase(repo.getId()); for (PmsProductLibraryComponent pmsProductLibraryComponent : pmsProductLibraryComponentList) { ComponentRawInputVo componentRawInputVo = pmsProductLibraryComponent.toComponentRawInputVo(snapshotRepo); List assetRawInputVoList = getAssetRawInputVoList(repo, pmsProductLibraryComponent, componentRawInputVo); @@ -215,8 +236,7 @@ public class PmsProductLibraryAsyncServiceImpl implements IPmsProductLibraryAsyn for (AssetRawInputVo assetRawInputVo : assetRawInputVoList) { if (StringUtils.isEmpty(assetRawInputVo.getFileIdentifier())) { assetRawInputVo.setFileIdentifier(""); - String nexusFileUrl = - EscapeUtil.removeExtraSlashOfUrl(repo.getUrl() + "/" + pmsProductLibraryComponent.getPath()); + String nexusFileUrl = EscapeUtil.removeExtraSlashOfUrl(repo.getUrl() + "/" + pmsProductLibraryComponent.getPath()); SysFileBytes fileBytes = nexusRequestHelper.getFileBytes(nexusFileUrl); if (fileBytes == null) { return null; @@ -232,8 +252,7 @@ public class PmsProductLibraryAsyncServiceImpl implements IPmsProductLibraryAsyn for (AssetMavenInputVo assetMavenInputVo : assetMavenInputVoList) { if (StringUtils.isEmpty(assetMavenInputVo.getFileIdentifier())) { assetMavenInputVo.setFileIdentifier(""); - String nexusFileUrl = - EscapeUtil.removeExtraSlashOfUrl(repo.getUrl() + "/" + pmsProductLibraryComponent.getPath()); + String nexusFileUrl = EscapeUtil.removeExtraSlashOfUrl(repo.getUrl() + "/" + pmsProductLibraryComponent.getPath()); SysFileBytes fileBytes = nexusRequestHelper.getFileBytes(nexusFileUrl); if (fileBytes == null) { return null; @@ -244,21 +263,15 @@ public class PmsProductLibraryAsyncServiceImpl implements IPmsProductLibraryAsyn return assetMavenInputVoList; } - private void insertRawComponentToProductRepo(PmsProductLibraryRepositories productRepo, - PmsProductLibraryRepositories repo, - NexusAssetResultDataVo nexusAssetResultDataVo, - List assetRawInputVoList) { + private void insertRawComponentToProductRepo(PmsProductLibraryRepositories productRepo, PmsProductLibraryRepositories repo, List assetRawInputVoList, String version, String customAttribute, String directory) { ComponentRawInputVo componentRawInputVo = new ComponentRawInputVo(); componentRawInputVo.setRepoName(productRepo.getName()); componentRawInputVo.setEnterpriseIdentifier(productRepo.getPmsEnterpriseIdentifier()); componentRawInputVo.setFormat(ProductLibraryRepositoriesFormatEnum.PRODUCT.getKey()); - componentRawInputVo.setVersion(nexusAssetResultDataVo.getVersion()); - if (nexusAssetResultDataVo.getAttributes() != null - && nexusAssetResultDataVo.getAttributes().containsKey("customAttribute")) { - componentRawInputVo.setAttributes(nexusAssetResultDataVo.getAttributes().getJSONObject("customAttribute").toJSONString()); - } + componentRawInputVo.setVersion(version); + componentRawInputVo.setAttributes(customAttribute); componentRawInputVo.setAssetList(assetRawInputVoList); - componentRawInputVo.setDirectory(String.format("/%s/%s", repo.getAlias(), nexusAssetResultDataVo.getDirectory())); + componentRawInputVo.setDirectory(String.format("/%s/%s", repo.getAlias(), directory)); pmsProductLibraryComponentService.insertRawComponent(componentRawInputVo); } @@ -268,10 +281,7 @@ public class PmsProductLibraryAsyncServiceImpl implements IPmsProductLibraryAsyn boolean isContinuationGet = true; NexusComponentAndAssetListVo assetListVo = new NexusComponentAndAssetListVo<>(); while (isContinuationGet) { - assetListVo = nexusRequestHelper.doRequestToJavaObject( - NexusRequestUrl.GET_ASSETS(repo.getNexusRepositoriesName(), assetListVo.getContinuationToken()), - NexusComponentAndAssetListVo.class, - NexusAssetVo.class); + assetListVo = nexusRequestHelper.doRequestToJavaObject(NexusRequestUrl.GET_ASSETS(repo.getNexusRepositoriesName(), assetListVo.getContinuationToken()), NexusComponentAndAssetListVo.class, NexusAssetVo.class); assetList.addAll(assetListVo.getItems()); if (StringUtils.isEmpty(assetListVo.getContinuationToken())) { isContinuationGet = false;