Compare commits

...

60 Commits

Author SHA1 Message Date
jshixiong 424b6e573b fix:检测概览空数据 2023-09-01 15:29:16 +08:00
jshixiong b8822ad3f8 fix:取消定时检测任务 2023-08-31 18:31:10 +08:00
jshixiong 15a04d36ae fix:缺陷详情 2023-08-31 17:01:35 +08:00
jshixiong 8eb748b9e8 token 2023-08-31 16:15:24 +08:00
jshixiong 3d1fc694f1 更新opensca url 2023-08-31 15:58:36 +08:00
jshixiong 288ec01fe0 更新opensca token 2023-08-31 15:44:55 +08:00
jshixiong 1aa700c742 log 2023-08-31 15:06:30 +08:00
jshixiong 3494074621 fix:检测结果对比 2023-08-31 11:33:31 +08:00
weishao 610b516ef2 目录不存在需要新建 2023-08-25 12:37:49 +08:00
youys a022d1fd01 fix 2023-03-17 17:00:23 +08:00
youys 1f4ea01314 fix 2023-03-17 16:55:54 +08:00
jshixiong 4fa7cb1a77 fix 2023-03-15 17:30:24 +08:00
jshixiong b5f8cdb1e8 fix:报告下载 2023-03-15 17:23:47 +08:00
jshixiong dbb4d28542 fix 2023-03-15 15:08:20 +08:00
jshixiong ba32373595 Merge remote-tracking branch 'origin/merge' into merge 2023-03-15 15:02:53 +08:00
jshixiong cf478ec902 fix:结果对比 2023-03-15 15:02:37 +08:00
weishao 367791da34 sast服务切换库 2023-03-15 13:59:20 +08:00
weishao 968426ec11 单条分析结果报错 2023-03-14 16:04:08 +08:00
weishao d6cfd77d68 文件路径 2023-03-14 13:53:23 +08:00
weishao 1ba3bd3d49 cert c 2023-03-14 12:13:26 +08:00
weishao 7765355130 缺陷分类导出,定时任务逻辑fix 2023-03-14 11:47:04 +08:00
jshixiong 34685e67bd fix 2023-03-14 09:27:46 +08:00
jshixiong f3bfc1cdca Merge remote-tracking branch 'origin/merge' into merge 2023-03-13 18:39:29 +08:00
jshixiong 4f2a2a0ac8 漏洞增量修改 2023-03-13 18:37:47 +08:00
youys 9f7115ff5b Merge branch 'merge' of https://gitlink.org.cn/youys/quality_analysis into merge 2023-03-13 18:26:36 +08:00
youys d79003788c 修改正则 2023-03-13 18:26:31 +08:00
jshixiong 77dcae5c56 fix 2023-03-13 17:31:42 +08:00
jshixiong 10a3c38bd5 缺陷增量 2023-03-13 16:48:54 +08:00
youys 2c4a80df90 行数计算 2023-03-13 10:57:11 +08:00
youys 216c73e2ae 各个语言代码行数不处理 2023-03-13 10:52:11 +08:00
youys 8521901b03 fix 2023-03-13 09:28:03 +08:00
youys de293760dc Merge branch 'merge' of https://gitlink.org.cn/youys/quality_analysis into merge 2023-03-13 09:22:57 +08:00
youys 08e13fd261 fix 2023-03-13 09:22:50 +08:00
jshixiong 640ff13a57 fix 2023-03-12 15:29:40 +08:00
jshixiong da4599f7d2 fix 2023-03-12 15:20:13 +08:00
jshixiong 7359902c83 drone触发检测 2023-03-12 15:15:55 +08:00
youys d322ffb172 fix 2023-03-11 22:42:59 +08:00
youys 95ec6a5d54 fix 2023-03-11 22:36:29 +08:00
youys 5c24798452 fix 2023-03-11 22:32:26 +08:00
youys 5e596ab716 判空 2023-03-11 21:53:56 +08:00
youys 8d44f9a09d fix 2023-03-10 18:32:17 +08:00
youys 1ba1f024db fix 2023-03-10 18:28:10 +08:00
youys 23c1d3d3aa 移除sonar 2023-03-10 18:24:39 +08:00
weishao 2e8175c613 language空格处理 2023-03-10 13:00:10 +08:00
weishao 76f677e716 Merge branch 'clone_detection' into merge 2023-03-10 12:48:40 +08:00
weishao 69f7a292da nil更新 2023-03-10 12:48:10 +08:00
weishao 3658950778 nil更新 2023-03-10 12:47:21 +08:00
jshixiong 6e06d1c503 fix 组件漏洞列表 2023-03-10 11:36:42 +08:00
jshixiong 6f94977201 Merge remote-tracking branch 'origin/merge' into merge 2023-03-10 11:25:20 +08:00
jshixiong 50ff819dcf fix 2023-03-10 11:25:05 +08:00
youys e58571fc56 Merge branch 'merge' of https://gitlink.org.cn/youys/quality_analysis into merge 2023-03-10 11:18:44 +08:00
youys 9fcc9b743f 增加修复建议字段 2023-03-10 11:18:40 +08:00
jshixiong 2fccbd18f7 fix 2023-03-10 11:13:24 +08:00
jshixiong ebc896c525 page 2023-03-10 11:11:24 +08:00
jshixiong 281dfc78f3 bug type 2023-03-10 10:59:43 +08:00
jshixiong 86aba22812 fix 2023-03-10 10:16:37 +08:00
jshixiong 4ed06f6e1c fix 2023-03-10 09:58:54 +08:00
jshixiong fb0e3bd0e2 bug right 2023-03-10 09:55:16 +08:00
jshixiong 11d2325931 fix 2023-03-10 09:22:34 +08:00
jshixiong 0a4ea731df bug 2023-03-09 19:57:15 +08:00
52 changed files with 1164 additions and 593 deletions

View File

@ -133,41 +133,6 @@ public class DynamicDataSourceConfig {
@Value("${spring.datasource.gitea.druid.test-while-idle}")
private boolean giteaTestWhileIdle;
// --------------------------sast----------------------------------
@Value("${spring.datasource.sast.driver-class-name}")
private String sastDriverClass;
@Value("${spring.datasource.sast.url}")
private String sastUrl;
@Value("${spring.datasource.sast.username}")
private String sastUsername;
@Value("${spring.datasource.sast.password}")
private String sastPassword;
@Value("${spring.datasource.sast.druid.initial-size}")
private Integer sastInitialSize;
@Value("${spring.datasource.sast.druid.max-active}")
private Integer sastMaxActive;
@Value("${spring.datasource.sast.druid.min-idle}")
private Integer sastMinIdle;
@Value("${spring.datasource.sast.druid.validation-query}")
private String sastValidationQuery;
@Value("${spring.datasource.sast.druid.test-on-borrow}")
private boolean sastTestOnBorrow;
@Value("${spring.datasource.sast.druid.test-on-return}")
private boolean sastTestOnReturn;
@Value("${spring.datasource.sast.druid.test-while-idle}")
private boolean sastTestWhileIdle;
@Bean("master")
public DataSource masterDataSource() {
@ -237,29 +202,6 @@ public class DynamicDataSourceConfig {
return giteaDataSource;
}
@Bean("sast")
public DataSource sastDataSource() {
DruidDataSource sastDataSource = new DruidDataSource();
sastDataSource.setDriverClassName(sastDriverClass);
sastDataSource.setUrl(sastUrl);
sastDataSource.setUsername(sastUsername);
sastDataSource.setPassword(sastPassword);
sastDataSource.setMinIdle(sastMinIdle);
sastDataSource.setMaxActive(sastMaxActive);
sastDataSource.setMinIdle(sastMinIdle);
sastDataSource.setTestOnBorrow(sastTestOnBorrow);
sastDataSource.setTestOnReturn(sastTestOnReturn);
sastDataSource.setTestWhileIdle(sastTestWhileIdle);
sastDataSource.setValidationQuery(sastValidationQuery);
// 连接失败后中断---重试10次
sastDataSource.setBreakAfterAcquireFailure(true);
sastDataSource.setConnectionErrorRetryAttempts(10);
// 快速失败
sastDataSource.setFailFast(true);
return sastDataSource;
}
@Bean
@Primary
public DataSource dynamicDataSource() {
@ -267,7 +209,6 @@ public class DynamicDataSourceConfig {
dataSourceMap.put(MASTER, masterDataSource());
dataSourceMap.put(READONLY, slaveDataSource());
dataSourceMap.put(GITEA, giteaDataSource());
dataSourceMap.put(SAST, sastDataSource());
//设置动态数据源
DynamicDataSource dynamicDataSource = new DynamicDataSource();
dynamicDataSource.setTargetDataSources(dataSourceMap);

View File

@ -1,35 +0,0 @@
package net.educoder.quality.common.config;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.transaction.PlatformTransactionManager;
import tk.mybatis.spring.annotation.MapperScan;
import javax.sql.DataSource;
@Configuration
@MapperScan(basePackages = "net.educoder.quality.mapper.sast", sqlSessionFactoryRef = "sqlSessionFactorySastDataSource")
public class SastDatasourceConfig {
@Bean(name = "sqlSessionFactorySastDataSource")
public SqlSessionFactory sqlSessionFactoryDdsDataSource(@Qualifier("sast") DataSource sastDataSource) throws Exception {
SqlSessionFactoryBean factoryBean = new SqlSessionFactoryBean();
factoryBean.setDataSource(sastDataSource);
factoryBean.setTypeAliasesPackage("net.educoder.quality.entity.sast");
factoryBean.setMapperLocations(new PathMatchingResourcePatternResolver().getResources("classpath:sast.mapper/*Mapper.xml"));
return factoryBean.getObject();
}
@Bean(name = "sastPlatformTransactionManager")
public PlatformTransactionManager platformTransactionManager(@Qualifier("sast") DataSource sastDataSource) {
DataSourceTransactionManager dataSourceTransactionManager = new DataSourceTransactionManager();
dataSourceTransactionManager.setDataSource(sastDataSource);
return dataSourceTransactionManager;
}
}

View File

@ -20,4 +20,6 @@ public interface QualityConstants {
String TAR_GZ = "tar.gz";
String SUCCESS = "SUCCESS";
String GET_LANGUAGE_API = "/api/v1/repos/%s/%s/issues/statistics/language";
}

View File

@ -13,8 +13,8 @@ import lombok.Getter;
public enum DetectResultCompareEnum {
COMPONENT(0, "组件"),
VULNERABILITY(1, "漏洞"),
BUG(2, "缺陷"),
BUG(1, "缺陷"),
VULNERABILITY(2, "漏洞"),
LICENSE(3, "许可证"),
;

View File

@ -0,0 +1,31 @@
package net.educoder.quality.common.enums;
/**
* sast bug enum
* @author jshixiong
*/
public enum SastBugTypeEnum {
DEADLY("0","致命"),
SEVERITY("1","严重"),
ORDINARY("2","一般"),
HINT("3","提示"),
FORCE("4","强制"),
PROPOSAL("5","建议");
String value;
String description;
SastBugTypeEnum(String value, String description) {
this.value = value;
this.description = description;
}
public String getValue() {
return value;
}
public String getDescription() {
return description;
}
}

View File

@ -147,7 +147,7 @@ public class WordUtil {
for (XWPFParagraph paragraph : paragraphs) {
// 待替换文本
String text = paragraph.getText();
if (StringUtils.isNotEmpty(text) && text.indexOf(key) != -1) {
if (StringUtils.isNotEmpty(text) && text.contains(key)) {
List<XWPFRun> runs = paragraph.getRuns();
// 只保留第一个Run
for (int i = (runs.size() - 1); i > 0; i--) {

View File

@ -39,7 +39,7 @@ public class ComponentController {
*/
@PostMapping("/component/{projectId}/parse")
public R parse(@PathVariable Long projectId) {
componentService.parseComponent(projectId);
componentService.parseComponent(projectId,1);
return R.success();
}

View File

@ -74,6 +74,18 @@ public class ProjectController {
return R.success();
}
/**
* drone项目检测
*
* @param detectionVO
* @return
*/
@PostMapping("/projects/droneDetect")
@OperateLogAnnotation(description = "drone项目检测")
public R<String> droneDetection(@RequestBody @Valid DroneDetectionVO detectionVO) {
String taskId = projectService.droneDetect(detectionVO);
return R.success(taskId);
}
/**
* 项目检测
@ -112,6 +124,7 @@ public class ProjectController {
*/
@PostMapping("/projects/detectResultCompare")
@OperateLogAnnotation(description = "检测结果对比")
@Deprecated
public R detectResultCompare(@RequestBody @Valid DetectResultCompareVO detectResultCompareVO) {
CompareResultDTO<?> compareResultDTO = projectService.detectResultCompare(detectResultCompareVO);
return R.success(compareResultDTO);
@ -177,12 +190,10 @@ public class ProjectController {
*/
@GetMapping("/projects/{projectId}/bug/center")
public R<PageInfo> projectBugCenter(@PathVariable Long projectId, @Valid ProjectBugCenterVO projectBugCenterVO) {
PageInfo<ProjectBugCenterDTO> projectBugCenterDTOPageInfo = projectService.projectBugCenter(projectId, projectBugCenterVO);
PageInfo<ProjectSastBugCenterDTO> projectBugCenterDTOPageInfo = projectService.projectSastBugCenter(projectId, projectBugCenterVO);
return R.success(projectBugCenterDTOPageInfo);
}
/**
* 缺陷列表导出
*
@ -190,7 +201,7 @@ public class ProjectController {
*/
@GetMapping("/projects/{projectId}/bug/export")
public void projectBugExport(@PathVariable Long projectId, @Valid ProjectBugCenterVO projectBugCenterVO, HttpServletResponse response) throws Exception {
List<ProjectBugCenterDTO> projectBugCenterDTOList = projectService.projectBugList(projectId, projectBugCenterVO);
List<ProjectSastBugCenterDTO> projectBugCenterDTOList = projectService.projectSastBugList(projectId, projectBugCenterVO);
Projects projects = projectService.getProjectById(projectId);
response.setCharacterEncoding("utf-8");
@ -198,7 +209,7 @@ public class ProjectController {
response.setHeader("Content-disposition", "attachment;filename=" + fileName + "_bug.xlsx");
response.setContentType("application/vnd.ms-excel");
EasyExcel.write(response.getOutputStream(), ProjectBugCenterDTO.class).sheet().doWrite(projectBugCenterDTOList);
EasyExcel.write(response.getOutputStream(), ProjectSastBugCenterDTO.class).sheet().doWrite(projectBugCenterDTOList);
}
/**

View File

@ -24,6 +24,7 @@ import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import javax.validation.Valid;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.*;
/**
@ -88,14 +89,14 @@ public class ReportCenterController {
Projects projects = projectService.getProjectById(reportCenter.getProjectId());
// 设置文件下载参数
response.setCharacterEncoding("utf-8");
String fileName = new String(reportCenter.getReportName().getBytes("utf-8"), "ISO8859-1");
String fileName = new String(reportCenter.getReportName().getBytes(StandardCharsets.UTF_8), StandardCharsets.ISO_8859_1);
// 获取模板填充数据
InputStream resourceAsStream = this.getClass().getClassLoader().getResourceAsStream("template/report-center-template.docx");
ServletOutputStream outputStream = response.getOutputStream();
ReportDetailDTO reportDetail = pgProjectsService.getReportDetail(projects.getProjectName());
ReportDetailDTO reportDetail = reportCenterService.getReportDetail(projects.getId());
Map<String, String> param = new HashMap<>(13);
param.put("projectName", projects.getProjectName());
@ -128,78 +129,4 @@ public class ReportCenterController {
}
}
/**
* 报告下载
*
* @param reportIds
* @return
*/
@GetMapping("/report/batchDownload")
public void batchDownloadReport(@RequestParam("reportIds") List<Long> reportIds, HttpServletResponse response) throws IOException {
// 设置文件下载参数
response.setCharacterEncoding("utf-8");
String fileName = new String("报告中心".getBytes("utf-8"), "ISO8859-1");
response.setHeader("Content-disposition", "attachment;filename=" + fileName + ".zip");
response.setContentType("application/octet-stream");
ServletOutputStream outputStream = response.getOutputStream();
String[] filePaths = new String[reportIds.size()];
InputStream[] ins = new InputStream[reportIds.size()];
int i = 0;
for (Long reportId : reportIds) {
ReportCenter reportCenter = reportCenterService.reportCenterDetail(reportId);
Projects projects = projectService.getProjectById(reportCenter.getProjectId());
// 获取模板填充数据
InputStream resourceAsStream = this.getClass().getClassLoader().getResourceAsStream("template/report-center-template.docx");
ReportDetailDTO reportDetail = pgProjectsService.getReportDetail(projects.getProjectName());
Map<String, String> param = new HashMap<>(13);
param.put("projectName", projects.getProjectName());
param.put("createTime", DateUtil.formatDateTime(new Date()));
param.put("issueCount",String.valueOf(reportDetail.getBugCount().getTotal()));
param.put("seriousIssueCount",String.valueOf(reportDetail.getBugCount().getCritical()));
param.put("highRiskIssueCount",String.valueOf(reportDetail.getBugCount().getHigh()));
param.put("midRiskIssueCount",String.valueOf(reportDetail.getBugCount().getMiddle()));
param.put("lowRiskIssueCount",String.valueOf(reportDetail.getBugCount().getLow()));
param.put("componentCount",String.valueOf(reportDetail.getComponents()));
param.put("componentBugCount",String.valueOf(reportDetail.getVulnerabilityCount().getTotal()));
param.put("seriousBugCount",String.valueOf(reportDetail.getVulnerabilityCount().getCritical()));
param.put("highRiskBugCount",String.valueOf(reportDetail.getVulnerabilityCount().getHigh()));
param.put("midRiskBugCount",String.valueOf(reportDetail.getVulnerabilityCount().getMiddle()));
param.put("lowRiskBugCount",String.valueOf(reportDetail.getVulnerabilityCount().getLow()));
XWPFDocument doc = new XWPFDocument(resourceAsStream);
WordUtil.changeText(doc, param);
pgProjectsService.fillReportTable(doc,reportDetail);
String filePath = filePath(reportCenter.getReportName());
doc.write(new FileOutputStream(new File(filePath)));
doc.close();
ins[i] = new FileInputStream(new File(filePath));
filePaths[i] = filePath;
i++;
}
ZipUtil.zip(outputStream, filePaths, ins);
}
private String filePath(String reportName){
String path = StringUtils.join(downloadTempDir, "/", IdUtil.fastSimpleUUID(), "/");
String name = reportName + ".docx";
File file = new File(path);
if (!file.exists()) {
file.mkdirs();
}
return path + name;
}
}

View File

@ -23,6 +23,6 @@ public class ClonePairDTO {
this.targetFileStartLine = targetFileStartLine;
this.targetFileEndLine = targetFileEndLine;
this.similarity = similarity;
this.similarLines = Math.max(Math.min(sourceFileEndLine - sourceFileStartLine, targetFileEndLine - targetFileStartLine), 0);
this.similarLines = Math.max(Math.min(sourceFileEndLine - sourceFileStartLine + 1, targetFileEndLine - targetFileStartLine + 1), 0);
}
}

View File

@ -25,6 +25,10 @@ public class ProjectBugCenterCodeDetailV2DTO {
* 缺陷描述
*/
private String description;
/**
* 修复建议
*/
private String repairOpinion;
/**
* 正确代码示例
*/

View File

@ -46,6 +46,11 @@ public class ProjectBugCenterDTO {
@ExcelProperty("检测时间")
private Date detectTime;
/**
* 是否为增量;0否1是
*/
private Integer isIncrement;
/**
* 找文件唯一标识
*/

View File

@ -74,27 +74,32 @@ public class ProjectBugDetailDTO {
private Long bugTotalNumber;
/**
* 严重的 对应 critical
* DEADLY 0 致命
*/
private Long critical;
/**
* 高危 对应 blocker
*/
private Long high;
private Long deadly = 0L;
/**
* 对应 major
* SEVERITY 1 严重
*/
private Long middle;
private Long severity = 0L;
/**
* 对应 minor
* ORDINARY 2 一般
*/
private Long low;
private Long ordinary = 0L;
/**
* 未知
* HINT 3 提示
*/
private Long unKnow = 0L;
private Long hint = 0L;
/**
* FORCE 4 强制
*/
private Long force = 0L;
/**
* PROPOSAL 5 建议
*/
private Long proposal = 0L;
}
}

View File

@ -13,7 +13,8 @@ import lombok.Data;
public class ProjectMetricsDTO {
private Integer bugNumber;
private Integer vulnerabilityNumber;
private Integer newBugNumber;
/**
* 代码行数

View File

@ -72,7 +72,7 @@ public class ProjectOverviewDTO {
/**
* 文件大小
*/
private String size = "0k";
private Integer size = 0;
}
/**

View File

@ -0,0 +1,63 @@
package net.educoder.quality.dto;
import com.alibaba.excel.annotation.ExcelIgnoreUnannotated;
import com.alibaba.excel.annotation.ExcelProperty;
import lombok.Data;
import java.util.Date;
/**
* @Author: jshixiong
* @Date: 2023/3/9
* @Description:
*/
@Data
@ExcelIgnoreUnannotated
public class ProjectSastBugCenterDTO {
/**
* 缺陷名称
*/
@ExcelProperty("缺陷名称")
private String bugName;
/**
* 缺陷描述
*/
@ExcelProperty("缺陷描述")
private String bugDescription;
/**
* 文件路径
*/
@ExcelProperty("文件路径")
private String filePath;
/**
* 行号
*/
@ExcelProperty("行号")
private String rowNumber;
/**
* 缺陷级别
*/
@ExcelProperty("缺陷级别")
private String bugLevel;
/**
* 检测时间
*/
@ExcelProperty("检测时间")
private Date detectTime;
@ExcelProperty("缺陷分类")
private String parentBugCatalog;
/**
* 是否为增量
*/
private Integer isIncrement;
/**
* 找文件唯一标识
*/
private String uuid;
private Integer ruleId;
}

View File

@ -52,4 +52,9 @@ public class ProjectVulnerabilityDetailDTO {
*/
private String repairSuggestions;
/**
* 是否为增量
*/
private Integer isIncrement;
}

View File

@ -4,7 +4,7 @@ package net.educoder.quality.dto;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import net.educoder.quality.entity.postgres.PgIssues;
import net.educoder.quality.entity.mysql.SastAnalysisDetail;
import java.util.List;
@ -23,7 +23,7 @@ public class ReportDetailDTO {
private Integer components;
private List<PgIssues> bugIssues;
private List<SastAnalysisDetail> bugIssues;
private List<ComponentParseRecordDetailDTO> parseRecordDetails;
}

View File

@ -0,0 +1,35 @@
package net.educoder.quality.dto;
import lombok.Data;
@Data
public class SastBugResultDTO extends BugResultDTO{
/**
* DEADLY 0
*/
private Long deadly = 0L;
/**
* SEVERITY 对应 1
*/
private Long severity = 0L;
/**
* ORDINARY 对应 2
*/
private Long ordinary = 0L;
/**
* HINT 对应 3
*/
private Long hint = 0L;
/**
* FORCE 对应 4
*/
private Long force = 0L;
/**
* PROPOSAL 对应 5
*/
private Long proposal = 0L;
}

View File

@ -1,8 +1,6 @@
package net.educoder.quality.dto;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* @Author: youys
@ -11,4 +9,6 @@ import lombok.NoArgsConstructor;
*/
@Data
public class VulnerabilityResultDTO extends CompareBaseResultDTO{
private Long newTotal = 0L;
}

View File

@ -1,4 +1,4 @@
package net.educoder.quality.entity.sast;
package net.educoder.quality.entity.mysql;
import java.util.Date;
import javax.persistence.*;

View File

@ -11,4 +11,5 @@ public class ComponentParseRecord extends AbstractDO {
private String result;
private String taskId;
}

View File

@ -58,6 +58,11 @@ public class ProjectVulnerabilityDetail extends AbstractDO {
*/
private Long projectId;
/**
* 是否为增量
*/
private Integer isIncrement;
@Override
public boolean equals(Object o) {
if (this == o){

View File

@ -1,6 +1,7 @@
package net.educoder.quality.entity.mysql;
import java.util.Date;
import java.util.Objects;
import javax.persistence.*;
import lombok.Data;
import net.educoder.quality.common.util.AbstractDO;
@ -130,6 +131,11 @@ public class SastAnalysisDetail extends AbstractDO {
@Column(name = "file_path")
private String filePath;
/**
* 是否为增量
*/
private Integer isIncrement;
public static final String ID = "id";
public static final String DB_ID = "id";
@ -205,4 +211,45 @@ public class SastAnalysisDetail extends AbstractDO {
public static final String ERROR_EXAMPLE = "errorExample";
public static final String DB_ERROR_EXAMPLE = "error_example";
@Override
public boolean equals(Object o) {
if (this == o){
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
SastAnalysisDetail detail = (SastAnalysisDetail) o;
return strEquals(getBugCatalog(), detail.getBugCatalog()) && strEquals(getBugNo(), detail.getBugNo()) && strEquals(getBugName(), detail.getBugName()) && strEquals(getParentBugNo(), detail.getParentBugNo()) && strEquals(getBugType(), detail.getBugType()) && strEquals(getBugTypeName(), detail.getBugTypeName()) && strEquals(getBugLevel(), detail.getBugLevel()) && strEquals(getMethodName(), detail.getMethodName()) && intEquals(getLine(), detail.getLine()) && pathEquals(getFilePath(), detail.getFilePath());
}
private boolean strEquals(String a,String b){
if (a==null || "".equals(a)){
return b == null || "".equals(b);
}
return a.equals(b);
}
private boolean intEquals(Integer a,Integer b){
if (a==null){
return b == null;
}
return a.equals(b);
}
private boolean pathEquals(String a,String b){
if (a==null || "".equals(a)){
return b == null || "".equals(b);
}
a = a.substring(a.lastIndexOf('/'));
b = b.substring(b.lastIndexOf('/'));
System.out.println(a+":"+b);
return a.equals(b);
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), getBugCatalog(), getBugNo(), getBugName(), getParentBugNo(), getBugType(), getBugTypeName(), getBugLevel(), getMethodName(), getLine(), getFilePath());
}
}

View File

@ -0,0 +1,9 @@
package net.educoder.quality.mapper.mysql;
import net.educoder.quality.entity.mysql.BizKnowledgeBug;
import java.util.List;
public interface BizKnowledgeBugMapper {
List<BizKnowledgeBug> selectBugAndCatalogByTypes(List<String> bugTypes);
}

View File

@ -21,4 +21,11 @@ public interface ComponentParseRecordMapper extends BaseMapper<ComponentParseRec
* @return
*/
List<ComponentParseRecord> quaryByProjectId(Long projectId);
/**
* getByTaskId
* @param taskId
* @return
*/
ComponentParseRecord getByTaskId(String taskId);
}

View File

@ -36,6 +36,14 @@ public interface ProjectDetectionTaskInfoMapper {
*/
List<ProjectDetectionTaskInfo> selectByProjectId(@Param("projectId") Long projectId, @Param("status") Integer status);
/**
* 根据projectId查询检测记录列表(不含未成功)
*
* @param projectId 项目id
* @return
*/
List<ProjectDetectionTaskInfo> selectByOnlyProjectId(@Param("projectId") Long projectId);
/**
* 获取最近的一次成功检测记录

View File

@ -3,5 +3,8 @@ package net.educoder.quality.mapper.mysql;
import net.educoder.quality.common.util.BaseMapper;
import net.educoder.quality.entity.mysql.SastAnalysisDetail;
import java.util.List;
public interface SastAnalysisDetailMapper extends BaseMapper<SastAnalysisDetail> {
}

View File

@ -74,6 +74,7 @@ public interface PgProjectMapper {
* @param issueType
* @return
*/
@Deprecated
Integer findIssueCountByProjectNameAndIssueType(String projectName, Integer issueType);
/**
@ -81,6 +82,7 @@ public interface PgProjectMapper {
* @param projectName
* @return
*/
@Deprecated
Integer findByCodeNumber(String projectName);

View File

@ -1,10 +0,0 @@
package net.educoder.quality.mapper.sast;
import net.educoder.quality.common.util.BaseMapper;
import net.educoder.quality.entity.sast.BizKnowledgeBug;
import java.util.List;
public interface BizKnowledgeBugMapper {
List<BizKnowledgeBug> selectBugAndCatalogByTypes(List<String> bugTypes);
}

View File

@ -21,7 +21,9 @@ public class ComponentTask {
/**
* 晚上0点1分执行
*/
@Scheduled(cron = "0 1 0 * * ?")
//取消该定时任务
// @Scheduled(cron = "0 1 0 * * ?")
@Deprecated
public void timingAnalysisTask() {
log.info("-----------------ComponentTask.timingAnalysisTask start-------------");

View File

@ -18,14 +18,22 @@ public interface ComponentService {
/**
* 解析组件
* @param projectId
* @param type 0增量1全量
*/
void parseComponent(Long projectId);
void parseComponent(Long projectId,Integer type);
/**
* 解析组件
* @param projectId
* @param type 0增量1全量
*/
void parseComponent(Long projectId,Integer type,String taskId);
/**
* 解析组件
* @param projects
*/
void parseComponent(Projects projects);
void parseComponent(Projects projects,Integer type,String taskId);
/**
* 组件统计

View File

@ -53,6 +53,12 @@ public interface ProjectService {
*/
String detection(DetectionVO detectionVO);
/**
* drone检测
* @return
*/
String droneDetect(DroneDetectionVO droneDetectionVO);
/**
* 更新task和project状态
*
@ -126,16 +132,34 @@ public interface ProjectService {
* @param projectId
* @param projectBugCenterVO
*/
@Deprecated
PageInfo<ProjectBugCenterDTO> projectBugCenter(Long projectId, ProjectBugCenterVO projectBugCenterVO);
/**
* 缺陷中心-sast
* @param projectId
* @param projectBugCenterVO
* @return
*/
PageInfo<ProjectSastBugCenterDTO> projectSastBugCenter(Long projectId, ProjectBugCenterVO projectBugCenterVO);
/**
* 缺陷列表
*
* @param projectId
* @param projectBugCenterVO
*/
@Deprecated
List<ProjectBugCenterDTO> projectBugList(Long projectId, ProjectBugCenterVO projectBugCenterVO);
/**
* 缺陷列表-sast
*
* @param projectId
* @param projectBugCenterVO
*/
List<ProjectSastBugCenterDTO> projectSastBugList(Long projectId, ProjectBugCenterVO projectBugCenterVO);
/**
* 获取缺陷中心代码详情

View File

@ -1,6 +1,7 @@
package net.educoder.quality.service;
import com.github.pagehelper.PageInfo;
import net.educoder.quality.dto.ReportDetailDTO;
import net.educoder.quality.entity.mysql.ReportCenter;
import net.educoder.quality.vo.ExportReportVO;
import net.educoder.quality.vo.ReportListVO;
@ -33,4 +34,11 @@ public interface ReportCenterService {
* @return
*/
ReportCenter reportCenterDetail(Long reportId);
/**
* 查询报告详细参数
* @param projectId
* @return
*/
ReportDetailDTO getReportDetail(Long projectId);
}

View File

@ -3,7 +3,7 @@ package net.educoder.quality.service;
import net.educoder.quality.entity.mysql.SastAnalysisDetail;
public interface SastService {
void analysis(Long projectId, String taskId);
void analysis(Long projectId, String taskId,Integer type);

View File

@ -30,7 +30,7 @@ import net.educoder.quality.service.ProjectService;
import net.educoder.quality.vo.AddCloneDetectionVO;
import net.educoder.quality.vo.PageVO;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.commons.io.FileUtils;
import org.aspectj.util.FileUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -38,9 +38,13 @@ import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.LineNumberReader;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Slf4j
@Service
@ -60,6 +64,98 @@ public class CloneDetectionServiceImpl implements CloneDetectionService {
@Resource
private CloneDetectionMapper cloneDetectionMapper;
private static final String[] textFileSuffix = new String[]{
"py",
"h",
"c",
"cpp",
"cc",
"java",
"php",
"html",
"css",
"scss",
"go",
"r",
"graphql",
"swift",
"xml",
"yaml",
"json",
"lua",
"scheme",
"less",
"ini",
"coffee",
"litcoffee",
"js",
"cs",
"kt",
"md",
"sql",
"m",
"mm",
"pas",
"perl",
"ejs",
"pl",
"rb",
"rs",
"rust",
"sh",
"makefile",
"circ",
"readme",
"yml",
"sml",
"conf",
"txt",
"gitignore",
"in",
"cu",
"gemfile",
"scala",
"net",
"l",
"v",
"config",
"properties",
"log",
"htm",
"cnf",
"hex",
"bat",
"asm",
"bash",
"ts",
"tsx",
"sass",
"jsx",
"jsp",
"gitkeep",
"sv",
"hql",
"y",
"jj",
"pls",
"sol",
"ignore",
"ctrl",
"vue",
"tex",
"bib",
"cls",
"bst",
"toc",
"sty",
"g4",
"sy",
"ipynb",
"m",
"mm",
"groovy"
};
@Override
@Transactional(rollbackFor = Exception.class)
public void add(Long projectId, AddCloneDetectionVO addCloneDetectionVO) {
@ -155,18 +251,21 @@ public class CloneDetectionServiceImpl implements CloneDetectionService {
log.error("save clone detail to database failed, source: {}, target: {}", fullPath + clonePairDTO.getSourceFile(), fullPath + clonePairDTO.getTargetFile(), e);
}
}
// 取总文件信息记录各项值
List<File> files = getFiles(fullPath + "/source/" + project.getProjectName());
CloneDetectionResultProjectLevel cloneDetectionResultProjectLevel = new CloneDetectionResultProjectLevel();
cloneDetectionResultProjectLevel.setCloneDetectionId(cloneDetectionId);
cloneDetectionResultProjectLevel.setSimilarityFile(similarSourceLinesMap.size());
long totalFile = getFileCount(fullPath + "/source");
long totalFile = files.size();
cloneDetectionResultProjectLevel.setSimilarityFilePercent(Math.min(((double)similarSourceLinesMap.size()) / totalFile, 1));
cloneDetectionResultProjectLevel.setTotalFile(totalFile);
cloneDetectionResultProjectLevel.setSimilarityLine(similarLines);
long totalLine = getLinesCount(fullPath + "/source/" + project.getProjectName());
long totalLine = getLinesCount(files);
cloneDetectionResultProjectLevel.setSimilarityLinePercent(Math.min(((double)similarLines) / totalLine, 1));
cloneDetectionResultProjectLevel.setTotalLine(totalLine);
cloneDetectionResultProjectLevel.setSimilarityCapacity(similarCapacityCount);
long totalCapacity = getCapacityCount(fullPath + "/source/" + project.getProjectName());
long totalCapacity = getCapacityCount(files);
cloneDetectionResultProjectLevel.setSimilarityCapacityPercent(Math.min(((double)similarCapacityCount) / totalCapacity, 1));
cloneDetectionResultProjectLevel.setTotalCapacity(totalCapacity);
cloneDetectionResultProjectLevel.setTargetProjectUrl(targetProjectUrl);
@ -267,7 +366,7 @@ public class CloneDetectionServiceImpl implements CloneDetectionService {
String resultCSVPath = sourcePath + "/result.csv";
// java -jar /Users/weiwang/IdeaProjects/NIL/build/libs/NIL-all.jar -s /Users/weiwang/IdeaProjects/bridge/business-service/game/src/main/java/com/educoder/bridge/game -o /tmp/a.csv
String command = "java -jar " + jarPath + " -o " + resultCSVPath +
" -s " + sourcePath + " -l " + language + " -mil " + propertiesConfig.getMil()
" -s " + sourcePath + " -l " + language.trim().replace(" ", "") + " -mil " + propertiesConfig.getMil()
+ " -mit " + propertiesConfig.getMit() + " -f " + propertiesConfig.getFiltrationThreshold()
+ " -v " + propertiesConfig.getVerificationThreshold();
String nilOut = ShellUtil.execute(command);
@ -310,30 +409,39 @@ public class CloneDetectionServiceImpl implements CloneDetectionService {
/**
* 统计文件夹下文件个数
*/
private int getFileCount(String repoPath) {
String cnt = ShellUtil.execute("ls -lR " + repoPath + "| grep \"^-\" | wc -l");
return NumberUtils.toInt(cnt, 0);
private List<File> getFiles(String repoPath) {
Collection<File> files = FileUtils.listFiles(new File(repoPath), textFileSuffix, true);
return files.stream().filter(file -> !file.isHidden()).collect(Collectors.toList());
}
/**
* 统计文件夹下文件行数
*/
private int getLinesCount(String repoPath) {
String cntResult = ShellUtil.execute("cd " + repoPath + " && git ls-files | xargs cat | wc -l");
String[] split = cntResult.trim().split("\n");
return NumberUtils.toInt(split[split.length - 1].trim(), Integer.MAX_VALUE);
private int getLinesCount(List<File> files) {
int count = 0;
for (File file : files) {
FileReader in;
try {
in = new FileReader(file);
LineNumberReader reader = new LineNumberReader(in);
reader.skip(Long.MAX_VALUE);
int lines = reader.getLineNumber();
reader.close();
count+= lines;
} catch (IOException e) {
}
}
return count;
}
/**
* 统计文件夹大小
* @param repoPath
* @param files
* @return
*/
private long getCapacityCount(String repoPath) {
// mac上为String cntResult = ShellUtil.execute("cd " + repoPath + " && du -s -k -I \"\\.git\" | awk '{print $1}'");
String cntResult = ShellUtil.execute("cd " + repoPath + " && du -s -k --exclude=\"\\.git\" | awk '{print $1}'");
String[] split = cntResult.trim().split("\n");
return NumberUtils.toInt(split[split.length - 1].trim(), Integer.MAX_VALUE) * 1024L;
private long getCapacityCount(List<File> files) {
return files.stream().mapToLong(File::length).reduce(0, Long::sum);
}
}

View File

@ -84,14 +84,21 @@ public class ComponentServiceImpl implements ComponentService {
@Override
@Transactional(rollbackFor = Exception.class)
public void parseComponent(Long projectId) {
public void parseComponent(Long projectId,Integer type) {
Projects projects = projectService.getProjectById(projectId);
parseComponent(projects);
parseComponent(projects,type,null);
}
@Override
@Transactional(rollbackFor = Exception.class)
public void parseComponent(Projects projects) {
public void parseComponent(Long projectId,Integer type, String taskId){
Projects projects = projectService.getProjectById(projectId);
parseComponent(projects,type,taskId);
}
@Override
@Transactional(rollbackFor = Exception.class)
public void parseComponent(Projects projects,Integer type,String taskId) {
if (projects == null) {
throw new BusinessException(ErrorCodeEnum.PROJECT_NOT_EXISTS);
}
@ -113,14 +120,15 @@ public class ComponentServiceImpl implements ComponentService {
//每次都重新分析
String outPath = fullPath + "/output.json";
String url = "http://opensca.xmirror.cn:8003";
String url = "https://opensca.xmirror.cn";
String token = "6cb2f6fb-cf87-463d-b5a3-d242eb97d7c1";
String command = StringUtils.join(openScaCliPath," -url "
, url, " -token ", token, " -path ", fullPath, " -out ", outPath);
ShellResult shellResult = ShellUtil.executeAndGetExitStatus(command);
log.info("command:{}, result:{}", command, shellResult);
log.info("组件分析parseComponent command:{}, result:{}", command, shellResult);
componentParseRecord = new ComponentParseRecord();
componentParseRecord.setProjectId(projectId);
componentParseRecord.setTaskId(taskId);
componentParseRecord.setCreateTime(new Date());
componentParseRecord.setUpdateTime(new Date());
@ -141,29 +149,11 @@ public class ComponentServiceImpl implements ComponentService {
componentParseRecord.setResult("");
componentParseRecordMapper.insertSelective(componentParseRecord);
}
// } else {
// log.info("projectId:{},组件已解析", projectId);
// ComponentParseRecordDetail detailQuery = new ComponentParseRecordDetail();
// detailQuery.setComponentId(componentParseRecord.getId());
// int count = componentParseRecordDetailMapper.selectCount(detailQuery);
// if (count > 0) {
// log.info("projectId:{},组件已解析,并且已入库", projectId);
// return;
// }
//
// String result = componentParseRecord.getResult();
// parseResult(result, componentParseRecord.getId(), parseRecordDetailSet);
// log.info("else需要插入的组件详情数据有{}条", parseRecordDetailSet.size());
// if (CollectionUtils.isNotEmpty(parseRecordDetailSet)) {
// List<ComponentParseRecordDetail> batchList = new ArrayList<>(parseRecordDetailSet);
// componentParseRecordDetailMapper.insertList(batchList);
// }
// }
// 插入漏洞
log.info("开始插入漏洞");
try{
batchInsertVulnerability(parseRecordDetailSet, projectId,componentParseRecord.getId());
batchInsertVulnerability(parseRecordDetailSet, projectId,componentParseRecord.getId(), type);
log.info("插入漏洞成功");
}catch (Exception e){
log.info("插入漏洞失败");
@ -174,7 +164,7 @@ public class ComponentServiceImpl implements ComponentService {
/**
* 批量插入漏洞增量
*/
private void batchInsertVulnerability(Set<ComponentParseRecordDetail> parseRecordDetailSet, Long projectId, Long componentId) {
private void batchInsertVulnerability(Set<ComponentParseRecordDetail> parseRecordDetailSet, Long projectId, Long componentId, Integer type) {
List<ProjectVulnerabilityDetail> projectVulnerabilityDetailList = new ArrayList<>();
for (ComponentParseRecordDetail componentParseRecordDetail : parseRecordDetailSet) {
String vulnerabilities = componentParseRecordDetail.getVulnerabilities();
@ -195,76 +185,31 @@ public class ComponentServiceImpl implements ComponentService {
pvd.setProjectId(projectId);
pvd.setCreateTime(DateTime.now());
pvd.setUpdateTime(DateTime.now());
pvd.setIsIncrement(0);
projectVulnerabilityDetailList.add(pvd);
}
}
}
//新漏洞排除已有旧漏洞
List<ComponentParseRecord> parseRecords = componentParseRecordMapper.quaryByProjectId(projectId);
List<ProjectVulnerabilityDetail> oldVulnerabilityList = new ArrayList<>();
if (parseRecords.size()>=2){
Long oldComponentId = parseRecords.get(1).getId();
ProjectVulnerabilityDetail projectVulnerabilityDetail = new ProjectVulnerabilityDetail();
projectVulnerabilityDetail.setComponentId(oldComponentId);
oldVulnerabilityList = projectVulnerabilityDetailMapper.select(projectVulnerabilityDetail);
}
if (oldVulnerabilityList.size()>0){
Iterator<ProjectVulnerabilityDetail> it = projectVulnerabilityDetailList.iterator();
while (it.hasNext()){
ProjectVulnerabilityDetail newVulnerability = it.next();
//增量
if (type==0){
List<ComponentParseRecord> parseRecords = componentParseRecordMapper.quaryByProjectId(projectId);
List<ProjectVulnerabilityDetail> oldVulnerabilityList = new ArrayList<>();
if (parseRecords.size()>=2){
Long oldComponentId = parseRecords.get(1).getId();
ProjectVulnerabilityDetail projectVulnerabilityDetail = new ProjectVulnerabilityDetail();
projectVulnerabilityDetail.setComponentId(oldComponentId);
oldVulnerabilityList = projectVulnerabilityDetailMapper.select(projectVulnerabilityDetail);
}
for (ProjectVulnerabilityDetail newVulnerability:projectVulnerabilityDetailList){
newVulnerability.setIsIncrement(1);
for (ProjectVulnerabilityDetail oldVulnerability:oldVulnerabilityList) {
if (newVulnerability.equals(oldVulnerability)){
it.remove();
newVulnerability.setIsIncrement(0);
break;
}
}
}
//新漏洞加上所有旧漏洞
for (ProjectVulnerabilityDetail oldVulnerability:oldVulnerabilityList) {
oldVulnerability.setId(null);
oldVulnerability.setComponentId(componentId);
oldVulnerability.setProjectId(projectId);
oldVulnerability.setCreateTime(new Date());
oldVulnerability.setUpdateTime(new Date());
}
projectVulnerabilityDetailList.addAll(oldVulnerabilityList);
}
log.info("本次漏洞数据有{}条", projectVulnerabilityDetailList.size());
if (CollectionUtils.isNotEmpty(projectVulnerabilityDetailList)) {
projectVulnerabilityDetailMapper.insertList(projectVulnerabilityDetailList);
}
}
/**
* 批量插入漏洞全量
*/
private void batchInsertVulnerabilityAll(Set<ComponentParseRecordDetail> parseRecordDetailSet, Long projectId) {
List<ProjectVulnerabilityDetail> projectVulnerabilityDetailList = new ArrayList<>();
for (ComponentParseRecordDetail componentParseRecordDetail : parseRecordDetailSet) {
String vulnerabilities = componentParseRecordDetail.getVulnerabilities();
Long componentId = componentParseRecordDetail.getComponentId();
if (StringUtils.isNotEmpty(vulnerabilities)) {
JSONArray vuls = JSON.parseArray(vulnerabilities);
for (int i = 0; i < vuls.size(); i++) {
ProjectVulnerabilityDetail pvd = new ProjectVulnerabilityDetail();
JSONObject vul = vuls.getJSONObject(i);
pvd.setVulnerabilityNumber(StringUtils.firstNonBlank(vul.getString("cwe_id"), vul.getString("id").replaceAll("XMIRROR-", "")));
pvd.setDescription(vul.getString("description"));
pvd.setAttackType(vul.getString("attack_type"));
pvd.setDifficulty(vul.getInteger("exploit_level_id"));
pvd.setRiskLevel(vul.getInteger("security_level_id"));
pvd.setPublishDate(vul.getString("release_date"));
pvd.setRepairSuggestions(vul.getString("suggestion"));
pvd.setVulnerabilityName(vul.getString("name"));
pvd.setComponentId(componentId);
pvd.setProjectId(projectId);
pvd.setCreateTime(DateTime.now());
pvd.setUpdateTime(DateTime.now());
projectVulnerabilityDetailList.add(pvd);
}
}
}
log.info("本次漏洞数据有{}条", projectVulnerabilityDetailList.size());

View File

@ -1,23 +1,26 @@
package net.educoder.quality.service.impl;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.date.DateUnit;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSONObject;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
import net.educoder.quality.common.bean.ShellResult;
import net.educoder.quality.common.constant.QualityConstants;
import net.educoder.quality.common.enums.*;
import net.educoder.quality.common.exception.BusinessException;
import net.educoder.quality.common.util.ShellUtil;
import net.educoder.quality.common.util.SnowflakeIdWorker;
import net.educoder.quality.common.util.html.HtmlSourceDecorator;
import net.educoder.quality.dto.*;
import net.educoder.quality.entity.mysql.*;
import net.educoder.quality.entity.postgres.PgFileSource;
import net.educoder.quality.entity.postgres.PgIssues;
import net.educoder.quality.entity.postgres.PgProjectMeasures;
import net.educoder.quality.entity.postgres.PgRule;
import net.educoder.quality.mapper.mysql.*;
import net.educoder.quality.protobuf.DbFileSources;
@ -27,7 +30,6 @@ import net.educoder.quality.service.postgres.PgProjectsService;
import net.educoder.quality.task.SonarDetectionRunnable;
import net.educoder.quality.vo.*;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
@ -38,8 +40,6 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.io.File;
import java.io.IOException;
import java.util.*;
/**
@ -92,6 +92,9 @@ public class ProjectServiceImpl implements ProjectService {
@Value("${sonar.donet.serverUrl}")
private String donetSonarServer;
@Value("${openi.domain}")
private String openIDomain;
@Autowired
private ComponentService componentService;
@ -102,6 +105,9 @@ public class ProjectServiceImpl implements ProjectService {
@Resource
private SastService sastService;
@Autowired
private SastAnalysisDetailMapper sastMapper;
@Override
public PageInfo<ProjectsDTO> getProjectList(ProjectsVO projectsVO) {
@ -205,7 +211,7 @@ public class ProjectServiceImpl implements ProjectService {
/**
* 提交扫描任务
*/
submitScanTask(projects, taskInfo.getTaskId(), detectionTemplate, taskInfo);
// submitScanTask(projects, taskInfo.getTaskId(), detectionTemplate, taskInfo);
// 异步处理时间太长
@ -215,7 +221,7 @@ public class ProjectServiceImpl implements ProjectService {
/**
* 组件分析
*/
componentService.parseComponent(projects.getId());
componentService.parseComponent(projects.getId(),detectionVO.getType(),taskInfo.getTaskId());
}
});
@ -223,7 +229,7 @@ public class ProjectServiceImpl implements ProjectService {
sonarQueryResultThreadPool.execute(new Runnable() {
@Override
public void run() {
sastService.analysis(projects.getId(), taskInfo.getTaskId());
sastService.analysis(projects.getId(), taskInfo.getTaskId(),detectionVO.getType());
}
});
@ -241,6 +247,44 @@ public class ProjectServiceImpl implements ProjectService {
return taskInfo.getTaskId();
}
@Override
public String droneDetect(DroneDetectionVO droneDetectionVO) {
//根据创建者分支名和项目名查找project
Projects d = new Projects();
d.setBranch(droneDetectionVO.getBranchName());
d.setProjectName(droneDetectionVO.getRepository());
d.setCreator(droneDetectionVO.getRepoOwner());
List<Projects> select = projectsMapper.select(d);
if (select.size()<=0){
throw new BusinessException(ErrorCodeEnum.PARAM_ERROR.getValue(), "项目检测任务不存在");
}
Projects projects = select.get(0);
if (projects.getStatus() == 1) {
throw new BusinessException(ErrorCodeEnum.UNDER_DETECTION_ERROR);
}
//按最近一次成功的来
ProjectDetectionTaskInfo projectDetectionTaskInfo = projectDetectionTaskInfoMapper.selectLastSuccessByProjectId(projects.getId());
if (projectDetectionTaskInfo==null){
throw new BusinessException(ErrorCodeEnum.PARAM_ERROR.getValue(), "请先进行首次检测!");
}
DetectionTemplate detectionTemplate = detectionTemplateMapper.selectByPrimaryKey(projectDetectionTaskInfo.getTemplateId());
if (detectionTemplate == null) {
throw new BusinessException(ErrorCodeEnum.PARAM_ERROR.getValue(), "检测模板不存在");
}
DetectionVO detectionVO = new DetectionVO();
detectionVO.setProjectId(projects.getId());
detectionVO.setTemplateId(projectDetectionTaskInfo.getTemplateId());
detectionVO.setRepository(droneDetectionVO.getRepository());
detectionVO.setRepoOwner(droneDetectionVO.getRepoOwner());
detectionVO.setCurrentUser(droneDetectionVO.getRepoOwner());
detectionVO.setType(0);
return detection(detectionVO);
}
@Override
@Transactional(rollbackFor = Exception.class)
public void updateTaskAndProjectStatus(String taskId, int status, Integer filesNum, String fileSize) {
@ -304,11 +348,10 @@ public class ProjectServiceImpl implements ProjectService {
ProjectDetectionTaskInfo projectDetectionTaskInfo = projectDetectionTaskInfos.get(0);
ProjectDetectionTaskInfo projectDetectionTaskInfo2 = projectDetectionTaskInfos.get(1);
String projectName = projectDetectionTaskInfo.getProjectName() + "-" + projectDetectionTaskInfo.getRandomStr();
String projectName2 = projectDetectionTaskInfo2.getProjectName() + "-" + projectDetectionTaskInfo2.getRandomStr();
List<String> rulesDetail = pgProjectsService.getRulesDetail(projectName, detectResultCompareDetailVO.getType(), detectResultCompareDetailVO.getFiledName());
List<String> rulesDetail2 = pgProjectsService.getRulesDetail(projectName2, detectResultCompareDetailVO.getType(), detectResultCompareDetailVO.getFiledName());
List<String> rulesDetail = pgProjectsService.getRulesDetailV2(detectResultCompareDetailVO.getProjectId(),detectResultCompareDetailVO.getType(),
projectDetectionTaskInfo.getTaskId(),detectResultCompareDetailVO.getFiledName());
List<String> rulesDetail2 = pgProjectsService.getRulesDetailV2(detectResultCompareDetailVO.getProjectId(),detectResultCompareDetailVO.getType(),
projectDetectionTaskInfo2.getTaskId(),detectResultCompareDetailVO.getFiledName());
Set<String> allDetail = new HashSet<>(rulesDetail.size() + rulesDetail2.size());
@ -335,69 +378,47 @@ public class ProjectServiceImpl implements ProjectService {
ProjectOverviewDTO projectOverviewDTO = new ProjectOverviewDTO();
// 获取最近2次成功检测记录
// 获取最近1次成功检测记录
List<ProjectDetectionTaskInfo> projectDetectionTaskInfos = projectDetectionTaskInfoMapper.selectByProjectId(projectId, TaskInfoStatusEnum.SUCCESS.getStatus());
// 如果有2次则最近的一次和倒数第二次进行比较
if (CollectionUtils.isEmpty(projectDetectionTaskInfos)) {
// 设置默认值
setDefaultProjectOverview(projectOverviewDTO);
} else if (projectDetectionTaskInfos.size() == 1) {
ProjectDetectionTaskInfo projectDetectionTaskInfo = projectDetectionTaskInfos.get(0);
String projectName = String.format("%s-%s", projectDetectionTaskInfo.getProjectName(), projectDetectionTaskInfo.getRandomStr());
// 缺陷&漏洞
BugResultDTO bug = pgProjectsService.getBug(projectName);
VulnerabilityResultDTO vulnerability = pgProjectsService.getNewComponentVulnerability(projectId);
projectOverviewDTO.setBugDetail(bug);
projectOverviewDTO.setVulnerabilityDetail(vulnerability);
// 获取缺陷漏洞
ProjectMetricsDTO projectMetrics = pgProjectsService.getProjectMetrics(projectName);
projectOverviewDTO.setBug(new ProjectOverviewDTO.Metres(projectMetrics.getBugNumber(), 0));
projectOverviewDTO.setVulnerability(new ProjectOverviewDTO.Metres(vulnerability.getTotal().intValue(), 0));
projectOverviewDTO.setComponent(new ProjectOverviewDTO.Metres());
projectOverviewDTO.setLicense(new ProjectOverviewDTO.Metres());
// 获取代码行数文件数容量
ProjectOverviewDTO.DetectContent detectContent = new ProjectOverviewDTO.DetectContent();
detectContent.setCodeNumber(projectMetrics.getCodeNumber());
detectContent.setSize(projects.getTargetFileSize());
detectContent.setFileNumber(projects.getTargetFileNum());
projectOverviewDTO.setDetectContent(detectContent);
} else {
ProjectDetectionTaskInfo projectDetectionTaskInfo = projectDetectionTaskInfos.get(0);
ProjectDetectionTaskInfo projectDetectionTaskInfo2 = projectDetectionTaskInfos.get(1);
String projectName = String.format("%s-%s", projectDetectionTaskInfo.getProjectName(), projectDetectionTaskInfo.getRandomStr());
String projectName2 = String.format("%s-%s", projectDetectionTaskInfo2.getProjectName(), projectDetectionTaskInfo2.getRandomStr());
// 缺陷&漏洞
BugResultDTO bug = pgProjectsService.getBug(projectName);
VulnerabilityResultDTO vulnerability = pgProjectsService.getNewComponentVulnerability(projectId);
VulnerabilityResultDTO vulnerability2 = pgProjectsService.getOldComponentVulnerability(projectId);
SastBugResultDTO bug = pgProjectsService.getSastBug(projectDetectionTaskInfo.getTaskId());
VulnerabilityResultDTO vulnerability = pgProjectsService.getComponentVulnerabilityByTaskId(projectDetectionTaskInfo.getTaskId());
projectOverviewDTO.setBugDetail(bug);
projectOverviewDTO.setVulnerabilityDetail(vulnerability);
// 获取缺陷漏洞
ProjectMetricsDTO projectMetrics = pgProjectsService.getProjectMetrics(projectName);
ProjectMetricsDTO projectMetrics2 = pgProjectsService.getProjectMetrics(projectName2);
ProjectMetricsDTO projectMetrics = pgProjectsService.getProjectMetrics(projectDetectionTaskInfo.getTaskId(),projectId);
projectOverviewDTO.setBug(new ProjectOverviewDTO.Metres(projectMetrics2.getBugNumber(),
projectMetrics.getBugNumber() - projectMetrics2.getBugNumber()));
projectOverviewDTO.setBug(new ProjectOverviewDTO.Metres(projectMetrics.getBugNumber(),
projectMetrics.getNewBugNumber()));
projectOverviewDTO.setVulnerability(new ProjectOverviewDTO.Metres(vulnerability.getTotal().intValue(),
vulnerability.getTotal().intValue() - vulnerability2.getTotal().intValue()));
vulnerability.getNewTotal().intValue()));
projectOverviewDTO.setComponent(new ProjectOverviewDTO.Metres());
projectOverviewDTO.setLicense(new ProjectOverviewDTO.Metres());
// 获取代码行数文件数容量
ProjectOverviewDTO.DetectContent detectContent = new ProjectOverviewDTO.DetectContent();
detectContent.setCodeNumber(projectMetrics.getCodeNumber());
detectContent.setSize(projects.getTargetFileSize());
try{
String response = HttpUtil.get(openIDomain + String.format(QualityConstants.GET_LANGUAGE_API, projects.getCreator(), projects.getRepository()));
JSONObject result = JSONObject.parseObject(response);
int repoSize = result.getIntValue("repo_size");
detectContent.setSize(repoSize);
}catch (Exception e){
log.error("get repo size error", e);
detectContent.setSize(0);
}
detectContent.setFileNumber(projects.getTargetFileNum());
projectOverviewDTO.setDetectContent(detectContent);
@ -414,9 +435,12 @@ public class ProjectServiceImpl implements ProjectService {
throw new BusinessException(ErrorCodeEnum.PROJECT_NOT_SUCCESS);
}
String projectName = String.format("%s-%s", projectDetectionTaskInfo.getProjectName(), projectDetectionTaskInfo.getRandomStr());
Projects projects = projectsMapper.selectByPrimaryKey(projectId);
if(projects == null){
throw new BusinessException(ErrorCodeEnum.PROJECT_NOT_EXISTS);
}
ProjectBugDTO projectBugInfo = pgProjectsService.getProjectBugInfo(projectName);
ProjectBugDTO projectBugInfo = pgProjectsService.getProjectBugInfo(projects,projectDetectionTaskInfo.getTaskId());
return projectBugInfo;
}
@ -454,15 +478,17 @@ public class ProjectServiceImpl implements ProjectService {
detectResult.setFileNumber(projects.getTargetFileNum());
String projectName = String.format("%s-%s", projectDetectionTaskInfo.getProjectName(), projectDetectionTaskInfo.getRandomStr());
BugResultDTO bug = pgProjectsService.getBug(projectName);
PgProjectMeasures projectMeasures = pgProjectsService.getProjectMeasureByMetricId(projectName, 1);
if (projectMeasures != null) {
detectResult.setCodeNumber(projectMeasures.getValue().intValue());
} else {
detectResult.setCodeNumber(0);
}
// String projectName = String.format("%s-%s", projectDetectionTaskInfo.getProjectName(), projectDetectionTaskInfo.getRandomStr());
SastBugResultDTO bug = pgProjectsService.getSastBug(projectDetectionTaskInfo.getTaskId());
//
// PgProjectMeasures projectMeasures = pgProjectsService.getProjectMeasureByMetricId(projectName, 1);
// if (projectMeasures != null) {
// detectResult.setCodeNumber(projectMeasures.getValue().intValue());
// } else {
// detectResult.setCodeNumber(0);
// }
ProjectBugDTO projectBugDTO = projectBug(projectId);
detectResult.setCodeNumber(projectBugDTO.getCodeNumber());
detectResult.setBugTotalNumber(bug.getTotal());
BeanUtils.copyProperties(bug, detectResult);
@ -524,6 +550,34 @@ public class ProjectServiceImpl implements ProjectService {
return pageInfo;
}
@Override
public PageInfo<ProjectSastBugCenterDTO> projectSastBugCenter(Long projectId, ProjectBugCenterVO projectBugCenterVO) {
ProjectDetectionTaskInfo projectDetectionTaskInfo = projectDetectionTaskInfoMapper.selectLastSuccessByProjectId(projectId);
PageHelper.startPage(projectBugCenterVO.getPageNum(), projectBugCenterVO.getPageSize());
SastAnalysisDetail find = new SastAnalysisDetail();
find.setTaskId(projectDetectionTaskInfo.getTaskId());
List<SastAnalysisDetail> sastBugList = sastMapper.select(find);
PageInfo<SastAnalysisDetail> pageInfo = new PageInfo<>(sastBugList);
List<ProjectSastBugCenterDTO> resList = new ArrayList<>();
for (SastAnalysisDetail detail:sastBugList){
ProjectSastBugCenterDTO projectBugCenterDTO = new ProjectSastBugCenterDTO();
projectBugCenterDTO.setBugName(detail.getBugName());
projectBugCenterDTO.setBugDescription(detail.getDescription());
projectBugCenterDTO.setBugLevel(getSastBugDescriptionByValue(detail.getBugLevel()));
projectBugCenterDTO.setRowNumber(detail.getLine().toString());
projectBugCenterDTO.setDetectTime(detail.getCreateTime());
projectBugCenterDTO.setFilePath(detail.getFilePath());
projectBugCenterDTO.setRuleId(detail.getId().intValue());
projectBugCenterDTO.setIsIncrement(detail.getIsIncrement());
resList.add(projectBugCenterDTO);
}
PageInfo<ProjectSastBugCenterDTO> result = new PageInfo<>();
BeanUtil.copyProperties(pageInfo, result);
result.setList(resList);
return result;
}
@Override
public List<ProjectBugCenterDTO> projectBugList(Long projectId, ProjectBugCenterVO projectBugCenterVO) {
@ -573,6 +627,31 @@ public class ProjectServiceImpl implements ProjectService {
return projectBugCenterDTOList;
}
@Override
public List<ProjectSastBugCenterDTO> projectSastBugList(Long projectId, ProjectBugCenterVO projectBugCenterVO) {
ProjectDetectionTaskInfo projectDetectionTaskInfo = projectDetectionTaskInfoMapper.selectLastSuccessByProjectId(projectId);
SastAnalysisDetail find = new SastAnalysisDetail();
find.setTaskId(projectDetectionTaskInfo.getTaskId());
List<SastAnalysisDetail> sastBugList = sastMapper.select(find);
List<ProjectSastBugCenterDTO> resList = new ArrayList<>();
for (SastAnalysisDetail detail:sastBugList){
ProjectSastBugCenterDTO projectBugCenterDTO = new ProjectSastBugCenterDTO();
projectBugCenterDTO.setBugName(detail.getBugName());
projectBugCenterDTO.setBugDescription(detail.getDescription());
projectBugCenterDTO.setBugLevel(getSastBugDescriptionByValue(detail.getBugLevel()));
projectBugCenterDTO.setRowNumber(detail.getLine().toString());
projectBugCenterDTO.setDetectTime(detail.getCreateTime());
projectBugCenterDTO.setFilePath(detail.getFilePath());
projectBugCenterDTO.setRuleId(detail.getId().intValue());
projectBugCenterDTO.setIsIncrement(detail.getIsIncrement());
projectBugCenterDTO.setParentBugCatalog(detail.getParentBugCatalog());
resList.add(projectBugCenterDTO);
}
return resList;
}
@Override
public ProjectBugCenterCodeDetailDTO projectBugCenterCodeDetail(Long projectId, ProjectBugCenterCodeDetailVO centerCodeDetailVO) {
PgFileSource fileSourceByUuid = pgProjectsService.getFileSourceByUuid(centerCodeDetailVO.getUuid());
@ -618,21 +697,19 @@ public class ProjectServiceImpl implements ProjectService {
String filePath = sastAnalysisDetail.getFilePath();
File file = new File(filePath);
if(file.exists()){
try {
List<String> codeStr = FileUtils.readLines(file, "UTF-8");
ShellResult shellResult = ShellUtil.executeAndGetExitStatus("sshpass -pWk_20230306wk ssh -p40022 -o StrictHostKeyChecking=no root@118.178.181.154 " +
"'cat " + filePath + "'");
List<FileSourceDTO> codes = new ArrayList();
for (int i = 0; i < codeStr.size(); i++) {
codes.add(new FileSourceDTO(i+1, codeStr.get(i)));
}
codeDetailV2DTO.setCodes(codes);
} catch (IOException e) {
log.error("projectBugCenterCodeDetailV2 读文件异常", e);
codeDetailV2DTO.setCodes(new ArrayList<>());
if(shellResult.getExitStatus() == 0){
String codeStr = shellResult.getOut();
List<String> codeStrList = new ArrayList<>(Arrays.asList(codeStr.split("\n")));
List<FileSourceDTO> codes = new ArrayList<>();
for (int i = 0; i < codeStrList.size(); i++) {
codes.add(new FileSourceDTO(i+1, codeStrList.get(i)));
}
codeDetailV2DTO.setCodes(codes);
}else{
log.error("projectBugCenterCodeDetailV2 读文件异常 shellResult:{}",shellResult);
codeDetailV2DTO.setCodes(new ArrayList<>());
}
@ -926,6 +1003,8 @@ public class ProjectServiceImpl implements ProjectService {
projectOverviewDTO.setVulnerability(metres);
projectOverviewDTO.setLicense(metres);
projectOverviewDTO.setComponent(metres);
projectOverviewDTO.setBugDetail(new BugResultDTO());
projectOverviewDTO.setVulnerabilityDetail(new VulnerabilityResultDTO());
projectOverviewDTO.setDetectContent(new ProjectOverviewDTO.DetectContent());
}
@ -946,30 +1025,18 @@ public class ProjectServiceImpl implements ProjectService {
return resultDTO;
} else if (DetectResultCompareEnum.VULNERABILITY.getType().equals(detectResultCompareEnum.getType())) {
CompareResultDTO<VulnerabilityResultDTO> resultDTO = new CompareResultDTO();
// ProjectDetectionTaskInfo projectDetectionTaskInfo = projectDetectionTaskInfos.get(0);
// ProjectDetectionTaskInfo projectDetectionTaskInfo2 = projectDetectionTaskInfos.get(1);
//
// String projectName = projectDetectionTaskInfo.getProjectName() + "-" + projectDetectionTaskInfo.getRandomStr();
// String projectName2 = projectDetectionTaskInfo2.getProjectName() + "-" + projectDetectionTaskInfo2.getRandomStr();
resultDTO.setFirstDetect(pgProjectsService.getOldComponentVulnerability(projectId));
resultDTO.setSecondDetect(pgProjectsService.getNewComponentVulnerability(projectId));
return resultDTO;
} else if (DetectResultCompareEnum.BUG.getType().equals(detectResultCompareEnum.getType())) {
CompareResultDTO<BugResultDTO> resultDTO = new CompareResultDTO();
ProjectDetectionTaskInfo projectDetectionTaskInfo = projectDetectionTaskInfos.get(0);
ProjectDetectionTaskInfo projectDetectionTaskInfo2 = projectDetectionTaskInfos.get(1);
String taskId = projectDetectionTaskInfos.get(0).getTaskId();
String taskId2 = projectDetectionTaskInfos.get(1).getTaskId();
String projectName = projectDetectionTaskInfo.getProjectName() + "-" + projectDetectionTaskInfo.getRandomStr();
String projectName2 = projectDetectionTaskInfo2.getProjectName() + "-" + projectDetectionTaskInfo2.getRandomStr();
resultDTO.setFirstDetect(pgProjectsService.getBug(projectName2));
resultDTO.setSecondDetect(pgProjectsService.getBug(projectName));
resultDTO.setFirstDetect(pgProjectsService.getSastBug(taskId2));
resultDTO.setSecondDetect(pgProjectsService.getSastBug(taskId));
return resultDTO;
} else if (DetectResultCompareEnum.LICENSE.getType().equals(detectResultCompareEnum.getType())) {
CompareResultDTO<LicenseResultDTO> resultDTO = new CompareResultDTO();
@ -997,4 +1064,23 @@ public class ProjectServiceImpl implements ProjectService {
}
return gitUrl.replaceAll(router, "") + ".git";
}
private String getSastBugDescriptionByValue(String value){
switch (value) {
case "0":
return SastBugTypeEnum.DEADLY.getDescription();
case "1":
return SastBugTypeEnum.SEVERITY.getDescription();
case "2":
return SastBugTypeEnum.ORDINARY.getDescription();
case "3":
return SastBugTypeEnum.HINT.getDescription();
case "4":
return SastBugTypeEnum.FORCE.getDescription();
case "5":
return SastBugTypeEnum.PROPOSAL.getDescription();
default:
return null;
}
}
}

View File

@ -2,9 +2,16 @@ package net.educoder.quality.service.impl;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import net.educoder.quality.common.enums.IssueTypeEnum;
import net.educoder.quality.common.enums.TaskInfoStatusEnum;
import net.educoder.quality.dto.*;
import net.educoder.quality.entity.mysql.ProjectDetectionTaskInfo;
import net.educoder.quality.entity.mysql.ReportCenter;
import net.educoder.quality.mapper.mysql.ReportCenterMapper;
import net.educoder.quality.entity.mysql.SastAnalysisDetail;
import net.educoder.quality.entity.postgres.PgIssues;
import net.educoder.quality.mapper.mysql.*;
import net.educoder.quality.service.ReportCenterService;
import net.educoder.quality.service.postgres.PgProjectsService;
import net.educoder.quality.vo.ExportReportVO;
import net.educoder.quality.vo.ReportListVO;
import org.springframework.beans.BeanUtils;
@ -12,7 +19,9 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* @Author: youys
@ -26,6 +35,24 @@ public class ReportCenterServiceImpl implements ReportCenterService {
@Autowired
private ReportCenterMapper reportCenterMapper;
@Autowired
private ComponentParseRecordMapper recordMapper;
@Autowired(required = false)
private ProjectDetectionTaskInfoMapper projectDetectionTaskInfoMapper;
@Autowired
private ComponentParseRecordMapper componentParseRecordMapper;
@Autowired
private ComponentParseRecordDetailMapper recordDetailMapper;
@Autowired
private SastAnalysisDetailMapper sastMapper;
@Autowired
private PgProjectsService pgProjectsService;
@Override
@Transactional(rollbackFor = Exception.class)
public Long exportReport(ExportReportVO exportReportVO) {
@ -54,4 +81,36 @@ public class ReportCenterServiceImpl implements ReportCenterService {
public ReportCenter reportCenterDetail(Long reportId) {
return reportCenterMapper.selectByPrimaryKey(reportId);
}
@Override
public ReportDetailDTO getReportDetail(Long projectId){
ReportDetailDTO reportDetailDTO = new ReportDetailDTO();
Long componentId = recordMapper.getByProjectId(projectId).getId();
List<ComponentParseRecordDetailDTO> parseRecordDetails = recordDetailMapper.queryByComponentId(componentId);
if (parseRecordDetails == null){
parseRecordDetails = new ArrayList<>();
}
ProjectDetectionTaskInfo projectDetectionTaskInfo = projectDetectionTaskInfoMapper.selectLastSuccessByProjectId(projectId);
//获取缺陷和漏洞指标
VulnerabilityResultDTO vulnerability = pgProjectsService.getComponentVulnerabilityByTaskId(projectDetectionTaskInfo.getTaskId());
SastBugResultDTO bugCount = pgProjectsService.getSastBug(projectDetectionTaskInfo.getTaskId());
reportDetailDTO.setVulnerabilityCount(vulnerability);
reportDetailDTO.setBugCount(bugCount);
//依赖组件数
reportDetailDTO.setComponents(parseRecordDetails.size());
//获取bug详情
SastAnalysisDetail record = new SastAnalysisDetail();
record.setProjectId(projectId);
record.setTaskId(projectDetectionTaskInfo.getTaskId());
List<SastAnalysisDetail> bugList = sastMapper.select(record);
//表格数据
reportDetailDTO.setBugIssues(bugList);
reportDetailDTO.setParseRecordDetails(parseRecordDetails);
return reportDetailDTO;
}
}

View File

@ -1,6 +1,7 @@
package net.educoder.quality.service.impl;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.RandomUtil;
@ -15,11 +16,13 @@ import net.educoder.quality.common.exception.BusinessException;
import net.educoder.quality.common.util.GitUtil;
import net.educoder.quality.common.util.ShellUtil;
import net.educoder.quality.dto.SastBugDTO;
import net.educoder.quality.entity.mysql.ProjectDetectionTaskInfo;
import net.educoder.quality.entity.mysql.Projects;
import net.educoder.quality.entity.mysql.SastAnalysisDetail;
import net.educoder.quality.entity.sast.BizKnowledgeBug;
import net.educoder.quality.entity.mysql.BizKnowledgeBug;
import net.educoder.quality.mapper.mysql.ProjectDetectionTaskInfoMapper;
import net.educoder.quality.mapper.mysql.SastAnalysisDetailMapper;
import net.educoder.quality.mapper.sast.BizKnowledgeBugMapper;
import net.educoder.quality.mapper.mysql.BizKnowledgeBugMapper;
import net.educoder.quality.service.ProjectService;
import net.educoder.quality.service.SastService;
import org.apache.commons.lang3.StringUtils;
@ -37,7 +40,7 @@ import java.util.stream.Collectors;
@Service
@Slf4j
public class SastServiceImpl implements SastService {
private Pattern p = Pattern.compile("[\\s\\S]*(\\{[\\s\\S]*Result.[\\s\\S]*\\})[\\s\\S]*");
private final Pattern p = Pattern.compile("[\\s\\S]*(\\{[\\s\\S]*\"Result.[\\s\\S]*\\})[\\s\\S]*");
@Autowired
private PropertiesConfig propertiesConfig;
@ -51,8 +54,11 @@ public class SastServiceImpl implements SastService {
@Autowired
private SastAnalysisDetailMapper sastAnalysisDetailMapper;
@Autowired
private ProjectDetectionTaskInfoMapper taskInfoMapper;
@Override
public void analysis(Long projectId, String taskId) {
public void analysis(Long projectId, String taskId,Integer type) {
Projects projects = projectService.getProjectById(projectId);
if (projects == null) {
throw new BusinessException(ErrorCodeEnum.PROJECT_NOT_EXISTS);
@ -62,6 +68,7 @@ public class SastServiceImpl implements SastService {
ShellResult gitCloneResult = GitUtil.gitClone(projects.getGitUrl(), projects.getBranch(), propertiesConfig.getGitUsername(), propertiesConfig.getGitPassword(), fullPath);
if (gitCloneResult.getExitStatus() != 0) {
log.info("projectId:{}克隆代码失败,终止执行", projectId);
projectService.updateTaskAndProjectStatus(taskId, -1, null,null);
return;
}
@ -70,28 +77,59 @@ public class SastServiceImpl implements SastService {
paramMap.put("driver", propertiesConfig.getSastDriver());
paramMap.put("outputDir", "/tmp/" + RandomUtil.randomString(10));
paramMap.put("targetDir", fullPath);
List<SastBugDTO> sastResult = execute(paramMap);
log.info("SAST结果: {}, projectId: {}, taskId: {}", sastResult, projectId, taskId);
List<String> bugTypes = sastResult.stream().map(SastBugDTO::getBugType).collect(Collectors.toList());
try {
List<SastBugDTO> sastResult = execute(paramMap);
log.info("SAST结果: {}, projectId: {}, taskId: {}", sastResult, projectId, taskId);
List<String> bugTypes = sastResult.stream().map(SastBugDTO::getBugType).collect(Collectors.toList());
// 取结果封装详情入库
List<BizKnowledgeBug> bizKnowledgeBugs = bizKnowledgeBugMapper.selectBugAndCatalogByTypes(bugTypes);
List<SastAnalysisDetail> sastAnalysisDetails = new ArrayList<>();
for (SastBugDTO sastBugDTO : sastResult) {
SastAnalysisDetail sastAnalysisDetail = new SastAnalysisDetail();
BeanUtils.copyProperties(sastBugDTO, sastAnalysisDetail);
if (CollectionUtil.isNotEmpty(bugTypes)) {
// 取结果封装详情入库
List<BizKnowledgeBug> bizKnowledgeBugs = bizKnowledgeBugMapper.selectBugAndCatalogByTypes(bugTypes);
List<SastAnalysisDetail> sastAnalysisDetails = new ArrayList<>();
List<SastAnalysisDetail> lastDetails = getLastSastByProjectId(projectId);
for (SastBugDTO sastBugDTO : sastResult) {
try {
SastAnalysisDetail sastAnalysisDetail = new SastAnalysisDetail();
BeanUtils.copyProperties(sastBugDTO, sastAnalysisDetail);
BizKnowledgeBug bizKnowledgeBug = bizKnowledgeBugs.stream().filter(bug -> bug.getBugType().equals(sastBugDTO.getBugType())).findFirst().get();
BeanUtils.copyProperties(bizKnowledgeBug, sastAnalysisDetail);
sastAnalysisDetail.setProjectId(projectId);
sastAnalysisDetail.setTaskId(taskId);
sastAnalysisDetail.setFilePath(fullPath + sastBugDTO.getFilePath());
sastAnalysisDetail.setCreateTime(DateTime.now());
sastAnalysisDetail.setUpdateTime(DateTime.now());
sastAnalysisDetails.add(sastAnalysisDetail);
BizKnowledgeBug bizKnowledgeBug = bizKnowledgeBugs.stream().filter(bug -> bug.getBugType().equals(sastBugDTO.getBugType())).findFirst().get();
BeanUtils.copyProperties(bizKnowledgeBug, sastAnalysisDetail);
sastAnalysisDetail.setProjectId(projectId);
sastAnalysisDetail.setTaskId(taskId);
if (!sastBugDTO.getFilePath().startsWith(fullPath)) {
sastAnalysisDetail.setFilePath(fullPath + sastBugDTO.getFilePath());
}
sastAnalysisDetail.setCreateTime(DateTime.now());
sastAnalysisDetail.setUpdateTime(DateTime.now());
sastAnalysisDetail.setIsIncrement(0);
if ("GJB-5369-2005".equals(sastAnalysisDetail.getParentBugCatalog())) {
sastAnalysisDetail.setParentBugCatalog("Cert C/ISO 17961");
}
if (type == null || type == 0) {
sastAnalysisDetail.setIsIncrement(1);
for (SastAnalysisDetail lastDetail : lastDetails) {
//与上次结果比较不同为增量
if (lastDetail.equals(sastAnalysisDetail)) {
sastAnalysisDetail.setIsIncrement(0);
break;
}
}
}
sastAnalysisDetails.add(sastAnalysisDetail);
} catch (Exception e) {
log.error("单条分析结果错误, e: {}", e);
}
}
sastAnalysisDetailMapper.insertList(sastAnalysisDetails);
}
log.info("SAST结果入库完成, projectId: {}, taskId: {}", projectId, taskId);
// 更新状态
projectService.updateTaskAndProjectStatus(taskId, 1, getDirectoryFilesNum(fullPath),getDirectoryFileSize(fullPath));
}catch (Exception e){
log.error("analysis is exception", e);
// 更新状态
projectService.updateTaskAndProjectStatus(taskId, -1, null,null);
}
sastAnalysisDetailMapper.insertList(sastAnalysisDetails);
log.info("SAST结果入库完成, projectId: {}, taskId: {}", sastResult, projectId, taskId);
}
public List<SastBugDTO> execute(Map<String, String> paramMap) {
@ -149,11 +187,12 @@ public class SastServiceImpl implements SastService {
sb.append(targetDir);
// TODO 临时
sb.append("'");
ShellUtil.execute("sshpass -pWk_20230306wk ssh -p40022 -o StrictHostKeyChecking=no root@118.178.181.154 'mkdir -p " + targetDir + "'");
ShellUtil.execute("sshpass -pWk_20230306wk scp -r -P40022 -o StrictHostKeyChecking=no " + targetDir + " root@118.178.181.154:" + targetDir);
ShellUtil.execute(sb.toString());
String analysisResult = ShellUtil.execute("sshpass -pWk_20230306wk ssh -p40022 -o StrictHostKeyChecking=no root@118.178.181.154 " +
"'cat " + outputDir + "/mergedJson'");
log.info("SAST扫描命令: {}, 结果: {}", sb.toString(), analysisResult);
log.info("SAST扫描命令: {},resultPath: {}, 结果: {}", sb.toString(),outputDir+"/mergedJson", analysisResult);
Matcher matcher = p.matcher(analysisResult);
if (matcher.find()) {
@ -180,5 +219,57 @@ public class SastServiceImpl implements SastService {
public SastAnalysisDetail findById(Long id) {
return sastAnalysisDetailMapper.selectByPrimaryKey(id);
}
/**
* 获取目录下的文件数量
*
* @param path
* @return
*/
private int getDirectoryFilesNum(String path) {
String command = StringUtils.join("cd ", path, " && ls -lR| grep \"^-\" | wc -l");
ShellResult shellResult = ShellUtil.executeAndGetExitStatus(command);
log.info("获取目录文件数返回command:{},result:{} ", command, JSONObject.toJSONString(shellResult));
try {
return Integer.parseInt(shellResult.getOut());
} catch (NumberFormatException e) {
return 0;
}
}
/**
* 获取目录下所有文件的大小
*
* @param path
* @return
*/
private String getDirectoryFileSize(String path) {
String command = StringUtils.join("cd ", path, "&& cd `ls` && du -sh . | awk -F ' ' '{print $1}'");
ShellResult shellResult = ShellUtil.executeAndGetExitStatus(command);
log.info("获取目录文件大小返回command:{},result:{} ", command, JSONObject.toJSONString(shellResult));
try {
if (shellResult.getExitStatus() == 0) {
return shellResult.getOut();
} else {
return "0k";
}
} catch (NumberFormatException e) {
return "0k";
}
}
private List<SastAnalysisDetail> getLastSastByProjectId(Long projectId){
List<ProjectDetectionTaskInfo> projectDetectionTaskInfos = taskInfoMapper.selectByOnlyProjectId(projectId);
if (projectDetectionTaskInfos.size()<=1){
return new ArrayList<>();
}
String lastTaskId = projectDetectionTaskInfos.get(1).getTaskId();
SastAnalysisDetail record = new SastAnalysisDetail();
record.setTaskId(lastTaskId);
return sastAnalysisDetailMapper.select(record);
}
}

View File

@ -33,11 +33,13 @@ public class VulnerabilityServiceImpl implements VulnerabilityService {
@Override
public PageInfo<ProjectVulnerabilityDetailDTO> vulnerabilityList(Long projectId, PageVO pageVO) {
Long componentId = componentParseRecordMapper.getByProjectId(projectId).getId();
PageHelper.startPage(pageVO.getPageNum(), pageVO.getPageSize());
ProjectVulnerabilityDetail projectVulnerabilityDetail = new ProjectVulnerabilityDetail();
projectVulnerabilityDetail.setProjectId(projectId);
projectVulnerabilityDetail.setComponentId(componentId);
List<ProjectVulnerabilityDetail> details = projectVulnerabilityDetailMapper.select(projectVulnerabilityDetail);
PageInfo<ProjectVulnerabilityDetail> doPageInfo = new PageInfo<>(details);
@ -65,6 +67,7 @@ public class VulnerabilityServiceImpl implements VulnerabilityService {
dto.setRepairSuggestions(detail.getRepairSuggestions());
dto.setDescription(detail.getDescription());
dto.setUuid(String.valueOf(detail.getId()));
dto.setIsIncrement(detail.getIsIncrement());
return dto;
}
@ -95,61 +98,39 @@ public class VulnerabilityServiceImpl implements VulnerabilityService {
switch (detail.getRiskLevel()) {
case 0:
result.setUnKnow(result.getUnKnow() + 1);
if (detail.getIsIncrement()==1){
result.setNewUnKnow(result.getNewUnKnow() + 1);
}
break;
case 1:
result.setLow(result.getLow() + 1);
if (detail.getIsIncrement()==1){
result.setNewLow(result.getNewLow() + 1);
}
break;
case 2:
result.setMiddle(result.getMiddle() + 1);
if (detail.getIsIncrement()==1){
result.setNewMiddle(result.getNewMiddle() + 1);
}
break;
case 3:
result.setHigh(result.getHigh() + 1);
if (detail.getIsIncrement()==1){
result.setNewHigh(result.getNewHigh() + 1);
}
break;
case 4:
result.setCritical(result.getCritical() + 1);
if (detail.getIsIncrement()==1){
result.setNewCritical(result.getNewCritical() + 1);
}
break;
default:
break;
}
}
//根据上次检测结果过滤出最近新增的Vulnerability数量
ComponentParseRecord componentParseRecord = null;
if (recordList.size() >= 2){
componentParseRecord = recordList.get(1);
}
if (componentParseRecord!=null){
Long componentId = componentParseRecord.getId();
projectVulnerabilityDetail.setComponentId(componentId);
List<ProjectVulnerabilityDetail> collects = projectVulnerabilityDetailMapper.select(projectVulnerabilityDetail);
for (ProjectVulnerabilityDetail detail : collects) {
switch (detail.getRiskLevel()) {
case 0:
result.setNewUnKnow(result.getNewUnKnow() + 1);
break;
case 1:
result.setNewLow(result.getNewLow() + 1);
break;
case 2:
result.setNewMiddle(result.getNewMiddle() + 1);
break;
case 3:
result.setNewHigh(result.getNewHigh() + 1);
break;
case 4:
result.setNewCritical(result.getNewCritical() + 1);
break;
default:
break;
}
}
result.setNewUnKnow(result.getUnKnow() - result.getNewUnKnow());
result.setNewLow(result.getLow() - result.getNewLow());
result.setNewMiddle(result.getMiddle() - result.getNewMiddle());
result.setNewHigh(result.getHigh() - result.getNewHigh());
result.setNewCritical(result.getCritical() - result.getNewCritical());
}
return result;
}
}

View File

@ -1,5 +1,6 @@
package net.educoder.quality.service.postgres;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.github.pagehelper.PageHelper;
@ -7,18 +8,14 @@ import com.github.pagehelper.PageInfo;
import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
import net.educoder.quality.common.constant.PgIssueConstant;
import net.educoder.quality.common.constant.QualityConstants;
import net.educoder.quality.common.enums.*;
import net.educoder.quality.common.exception.BusinessException;
import net.educoder.quality.common.util.WordUtil;
import net.educoder.quality.dto.*;
import net.educoder.quality.entity.mysql.ComponentParseRecord;
import net.educoder.quality.entity.mysql.ComponentParseRecordDetail;
import net.educoder.quality.entity.mysql.ProjectVulnerabilityDetail;
import net.educoder.quality.entity.mysql.*;
import net.educoder.quality.entity.postgres.*;
import net.educoder.quality.mapper.mysql.ComponentParseRecordDetailMapper;
import net.educoder.quality.mapper.mysql.ComponentParseRecordMapper;
import net.educoder.quality.mapper.mysql.ProjectVulnerabilityDetailMapper;
import net.educoder.quality.mapper.mysql.ProjectsMapper;
import net.educoder.quality.mapper.mysql.*;
import net.educoder.quality.mapper.postgres.PgCeActivityMapper;
import net.educoder.quality.mapper.postgres.PgProjectMapper;
import net.educoder.quality.protobuf.DbIssues;
@ -28,9 +25,12 @@ import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFTable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -50,21 +50,18 @@ public class PgProjectsService {
@Autowired
private PgCeActivityMapper ceActivityMapper;
@Autowired
private ComponentParseRecordMapper recordMapper;
@Autowired
private ComponentParseRecordDetailMapper recordDetailMapper;
@Autowired
private ProjectsMapper mysqlProjectMapper;
@Resource
private ProjectVulnerabilityDetailMapper projectVulnerabilityDetailMapper;
@Autowired
private ComponentParseRecordMapper componentParseRecordMapper;
@Autowired
private SastAnalysisDetailMapper sastMapper;
@Value("${openi.domain}")
private String openIDomain;
public PgProjects getProjectsByProjectName(String projectName) {
PgProjects project = projectMapper.findByName(projectName);
if (project == null) {
@ -81,39 +78,8 @@ public class PgProjectsService {
return ceActivityMapper.queryActivityStatus(project.getProjectUuid());
}
public ReportDetailDTO getReportDetail(String projectName){
ReportDetailDTO reportDetailDTO = new ReportDetailDTO();
PgProjects projects = getProjectsByProjectName(projectName);
String projectUuid = projects.getProjectUuid();
Long componentId = recordMapper.getByProjectId(mysqlProjectMapper.getByName(projectName).getId()).getId();
List<ComponentParseRecordDetailDTO> parseRecordDetails = recordDetailMapper.queryByComponentId(componentId);
if (parseRecordDetails == null){
parseRecordDetails = new ArrayList<>();
}
List<PgIssues> issuesByProjectName = projectMapper.findPageIssues(projectUuid,null,null);
//获取缺陷和漏洞指标
CompareBaseResultDTO vulnerabilityCount = getCountByIssueType(issuesByProjectName,IssueTypeEnum.VULNERABILITY.getValue());
CompareBaseResultDTO bugCount = getCountByIssueType(issuesByProjectName,IssueTypeEnum.BUG.getValue());
reportDetailDTO.setVulnerabilityCount(vulnerabilityCount);
reportDetailDTO.setBugCount(bugCount);
//依赖组件数
reportDetailDTO.setComponents(parseRecordDetails.size());
//获取详情
List<PgIssues> bugIssues = issuesByProjectName.stream()
.filter(issue -> IssueTypeEnum.BUG.getValue().equals(issue.getIssueType())).collect(Collectors.toList());
//表格数据
reportDetailDTO.setBugIssues(bugIssues);
reportDetailDTO.setParseRecordDetails(parseRecordDetails);
return reportDetailDTO;
}
public void fillReportTable(XWPFDocument document, ReportDetailDTO reportDetailDTO){
List<PgIssues> bugIssues = reportDetailDTO.getBugIssues();
List<SastAnalysisDetail> SastBugList = reportDetailDTO.getBugIssues();
List<ComponentParseRecordDetailDTO> parseRecordDetails = reportDetailDTO.getParseRecordDetails();
//缺陷
@ -124,8 +90,8 @@ public class PgProjectsService {
bugTable.removeRow(tempLine);
}
//循环添加表格行并填充数据
for (int kk = 1;kk <= bugIssues.size();kk++){
PgIssues pgIssues = bugIssues.get(kk-1);
for (int kk = 1;kk <= SastBugList.size();kk++){
SastAnalysisDetail sastBug = SastBugList.get(kk-1);
for (int i = 0;i < tempLine;i++){
//先在模板表格下复制模板的行
WordUtil.insertRow(bugTable,bugTable.getRow(i),bugTable.getRows().size());
@ -134,18 +100,11 @@ public class PgProjectsService {
List<XWPFParagraph> tableParagraphs = WordUtil.getTableRowsParagraphs(bugTable,kk*tempLine,kk*tempLine+3);
Map<String, String> tableTextMap = new HashMap<>(6);
tableTextMap.put("${issueNo}",kk+"");
tableTextMap.put("${issueLevel}",BugTypeEnum.getDescriptionByDbValue(pgIssues.getSeverity()));
tableTextMap.put("${issueName}",pgIssues.getName());
tableTextMap.put("${issueFilePath}",pgIssues.getPath());
String rows = "0";
try {
DbIssues.Locations locations = DbIssues.Locations.parseFrom(pgIssues.getLocations());
rows = String.valueOf(locations.getTextRange().getStartLine());
} catch (InvalidProtocolBufferException e) {
log.error("Fail to read ISSUES.LOCATIONS [KEE=%s]", e);
}
tableTextMap.put("${issueLine}",rows);
tableTextMap.put("${issueDescription}",pgIssues.getMessage());
tableTextMap.put("${issueLevel}",sastBug.getBugCatalog());
tableTextMap.put("${issueName}",sastBug.getBugName());
tableTextMap.put("${issueFilePath}",sastBug.getFilePath());
tableTextMap.put("${issueLine}",sastBug.getLine().toString());
tableTextMap.put("${issueDescription}",sastBug.getDescription().replace("<p>","").replace("</p>",""));
//填充数据
WordUtil.replaceAllTexts(tableTextMap,tableParagraphs);
}
@ -177,7 +136,7 @@ public class PgProjectsService {
for (int i = 0;i<jsonArray.size();i++){
JSONObject jsonObject = jsonArray.getJSONObject(i);
String description = jsonObject.getString("description");
vulnerabilityDes.append(i + 1).append(".漏洞").append(i + 1).append(":").append(description).append('\n');
vulnerabilityDes.append(i + 1).append(".漏洞").append(i + 1).append(":").append(description).append("\n");
}
}else {
vulnerabilityDes.append("");
@ -268,6 +227,47 @@ public class PgProjectsService {
List<ProjectVulnerabilityDetail> details = projectVulnerabilityDetailMapper.select(projectVulnerabilityDetail);
result.setTotal((long)details.size());
for (ProjectVulnerabilityDetail detail : details) {
if (detail.getIsIncrement()==1){
result.setNewTotal(result.getNewTotal() + 1);
}
switch (detail.getRiskLevel()) {
case 1:
result.setLow(result.getLow() + 1);
break;
case 2:
result.setMiddle(result.getMiddle() + 1);
break;
case 3:
result.setHigh(result.getHigh() + 1);
break;
case 4:
result.setCritical(result.getCritical() + 1);
break;
default:
break;
}
}
return result;
}
/**
* 某次检测对应的组件漏洞指标
* @param taskId
* @return
*/
public VulnerabilityResultDTO getComponentVulnerabilityByTaskId(String taskId) {
VulnerabilityResultDTO result = new VulnerabilityResultDTO();
ComponentParseRecord newRecord = componentParseRecordMapper.getByTaskId(taskId);
ProjectVulnerabilityDetail projectVulnerabilityDetail = new ProjectVulnerabilityDetail();
projectVulnerabilityDetail.setComponentId(newRecord.getId());
List<ProjectVulnerabilityDetail> details = projectVulnerabilityDetailMapper.select(projectVulnerabilityDetail);
result.setTotal((long)details.size());
for (ProjectVulnerabilityDetail detail : details) {
if (detail.getIsIncrement()==1){
result.setNewTotal(result.getNewTotal() + 1);
}
switch (detail.getRiskLevel()) {
case 1:
result.setLow(result.getLow() + 1);
@ -367,6 +367,44 @@ public class PgProjectsService {
return resultDTO;
}
public SastBugResultDTO getSastBug(String taskId) {
SastAnalysisDetail find = new SastAnalysisDetail();
find.setTaskId(taskId);
List<SastAnalysisDetail> sastBugList = sastMapper.select(find);
SastBugResultDTO result = new SastBugResultDTO();
result.setTotal((long)sastBugList.size());
for (SastAnalysisDetail detail : sastBugList) {
switch (detail.getBugLevel()) {
case "0":
result.setDeadly(result.getDeadly() + 1);
result.setCritical(result.getCritical() + 1);
break;
case "1":
result.setSeverity(result.getSeverity() + 1);
result.setHigh(result.getHigh() + 1);
break;
case "2":
result.setOrdinary(result.getOrdinary() + 1);
result.setMiddle(result.getMiddle() + 1);
break;
case "3":
result.setHint(result.getHint() + 1);
result.setLow(result.getLow() + 1);
break;
case "4":
result.setForce(result.getForce() + 1);
break;
case "5":
result.setProposal(result.getProposal() + 1);
break;
default:
break;
}
}
return result;
}
/**
* 获取规则(漏洞)详情
@ -376,6 +414,7 @@ public class PgProjectsService {
* @param filedName
* @return
*/
@Deprecated
public List<String> getRulesDetail(String projectName, Integer type, String filedName) {
List<PgIssues> issuesByProjectName = projectMapper.findIssuesByProjectName(projectName);
@ -422,55 +461,126 @@ public class PgProjectsService {
/**
* 获取项目指标
*
* @param projectName
* @return
* 获取规则(漏洞)详情
*/
public ProjectMetricsDTO getProjectMetrics(String projectName) {
Integer bugNumber = projectMapper.findIssueCountByProjectNameAndIssueType(projectName, IssueTypeEnum.BUG.getValue());
Integer vulnerabilityNumber = projectMapper.findIssueCountByProjectNameAndIssueType(projectName, IssueTypeEnum.VULNERABILITY.getValue());
public List<String> getRulesDetailV2(Long projectId, Integer type, String taskId, String filedName) {
List<String> details = new ArrayList<>();
if (DetectResultCompareEnum.VULNERABILITY.getType().equals(type)) {
Integer level = getVulnerabilityLevelByFiledName(filedName);
ComponentParseRecord component = componentParseRecordMapper.getByTaskId(taskId);
if (component!=null){
ProjectVulnerabilityDetail vulnerabilityDetail = new ProjectVulnerabilityDetail();
vulnerabilityDetail.setProjectId(projectId);
vulnerabilityDetail.setComponentId(component.getId());
if (level!=null){
vulnerabilityDetail.setRiskLevel(level);
}
List<ProjectVulnerabilityDetail> list = projectVulnerabilityDetailMapper.select(vulnerabilityDetail);
return list.stream().map(ProjectVulnerabilityDetail::getVulnerabilityName).collect(Collectors.toList());
}
} else if (DetectResultCompareEnum.BUG.getType().equals(type)) {
String level = getBugLevelByFiledName(filedName);
SastAnalysisDetail record = new SastAnalysisDetail();
record.setProjectId(projectId);
record.setTaskId(taskId);
if (level!=null){
record.setBugLevel(level);
}
List<SastAnalysisDetail> list = sastMapper.select(record);
return list.stream().map(SastAnalysisDetail::getBugName).collect(Collectors.toList());
}
return details;
}
Integer codeNumber = projectMapper.findByCodeNumber(projectName);
return new ProjectMetricsDTO(nullConvertZero(bugNumber), nullConvertZero(vulnerabilityNumber), nullConvertZero(codeNumber));
private Integer getVulnerabilityLevelByFiledName(String filedName){
Integer level;
switch (filedName){
case "critical":
level = 4;
break;
case "high":
level = 3;
break;
case "middle":
level = 2;
break;
case "low":
level = 1;
break;
default:
level = null;
break;
}
return level;
}
private String getBugLevelByFiledName(String filedName){
String level;
switch (filedName){
case "critical":
level = "0";
break;
case "high":
level = "1";
break;
case "middle":
level = "2";
break;
case "low":
level = "3";
break;
default:
level = null;
break;
}
return level;
}
/**
* 获取项目指标
*/
public ProjectMetricsDTO getProjectMetrics(String taskId,Long projectId) {
SastAnalysisDetail find = new SastAnalysisDetail();
find.setProjectId(projectId);
find.setTaskId(taskId);
List<SastAnalysisDetail> sastBugList = sastMapper.select(find);
Integer bugNumber = sastBugList.size();
Integer newBugNumber = (int) sastBugList.stream().filter(e -> {
return e.getIsIncrement() == 1;
}).count();
return new ProjectMetricsDTO(nullConvertZero(bugNumber),newBugNumber, 0);
}
/**
* 获取项目缺陷信息
*
* @param projectName
* @param projects
* @return
*/
public ProjectBugDTO getProjectBugInfo(String projectName) {
public ProjectBugDTO getProjectBugInfo(Projects projects,String taskId) {
ProjectBugDTO projectBugDTO = new ProjectBugDTO();
Integer codeNumber = projectMapper.findByCodeNumber(projectName);
Integer bugNumber = projectMapper.findIssueCountByProjectNameAndIssueType(projectName, IssueTypeEnum.BUG.getValue());
SastAnalysisDetail find = new SastAnalysisDetail();
find.setTaskId(taskId);
List<SastAnalysisDetail> sastBugList = sastMapper.select(find);
projectBugDTO.setCodeNumber(codeNumber);
Integer bugNumber = sastBugList.size();
projectBugDTO.setBugNumber(bugNumber);
PgProjectMeasures projectMeasures = projectMapper.findProjectMeasureByMetricId(projectName, 5);
getAndSetLanguage(projectBugDTO, projects.getCreator(), projects.getRepository());
String[] split = projectMeasures.getTextValue().split(";");
List<KeyValuePair> left = new ArrayList<>(split.length);
projectBugDTO.setLeft(left);
for (String languageAndLineNumber : split) {
String[] array = languageAndLineNumber.split("=");
left.add(new KeyValuePair(array[0], array[1]));
}
List<KeyValuePair> right = new ArrayList<>(split.length);
List<KeyValuePair> right = new ArrayList<>();
projectBugDTO.setRight(right);
BugResultDTO bug = getBug(projectName);
right.add(new KeyValuePair("严重", String.valueOf(bug.getCritical())));
right.add(new KeyValuePair("高危", String.valueOf(bug.getHigh())));
right.add(new KeyValuePair("中危", String.valueOf(bug.getMiddle())));
right.add(new KeyValuePair("低危", String.valueOf(bug.getLow())));
right.add(new KeyValuePair("未知", "0"));
SastBugResultDTO bug = getSastBug(taskId);
right.add(new KeyValuePair("致命", String.valueOf(bug.getDeadly())));
right.add(new KeyValuePair("严重", String.valueOf(bug.getSeverity())));
right.add(new KeyValuePair("一般", String.valueOf(bug.getOrdinary())));
right.add(new KeyValuePair("提示", String.valueOf(bug.getHint())));
right.add(new KeyValuePair("强制", String.valueOf(bug.getForce())));
right.add(new KeyValuePair("建议", String.valueOf(bug.getProposal())));
return projectBugDTO;
}
@ -525,4 +635,46 @@ public class PgProjectsService {
return number == null ? 0 : number.intValue();
}
private void getAndSetLanguage(ProjectBugDTO projectBugDTO, String repoOwner, String repoName){
try {
List<KeyValuePair> keyValuePairs = new ArrayList<>();
String response = HttpUtil.get(openIDomain + String.format(QualityConstants.GET_LANGUAGE_API, repoOwner, repoName));
JSONObject result = JSONObject.parseObject(response);
int repoSize = result.getIntValue("repo_size");
projectBugDTO.setCodeNumber(repoSize);
projectBugDTO.setLeft(keyValuePairs);
JSONArray langs = result.getJSONArray("langs");
if(langs != null) {
for (Object lang : langs) {
JSONObject obj = (JSONObject)lang;
BigDecimal percentage = BigDecimal.valueOf(obj.getDoubleValue("Percentage")).divide(BigDecimal.valueOf(100)).multiply(BigDecimal.valueOf(repoSize)).setScale(0,RoundingMode.HALF_UP);
keyValuePairs.add(new KeyValuePair(obj.getString("Language"), percentage.toString()));
}
}
}catch (Exception e){
log.error("getLanguage error", e);
projectBugDTO.setLeft(new ArrayList<>());
}
}
private static String setUnit(BigDecimal number){
if (number.divide(BigDecimal.valueOf(1024 * 1024 * 1024)).setScale(0, RoundingMode.HALF_UP).doubleValue() > 0) {
return number.divide(BigDecimal.valueOf(1024 * 1024 * 1024)).setScale(0, RoundingMode.HALF_UP).toString() + "G";
}else if(number.divide(BigDecimal.valueOf(1024 * 1024)).setScale(0, RoundingMode.HALF_UP).doubleValue() > 0){
return number.divide(BigDecimal.valueOf(1024 * 1024)).setScale(0, RoundingMode.HALF_UP).toString() + "M";
}else if(number.divide(BigDecimal.valueOf(1024)).setScale(0, RoundingMode.HALF_UP).doubleValue() > 0){
return number.divide(BigDecimal.valueOf(1024)).setScale(0, RoundingMode.HALF_UP).toString() + "K";
}else{
return number.setScale(0,RoundingMode.HALF_UP).toString();
}
}
public static void main(String[] args) {
System.out.println(setUnit(BigDecimal.valueOf(10000)));
}
}

View File

@ -24,6 +24,6 @@ public class ComponentParseRunnable implements Runnable {
@Override
public void run() {
log.info("开始执行组件解析,projectId:{}", projects.getId());
componentService.parseComponent(projects);
componentService.parseComponent(projects,1,null);
}
}

View File

@ -0,0 +1,12 @@
package net.educoder.quality.vo;
import lombok.Data;
import javax.validation.constraints.NotBlank;
@Data
public class DroneDetectionVO extends CommonVO{
@NotBlank(message = "branchName不能为空")
private String branchName;
}

View File

@ -2,6 +2,7 @@
openi:
gitUsername: wangwei
gitPassword: zq123456
domain: http://118.31.13.117:64300
# sonar相关配置
sonar:
# serverUrl: http://117.50.14.123:9000
@ -18,8 +19,8 @@ opensca-cli:
#path: /data/ww/open-cli/opensca-cli
path: /Users/youyongsheng/Downloads/opensca-cli_v1.0.9_Darwin_x86_64/opensca-cli
nil:
jarPath: /Users/weiwang/IdeaProjects/NIL/build/libs/NIL-all.jar
mil: 6
mit: 50
jarPath: /Users/weiwang/IdeaProjects/quality_analysis/web/src/main/resources/nil/nil.jar
mil: 1
mit: 1
filtrationThreshold: 10
verificationThreshold: 70

View File

@ -55,21 +55,6 @@ spring:
test-on-borrow: false
test-on-return: false
test-while-idle: true
# sast
sast:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://118.178.181.154:43306/zktq_wukong?useUnicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false
username: root
password: Wk_20230306wk
type: com.alibaba.druid.pool.DruidDataSource
druid:
initial-size: 20
max-active: 40
min-idle: 20
validation-query: select 1
test-on-borrow: false
test-on-return: false
test-while-idle: true
# redis配置
redis:
host: 127.0.0.1

View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="net.educoder.quality.mapper.sast.BizKnowledgeBugMapper">
<resultMap id="BaseResultMap" type="net.educoder.quality.entity.sast.BizKnowledgeBug">
<mapper namespace="net.educoder.quality.mapper.mysql.BizKnowledgeBugMapper">
<resultMap id="BaseResultMap" type="net.educoder.quality.entity.mysql.BizKnowledgeBug">
<!--
WARNING - @mbg.generated
-->
@ -26,7 +26,7 @@
<result column="EXAMPLE" jdbcType="LONGVARCHAR" property="example"/>
<result column="ERROR_EXAMPLE" jdbcType="LONGVARCHAR" property="errorExample"/>
</resultMap>
<select id="selectBugAndCatalogByTypes" resultType="net.educoder.quality.entity.sast.BizKnowledgeBug"
<select id="selectBugAndCatalogByTypes" resultType="net.educoder.quality.entity.mysql.BizKnowledgeBug"
parameterType="java.util.ArrayList">
select bug.BUG_NO as bugNo,
bug.BUG_NAME as bugName,

View File

@ -24,4 +24,16 @@
order by
create_time desc
</select>
<select id="getByTaskId" resultType="net.educoder.quality.entity.mysql.ComponentParseRecord">
select
id,project_id,result
from
component_parse_record
where
task_id=#{taskId}
order by
create_time desc
limit 1
</select>
</mapper>

View File

@ -159,6 +159,15 @@
order by create_time desc limit 2
</select>
<select id="selectByOnlyProjectId" resultMap="BaseResultMap">
select
<include refid="Base_Column_List"/>
from project_detection_task_info
where project_id=#{projectId,jdbcType=BIGINT}
and status != -1
order by create_time desc limit 2
</select>
<!-- 获取最近一次成功的记录 -->
<select id="selectLastSuccessByProjectId" resultMap="BaseResultMap">
select

View File

@ -29,5 +29,7 @@
<result column="variable" property="variable" jdbcType="VARCHAR" />
<result column="line" property="line" jdbcType="INTEGER" />
<result column="file_path" property="filePath" jdbcType="VARCHAR" />
<result column="is_increment" property="isIncrement" jdbcType="SMALLINT" />
</resultMap>
</mapper>

Binary file not shown.

View File

@ -1,17 +1,23 @@
package net.educoder.quality;
import cn.hutool.core.date.DateUtil;
import com.github.pagehelper.PageInfo;
import net.educoder.quality.common.util.WordUtil;
import net.educoder.quality.dto.ProjectSastBugCenterDTO;
import net.educoder.quality.dto.ReportDetailDTO;
import net.educoder.quality.entity.mysql.Projects;
import net.educoder.quality.entity.mysql.ReportCenter;
import net.educoder.quality.entity.mysql.SastAnalysisDetail;
import net.educoder.quality.entity.postgres.PgIssues;
import net.educoder.quality.entity.postgres.PgProjects;
import net.educoder.quality.mapper.mysql.SastAnalysisDetailMapper;
import net.educoder.quality.mapper.mysql.TestMapper;
import net.educoder.quality.mapper.postgres.PgProjectMapper;
import net.educoder.quality.service.ProjectService;
import net.educoder.quality.service.ReportCenterService;
import net.educoder.quality.service.postgres.PgProjectsService;
import net.educoder.quality.vo.DetectResultCompareDetailVO;
import net.educoder.quality.vo.ProjectBugCenterVO;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
@ -49,25 +55,38 @@ public class MapperTest {
@Autowired
private PgProjectsService pgProjectsService;
@Autowired
private SastAnalysisDetailMapper sastMapper;
@Test
@Transactional
public void testInsert(){
testMapper.insertTest("张三");
// testMapper.insertTest("李四");
int i=1/0;
ProjectBugCenterVO projectBugCenterVO = new ProjectBugCenterVO();
projectBugCenterVO.setPageSize(5);
projectBugCenterVO.setPageNum(2);
PageInfo<ProjectSastBugCenterDTO> res = projectService.projectSastBugCenter(86L, projectBugCenterVO);
System.out.println(res);
// System.out.println(pgProjectsService.getSastBug("475336319468306432"));
}
@Test
public void testFix(){
DetectResultCompareDetailVO detectResultCompareDetailVO = new DetectResultCompareDetailVO();
detectResultCompareDetailVO.setProjectId(89L);
detectResultCompareDetailVO.setFiledName("all");
detectResultCompareDetailVO.setType(1);
System.out.println(projectService.detectResultCompareDetail(detectResultCompareDetailVO));
}
@Test
public void testS() throws Exception {
Long reportId = 5L;
Long reportId = 11L;
ReportCenter reportCenter = reportCenterService.reportCenterDetail(reportId);
Projects projects = projectService.getProjectById(reportCenter.getProjectId());
// 获取模板填充数据
InputStream resourceAsStream = this.getClass().getClassLoader().getResourceAsStream("template/report-center-template.docx");
ReportDetailDTO reportDetail = pgProjectsService.getReportDetail(projects.getProjectName());
ReportDetailDTO reportDetail = reportCenterService.getReportDetail(projects.getId());
Map<String, String> param = new HashMap<>(13);
param.put("projectName", projects.getProjectName());
@ -88,6 +107,6 @@ public class MapperTest {
WordUtil.changeText(doc, param);
pgProjectsService.fillReportTable(doc,reportDetail);
WordUtil.save(doc,"C:\\Users\\14666\\Desktop\\res004.docx");
WordUtil.save(doc,"C:\\Users\\14666\\Desktop\\report.docx");
}
}