Compare commits

...

27 Commits

Author SHA1 Message Date
jshixiong 640ff13a57 fix 2023-03-12 15:29:40 +08:00
jshixiong da4599f7d2 fix 2023-03-12 15:20:13 +08:00
jshixiong 7359902c83 drone触发检测 2023-03-12 15:15:55 +08:00
youys d322ffb172 fix 2023-03-11 22:42:59 +08:00
youys 95ec6a5d54 fix 2023-03-11 22:36:29 +08:00
youys 5c24798452 fix 2023-03-11 22:32:26 +08:00
youys 5e596ab716 判空 2023-03-11 21:53:56 +08:00
youys 8d44f9a09d fix 2023-03-10 18:32:17 +08:00
youys 1ba1f024db fix 2023-03-10 18:28:10 +08:00
youys 23c1d3d3aa 移除sonar 2023-03-10 18:24:39 +08:00
weishao 2e8175c613 language空格处理 2023-03-10 13:00:10 +08:00
weishao 76f677e716 Merge branch 'clone_detection' into merge 2023-03-10 12:48:40 +08:00
weishao 69f7a292da nil更新 2023-03-10 12:48:10 +08:00
weishao 3658950778 nil更新 2023-03-10 12:47:21 +08:00
jshixiong 6e06d1c503 fix 组件漏洞列表 2023-03-10 11:36:42 +08:00
jshixiong 6f94977201 Merge remote-tracking branch 'origin/merge' into merge 2023-03-10 11:25:20 +08:00
jshixiong 50ff819dcf fix 2023-03-10 11:25:05 +08:00
youys e58571fc56 Merge branch 'merge' of https://gitlink.org.cn/youys/quality_analysis into merge 2023-03-10 11:18:44 +08:00
youys 9fcc9b743f 增加修复建议字段 2023-03-10 11:18:40 +08:00
jshixiong 2fccbd18f7 fix 2023-03-10 11:13:24 +08:00
jshixiong ebc896c525 page 2023-03-10 11:11:24 +08:00
jshixiong 281dfc78f3 bug type 2023-03-10 10:59:43 +08:00
jshixiong 86aba22812 fix 2023-03-10 10:16:37 +08:00
jshixiong 4ed06f6e1c fix 2023-03-10 09:58:54 +08:00
jshixiong fb0e3bd0e2 bug right 2023-03-10 09:55:16 +08:00
jshixiong 11d2325931 fix 2023-03-10 09:22:34 +08:00
jshixiong 0a4ea731df bug 2023-03-09 19:57:15 +08:00
20 changed files with 652 additions and 108 deletions

View File

@ -20,4 +20,6 @@ public interface QualityConstants {
String TAR_GZ = "tar.gz";
String SUCCESS = "SUCCESS";
String GET_LANGUAGE_API = "/api/v1/repos/%s/%s/issues/statistics/language";
}

View File

@ -0,0 +1,31 @@
package net.educoder.quality.common.enums;
/**
* sast bug enum
* @author jshixiong
*/
public enum SastBugTypeEnum {
DEADLY("0","致命"),
SEVERITY("1","严重"),
ORDINARY("2","一般"),
HINT("3","提示"),
FORCE("4","强制"),
PROPOSAL("5","建议");
String value;
String description;
SastBugTypeEnum(String value, String description) {
this.value = value;
this.description = description;
}
public String getValue() {
return value;
}
public String getDescription() {
return description;
}
}

View File

@ -74,6 +74,18 @@ public class ProjectController {
return R.success();
}
/**
* drone项目检测
*
* @param detectionVO
* @return
*/
@PostMapping("/projects/droneDetect")
@OperateLogAnnotation(description = "drone项目检测")
public R<String> droneDetection(@RequestBody @Valid DroneDetectionVO detectionVO) {
String taskId = projectService.droneDetect(detectionVO);
return R.success(taskId);
}
/**
* 项目检测
@ -177,12 +189,10 @@ public class ProjectController {
*/
@GetMapping("/projects/{projectId}/bug/center")
public R<PageInfo> projectBugCenter(@PathVariable Long projectId, @Valid ProjectBugCenterVO projectBugCenterVO) {
PageInfo<ProjectBugCenterDTO> projectBugCenterDTOPageInfo = projectService.projectBugCenter(projectId, projectBugCenterVO);
PageInfo<ProjectSastBugCenterDTO> projectBugCenterDTOPageInfo = projectService.projectSastBugCenter(projectId, projectBugCenterVO);
return R.success(projectBugCenterDTOPageInfo);
}
/**
* 缺陷列表导出
*
@ -190,7 +200,7 @@ public class ProjectController {
*/
@GetMapping("/projects/{projectId}/bug/export")
public void projectBugExport(@PathVariable Long projectId, @Valid ProjectBugCenterVO projectBugCenterVO, HttpServletResponse response) throws Exception {
List<ProjectBugCenterDTO> projectBugCenterDTOList = projectService.projectBugList(projectId, projectBugCenterVO);
List<ProjectSastBugCenterDTO> projectBugCenterDTOList = projectService.projectSastBugList(projectId, projectBugCenterVO);
Projects projects = projectService.getProjectById(projectId);
response.setCharacterEncoding("utf-8");
@ -198,7 +208,7 @@ public class ProjectController {
response.setHeader("Content-disposition", "attachment;filename=" + fileName + "_bug.xlsx");
response.setContentType("application/vnd.ms-excel");
EasyExcel.write(response.getOutputStream(), ProjectBugCenterDTO.class).sheet().doWrite(projectBugCenterDTOList);
EasyExcel.write(response.getOutputStream(), ProjectSastBugCenterDTO.class).sheet().doWrite(projectBugCenterDTOList);
}
/**

View File

@ -23,6 +23,6 @@ public class ClonePairDTO {
this.targetFileStartLine = targetFileStartLine;
this.targetFileEndLine = targetFileEndLine;
this.similarity = similarity;
this.similarLines = Math.max(Math.min(sourceFileEndLine - sourceFileStartLine, targetFileEndLine - targetFileStartLine), 0);
this.similarLines = Math.max(Math.min(sourceFileEndLine - sourceFileStartLine + 1, targetFileEndLine - targetFileStartLine + 1), 0);
}
}

View File

@ -25,6 +25,10 @@ public class ProjectBugCenterCodeDetailV2DTO {
* 缺陷描述
*/
private String description;
/**
* 修复建议
*/
private String repairOpinion;
/**
* 正确代码示例
*/

View File

@ -74,27 +74,32 @@ public class ProjectBugDetailDTO {
private Long bugTotalNumber;
/**
* 严重的 对应 critical
* DEADLY 0 致命
*/
private Long critical;
/**
* 高危 对应 blocker
*/
private Long high;
private Long deadly = 0L;
/**
* 对应 major
* SEVERITY 1 严重
*/
private Long middle;
private Long severity = 0L;
/**
* 对应 minor
* ORDINARY 2 一般
*/
private Long low;
private Long ordinary = 0L;
/**
* 未知
* HINT 3 提示
*/
private Long unKnow = 0L;
private Long hint = 0L;
/**
* FORCE 4 强制
*/
private Long force = 0L;
/**
* PROPOSAL 5 建议
*/
private Long proposal = 0L;
}
}

View File

@ -0,0 +1,56 @@
package net.educoder.quality.dto;
import com.alibaba.excel.annotation.ExcelIgnoreUnannotated;
import com.alibaba.excel.annotation.ExcelProperty;
import lombok.Data;
import java.util.Date;
/**
* @Author: jshixiong
* @Date: 2023/3/9
* @Description:
*/
@Data
@ExcelIgnoreUnannotated
public class ProjectSastBugCenterDTO {
/**
* 缺陷名称
*/
@ExcelProperty("缺陷名称")
private String bugName;
/**
* 缺陷描述
*/
@ExcelProperty("缺陷描述")
private String bugDescription;
/**
* 文件路径
*/
@ExcelProperty("文件路径")
private String filePath;
/**
* 行号
*/
@ExcelProperty("行号")
private String rowNumber;
/**
* 缺陷级别
*/
@ExcelProperty("缺陷级别")
private String bugLevel;
/**
* 检测时间
*/
@ExcelProperty("检测时间")
private Date detectTime;
/**
* 找文件唯一标识
*/
private String uuid;
private Integer ruleId;
}

View File

@ -0,0 +1,35 @@
package net.educoder.quality.dto;
import lombok.Data;
@Data
public class SastBugResultDTO extends BugResultDTO{
/**
* DEADLY 0
*/
private Long deadly = 0L;
/**
* SEVERITY 对应 1
*/
private Long severity = 0L;
/**
* ORDINARY 对应 2
*/
private Long ordinary = 0L;
/**
* HINT 对应 3
*/
private Long hint = 0L;
/**
* FORCE 对应 4
*/
private Long force = 0L;
/**
* PROPOSAL 对应 5
*/
private Long proposal = 0L;
}

View File

@ -3,5 +3,8 @@ package net.educoder.quality.mapper.mysql;
import net.educoder.quality.common.util.BaseMapper;
import net.educoder.quality.entity.mysql.SastAnalysisDetail;
import java.util.List;
public interface SastAnalysisDetailMapper extends BaseMapper<SastAnalysisDetail> {
}

View File

@ -53,6 +53,12 @@ public interface ProjectService {
*/
String detection(DetectionVO detectionVO);
/**
* drone检测
* @return
*/
String droneDetect(DroneDetectionVO droneDetectionVO);
/**
* 更新task和project状态
*
@ -128,6 +134,14 @@ public interface ProjectService {
*/
PageInfo<ProjectBugCenterDTO> projectBugCenter(Long projectId, ProjectBugCenterVO projectBugCenterVO);
/**
* 缺陷中心-sast
* @param projectId
* @param projectBugCenterVO
* @return
*/
PageInfo<ProjectSastBugCenterDTO> projectSastBugCenter(Long projectId, ProjectBugCenterVO projectBugCenterVO);
/**
* 缺陷列表
*
@ -136,6 +150,14 @@ public interface ProjectService {
*/
List<ProjectBugCenterDTO> projectBugList(Long projectId, ProjectBugCenterVO projectBugCenterVO);
/**
* 缺陷列表-sast
*
* @param projectId
* @param projectBugCenterVO
*/
List<ProjectSastBugCenterDTO> projectSastBugList(Long projectId, ProjectBugCenterVO projectBugCenterVO);
/**
* 获取缺陷中心代码详情

View File

@ -30,7 +30,7 @@ import net.educoder.quality.service.ProjectService;
import net.educoder.quality.vo.AddCloneDetectionVO;
import net.educoder.quality.vo.PageVO;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.commons.io.FileUtils;
import org.aspectj.util.FileUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -38,9 +38,13 @@ import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.LineNumberReader;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Slf4j
@Service
@ -60,6 +64,98 @@ public class CloneDetectionServiceImpl implements CloneDetectionService {
@Resource
private CloneDetectionMapper cloneDetectionMapper;
private static final String[] textFileSuffix = new String[]{
"py",
"h",
"c",
"cpp",
"cc",
"java",
"php",
"html",
"css",
"scss",
"go",
"r",
"graphql",
"swift",
"xml",
"yaml",
"json",
"lua",
"scheme",
"less",
"ini",
"coffee",
"litcoffee",
"js",
"cs",
"kt",
"md",
"sql",
"m",
"mm",
"pas",
"perl",
"ejs",
"pl",
"rb",
"rs",
"rust",
"sh",
"makefile",
"circ",
"readme",
"yml",
"sml",
"conf",
"txt",
"gitignore",
"in",
"cu",
"gemfile",
"scala",
"net",
"l",
"v",
"config",
"properties",
"log",
"htm",
"cnf",
"hex",
"bat",
"asm",
"bash",
"ts",
"tsx",
"sass",
"jsx",
"jsp",
"gitkeep",
"sv",
"hql",
"y",
"jj",
"pls",
"sol",
"ignore",
"ctrl",
"vue",
"tex",
"bib",
"cls",
"bst",
"toc",
"sty",
"g4",
"sy",
"ipynb",
"m",
"mm",
"groovy"
};
@Override
@Transactional(rollbackFor = Exception.class)
public void add(Long projectId, AddCloneDetectionVO addCloneDetectionVO) {
@ -155,18 +251,21 @@ public class CloneDetectionServiceImpl implements CloneDetectionService {
log.error("save clone detail to database failed, source: {}, target: {}", fullPath + clonePairDTO.getSourceFile(), fullPath + clonePairDTO.getTargetFile(), e);
}
}
// 取总文件信息记录各项值
List<File> files = getFiles(fullPath + "/source/" + project.getProjectName());
CloneDetectionResultProjectLevel cloneDetectionResultProjectLevel = new CloneDetectionResultProjectLevel();
cloneDetectionResultProjectLevel.setCloneDetectionId(cloneDetectionId);
cloneDetectionResultProjectLevel.setSimilarityFile(similarSourceLinesMap.size());
long totalFile = getFileCount(fullPath + "/source");
long totalFile = files.size();
cloneDetectionResultProjectLevel.setSimilarityFilePercent(Math.min(((double)similarSourceLinesMap.size()) / totalFile, 1));
cloneDetectionResultProjectLevel.setTotalFile(totalFile);
cloneDetectionResultProjectLevel.setSimilarityLine(similarLines);
long totalLine = getLinesCount(fullPath + "/source/" + project.getProjectName());
long totalLine = getLinesCount(files);
cloneDetectionResultProjectLevel.setSimilarityLinePercent(Math.min(((double)similarLines) / totalLine, 1));
cloneDetectionResultProjectLevel.setTotalLine(totalLine);
cloneDetectionResultProjectLevel.setSimilarityCapacity(similarCapacityCount);
long totalCapacity = getCapacityCount(fullPath + "/source/" + project.getProjectName());
long totalCapacity = getCapacityCount(files);
cloneDetectionResultProjectLevel.setSimilarityCapacityPercent(Math.min(((double)similarCapacityCount) / totalCapacity, 1));
cloneDetectionResultProjectLevel.setTotalCapacity(totalCapacity);
cloneDetectionResultProjectLevel.setTargetProjectUrl(targetProjectUrl);
@ -267,7 +366,7 @@ public class CloneDetectionServiceImpl implements CloneDetectionService {
String resultCSVPath = sourcePath + "/result.csv";
// java -jar /Users/weiwang/IdeaProjects/NIL/build/libs/NIL-all.jar -s /Users/weiwang/IdeaProjects/bridge/business-service/game/src/main/java/com/educoder/bridge/game -o /tmp/a.csv
String command = "java -jar " + jarPath + " -o " + resultCSVPath +
" -s " + sourcePath + " -l " + language + " -mil " + propertiesConfig.getMil()
" -s " + sourcePath + " -l " + language.trim().replace(" ", "") + " -mil " + propertiesConfig.getMil()
+ " -mit " + propertiesConfig.getMit() + " -f " + propertiesConfig.getFiltrationThreshold()
+ " -v " + propertiesConfig.getVerificationThreshold();
String nilOut = ShellUtil.execute(command);
@ -310,30 +409,39 @@ public class CloneDetectionServiceImpl implements CloneDetectionService {
/**
* 统计文件夹下文件个数
*/
private int getFileCount(String repoPath) {
String cnt = ShellUtil.execute("ls -lR " + repoPath + "| grep \"^-\" | wc -l");
return NumberUtils.toInt(cnt, 0);
private List<File> getFiles(String repoPath) {
Collection<File> files = FileUtils.listFiles(new File(repoPath), textFileSuffix, true);
return files.stream().filter(file -> !file.isHidden()).collect(Collectors.toList());
}
/**
* 统计文件夹下文件行数
*/
private int getLinesCount(String repoPath) {
String cntResult = ShellUtil.execute("cd " + repoPath + " && git ls-files | xargs cat | wc -l");
String[] split = cntResult.trim().split("\n");
return NumberUtils.toInt(split[split.length - 1].trim(), Integer.MAX_VALUE);
private int getLinesCount(List<File> files) {
int count = 0;
for (File file : files) {
FileReader in;
try {
in = new FileReader(file);
LineNumberReader reader = new LineNumberReader(in);
reader.skip(Long.MAX_VALUE);
int lines = reader.getLineNumber();
reader.close();
count+= lines;
} catch (IOException e) {
}
}
return count;
}
/**
* 统计文件夹大小
* @param repoPath
* @param files
* @return
*/
private long getCapacityCount(String repoPath) {
// mac上为String cntResult = ShellUtil.execute("cd " + repoPath + " && du -s -k -I \"\\.git\" | awk '{print $1}'");
String cntResult = ShellUtil.execute("cd " + repoPath + " && du -s -k --exclude=\"\\.git\" | awk '{print $1}'");
String[] split = cntResult.trim().split("\n");
return NumberUtils.toInt(split[split.length - 1].trim(), Integer.MAX_VALUE) * 1024L;
private long getCapacityCount(List<File> files) {
return files.stream().mapToLong(File::length).reduce(0, Long::sum);
}
}

View File

@ -1,5 +1,6 @@
package net.educoder.quality.service.impl;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.date.DateUnit;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.RandomUtil;
@ -102,6 +103,9 @@ public class ProjectServiceImpl implements ProjectService {
@Resource
private SastService sastService;
@Autowired
private SastAnalysisDetailMapper sastMapper;
@Override
public PageInfo<ProjectsDTO> getProjectList(ProjectsVO projectsVO) {
@ -205,7 +209,7 @@ public class ProjectServiceImpl implements ProjectService {
/**
* 提交扫描任务
*/
submitScanTask(projects, taskInfo.getTaskId(), detectionTemplate, taskInfo);
// submitScanTask(projects, taskInfo.getTaskId(), detectionTemplate, taskInfo);
// 异步处理时间太长
@ -241,6 +245,44 @@ public class ProjectServiceImpl implements ProjectService {
return taskInfo.getTaskId();
}
@Override
public String droneDetect(DroneDetectionVO droneDetectionVO) {
//根据创建者分支名和项目名查找project
Projects d = new Projects();
d.setBranch(droneDetectionVO.getBranchName());
d.setProjectName(droneDetectionVO.getRepository());
d.setCreator(droneDetectionVO.getRepoOwner());
List<Projects> select = projectsMapper.select(d);
if (select.size()<=0){
throw new BusinessException(ErrorCodeEnum.PARAM_ERROR.getValue(), "项目检测任务不存在");
}
Projects projects = select.get(0);
if (projects.getStatus() == 1) {
throw new BusinessException(ErrorCodeEnum.UNDER_DETECTION_ERROR);
}
//按最近一次成功的来
ProjectDetectionTaskInfo projectDetectionTaskInfo = projectDetectionTaskInfoMapper.selectLastSuccessByProjectId(projects.getId());
if (projectDetectionTaskInfo==null){
throw new BusinessException(ErrorCodeEnum.PARAM_ERROR.getValue(), "请先进行首次检测!");
}
DetectionTemplate detectionTemplate = detectionTemplateMapper.selectByPrimaryKey(projectDetectionTaskInfo.getTemplateId());
if (detectionTemplate == null) {
throw new BusinessException(ErrorCodeEnum.PARAM_ERROR.getValue(), "检测模板不存在");
}
DetectionVO detectionVO = new DetectionVO();
detectionVO.setProjectId(projects.getId());
detectionVO.setTemplateId(projectDetectionTaskInfo.getTemplateId());
detectionVO.setRepository(droneDetectionVO.getRepository());
detectionVO.setRepoOwner(droneDetectionVO.getRepoOwner());
detectionVO.setCurrentUser(droneDetectionVO.getRepoOwner());
detectionVO.setType(0);
return detection(detectionVO);
}
@Override
@Transactional(rollbackFor = Exception.class)
public void updateTaskAndProjectStatus(String taskId, int status, Integer filesNum, String fileSize) {
@ -347,14 +389,14 @@ public class ProjectServiceImpl implements ProjectService {
String projectName = String.format("%s-%s", projectDetectionTaskInfo.getProjectName(), projectDetectionTaskInfo.getRandomStr());
// 缺陷&漏洞
BugResultDTO bug = pgProjectsService.getBug(projectName);
SastBugResultDTO bug = pgProjectsService.getSastBug(projectDetectionTaskInfo.getTaskId());
VulnerabilityResultDTO vulnerability = pgProjectsService.getNewComponentVulnerability(projectId);
projectOverviewDTO.setBugDetail(bug);
projectOverviewDTO.setVulnerabilityDetail(vulnerability);
// 获取缺陷漏洞
ProjectMetricsDTO projectMetrics = pgProjectsService.getProjectMetrics(projectName);
ProjectMetricsDTO projectMetrics = pgProjectsService.getProjectMetrics(projectName,projectDetectionTaskInfo.getTaskId());
projectOverviewDTO.setBug(new ProjectOverviewDTO.Metres(projectMetrics.getBugNumber(), 0));
projectOverviewDTO.setVulnerability(new ProjectOverviewDTO.Metres(vulnerability.getTotal().intValue(), 0));
@ -376,7 +418,7 @@ public class ProjectServiceImpl implements ProjectService {
String projectName2 = String.format("%s-%s", projectDetectionTaskInfo2.getProjectName(), projectDetectionTaskInfo2.getRandomStr());
// 缺陷&漏洞
BugResultDTO bug = pgProjectsService.getBug(projectName);
SastBugResultDTO bug = pgProjectsService.getSastBug(projectDetectionTaskInfo.getTaskId());
VulnerabilityResultDTO vulnerability = pgProjectsService.getNewComponentVulnerability(projectId);
VulnerabilityResultDTO vulnerability2 = pgProjectsService.getOldComponentVulnerability(projectId);
@ -384,10 +426,10 @@ public class ProjectServiceImpl implements ProjectService {
projectOverviewDTO.setVulnerabilityDetail(vulnerability);
// 获取缺陷漏洞
ProjectMetricsDTO projectMetrics = pgProjectsService.getProjectMetrics(projectName);
ProjectMetricsDTO projectMetrics2 = pgProjectsService.getProjectMetrics(projectName2);
ProjectMetricsDTO projectMetrics = pgProjectsService.getProjectMetrics(projectName,projectDetectionTaskInfo.getTaskId());
ProjectMetricsDTO projectMetrics2 = pgProjectsService.getProjectMetrics(projectName2,projectDetectionTaskInfo2.getTaskId());
projectOverviewDTO.setBug(new ProjectOverviewDTO.Metres(projectMetrics2.getBugNumber(),
projectOverviewDTO.setBug(new ProjectOverviewDTO.Metres(projectMetrics.getBugNumber(),
projectMetrics.getBugNumber() - projectMetrics2.getBugNumber()));
projectOverviewDTO.setVulnerability(new ProjectOverviewDTO.Metres(vulnerability.getTotal().intValue(),
vulnerability.getTotal().intValue() - vulnerability2.getTotal().intValue()));
@ -414,9 +456,12 @@ public class ProjectServiceImpl implements ProjectService {
throw new BusinessException(ErrorCodeEnum.PROJECT_NOT_SUCCESS);
}
String projectName = String.format("%s-%s", projectDetectionTaskInfo.getProjectName(), projectDetectionTaskInfo.getRandomStr());
Projects projects = projectsMapper.selectByPrimaryKey(projectId);
if(projects == null){
throw new BusinessException(ErrorCodeEnum.PROJECT_NOT_EXISTS);
}
ProjectBugDTO projectBugInfo = pgProjectsService.getProjectBugInfo(projectName);
ProjectBugDTO projectBugInfo = pgProjectsService.getProjectBugInfo(projects,projectDetectionTaskInfo.getTaskId());
return projectBugInfo;
}
@ -454,15 +499,17 @@ public class ProjectServiceImpl implements ProjectService {
detectResult.setFileNumber(projects.getTargetFileNum());
String projectName = String.format("%s-%s", projectDetectionTaskInfo.getProjectName(), projectDetectionTaskInfo.getRandomStr());
BugResultDTO bug = pgProjectsService.getBug(projectName);
PgProjectMeasures projectMeasures = pgProjectsService.getProjectMeasureByMetricId(projectName, 1);
if (projectMeasures != null) {
detectResult.setCodeNumber(projectMeasures.getValue().intValue());
} else {
detectResult.setCodeNumber(0);
}
// String projectName = String.format("%s-%s", projectDetectionTaskInfo.getProjectName(), projectDetectionTaskInfo.getRandomStr());
SastBugResultDTO bug = pgProjectsService.getSastBug(projectDetectionTaskInfo.getTaskId());
//
// PgProjectMeasures projectMeasures = pgProjectsService.getProjectMeasureByMetricId(projectName, 1);
// if (projectMeasures != null) {
// detectResult.setCodeNumber(projectMeasures.getValue().intValue());
// } else {
// detectResult.setCodeNumber(0);
// }
ProjectBugDTO projectBugDTO = projectBug(projectId);
detectResult.setCodeNumber(projectBugDTO.getCodeNumber());
detectResult.setBugTotalNumber(bug.getTotal());
BeanUtils.copyProperties(bug, detectResult);
@ -524,6 +571,33 @@ public class ProjectServiceImpl implements ProjectService {
return pageInfo;
}
@Override
public PageInfo<ProjectSastBugCenterDTO> projectSastBugCenter(Long projectId, ProjectBugCenterVO projectBugCenterVO) {
ProjectDetectionTaskInfo projectDetectionTaskInfo = projectDetectionTaskInfoMapper.selectLastSuccessByProjectId(projectId);
PageHelper.startPage(projectBugCenterVO.getPageNum(), projectBugCenterVO.getPageSize());
SastAnalysisDetail find = new SastAnalysisDetail();
find.setTaskId(projectDetectionTaskInfo.getTaskId());
List<SastAnalysisDetail> sastBugList = sastMapper.select(find);
PageInfo<SastAnalysisDetail> pageInfo = new PageInfo<>(sastBugList);
List<ProjectSastBugCenterDTO> resList = new ArrayList<>();
for (SastAnalysisDetail detail:sastBugList){
ProjectSastBugCenterDTO projectBugCenterDTO = new ProjectSastBugCenterDTO();
projectBugCenterDTO.setBugName(detail.getBugName());
projectBugCenterDTO.setBugDescription(detail.getDescription());
projectBugCenterDTO.setBugLevel(getSastBugDescriptionByValue(detail.getBugLevel()));
projectBugCenterDTO.setRowNumber(detail.getLine().toString());
projectBugCenterDTO.setDetectTime(detail.getCreateTime());
projectBugCenterDTO.setFilePath(detail.getFilePath());
projectBugCenterDTO.setRuleId(detail.getId().intValue());
resList.add(projectBugCenterDTO);
}
PageInfo<ProjectSastBugCenterDTO> result = new PageInfo<>();
BeanUtil.copyProperties(pageInfo, result);
result.setList(resList);
return result;
}
@Override
public List<ProjectBugCenterDTO> projectBugList(Long projectId, ProjectBugCenterVO projectBugCenterVO) {
@ -573,6 +647,29 @@ public class ProjectServiceImpl implements ProjectService {
return projectBugCenterDTOList;
}
@Override
public List<ProjectSastBugCenterDTO> projectSastBugList(Long projectId, ProjectBugCenterVO projectBugCenterVO) {
ProjectDetectionTaskInfo projectDetectionTaskInfo = projectDetectionTaskInfoMapper.selectLastSuccessByProjectId(projectId);
SastAnalysisDetail find = new SastAnalysisDetail();
find.setTaskId(projectDetectionTaskInfo.getTaskId());
List<SastAnalysisDetail> sastBugList = sastMapper.select(find);
List<ProjectSastBugCenterDTO> resList = new ArrayList<>();
for (SastAnalysisDetail detail:sastBugList){
ProjectSastBugCenterDTO projectBugCenterDTO = new ProjectSastBugCenterDTO();
projectBugCenterDTO.setBugName(detail.getBugName());
projectBugCenterDTO.setBugDescription(detail.getDescription());
projectBugCenterDTO.setBugLevel(getSastBugDescriptionByValue(detail.getBugLevel()));
projectBugCenterDTO.setRowNumber(detail.getLine().toString());
projectBugCenterDTO.setDetectTime(detail.getCreateTime());
projectBugCenterDTO.setFilePath(detail.getFilePath());
projectBugCenterDTO.setRuleId(detail.getId().intValue());
resList.add(projectBugCenterDTO);
}
return resList;
}
@Override
public ProjectBugCenterCodeDetailDTO projectBugCenterCodeDetail(Long projectId, ProjectBugCenterCodeDetailVO centerCodeDetailVO) {
PgFileSource fileSourceByUuid = pgProjectsService.getFileSourceByUuid(centerCodeDetailVO.getUuid());
@ -997,4 +1094,23 @@ public class ProjectServiceImpl implements ProjectService {
}
return gitUrl.replaceAll(router, "") + ".git";
}
private String getSastBugDescriptionByValue(String value){
switch (value) {
case "0":
return SastBugTypeEnum.DEADLY.getDescription();
case "1":
return SastBugTypeEnum.SEVERITY.getDescription();
case "2":
return SastBugTypeEnum.ORDINARY.getDescription();
case "3":
return SastBugTypeEnum.HINT.getDescription();
case "4":
return SastBugTypeEnum.FORCE.getDescription();
case "5":
return SastBugTypeEnum.PROPOSAL.getDescription();
default:
return null;
}
}
}

View File

@ -1,6 +1,7 @@
package net.educoder.quality.service.impl;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.RandomUtil;
@ -10,6 +11,7 @@ import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;
import net.educoder.quality.common.bean.ShellResult;
import net.educoder.quality.common.config.PropertiesConfig;
import net.educoder.quality.common.constant.QualityConstants;
import net.educoder.quality.common.enums.ErrorCodeEnum;
import net.educoder.quality.common.exception.BusinessException;
import net.educoder.quality.common.util.GitUtil;
@ -62,6 +64,7 @@ public class SastServiceImpl implements SastService {
ShellResult gitCloneResult = GitUtil.gitClone(projects.getGitUrl(), projects.getBranch(), propertiesConfig.getGitUsername(), propertiesConfig.getGitPassword(), fullPath);
if (gitCloneResult.getExitStatus() != 0) {
log.info("projectId:{}克隆代码失败,终止执行", projectId);
projectService.updateTaskAndProjectStatus(taskId, -1, null,null);
return;
}
@ -74,24 +77,29 @@ public class SastServiceImpl implements SastService {
log.info("SAST结果: {}, projectId: {}, taskId: {}", sastResult, projectId, taskId);
List<String> bugTypes = sastResult.stream().map(SastBugDTO::getBugType).collect(Collectors.toList());
// 取结果封装详情入库
List<BizKnowledgeBug> bizKnowledgeBugs = bizKnowledgeBugMapper.selectBugAndCatalogByTypes(bugTypes);
List<SastAnalysisDetail> sastAnalysisDetails = new ArrayList<>();
for (SastBugDTO sastBugDTO : sastResult) {
SastAnalysisDetail sastAnalysisDetail = new SastAnalysisDetail();
BeanUtils.copyProperties(sastBugDTO, sastAnalysisDetail);
if(CollectionUtil.isNotEmpty(bugTypes)) {
// 取结果封装详情入库
List<BizKnowledgeBug> bizKnowledgeBugs = bizKnowledgeBugMapper.selectBugAndCatalogByTypes(bugTypes);
List<SastAnalysisDetail> sastAnalysisDetails = new ArrayList<>();
for (SastBugDTO sastBugDTO : sastResult) {
SastAnalysisDetail sastAnalysisDetail = new SastAnalysisDetail();
BeanUtils.copyProperties(sastBugDTO, sastAnalysisDetail);
BizKnowledgeBug bizKnowledgeBug = bizKnowledgeBugs.stream().filter(bug -> bug.getBugType().equals(sastBugDTO.getBugType())).findFirst().get();
BeanUtils.copyProperties(bizKnowledgeBug, sastAnalysisDetail);
sastAnalysisDetail.setProjectId(projectId);
sastAnalysisDetail.setTaskId(taskId);
sastAnalysisDetail.setFilePath(fullPath + sastBugDTO.getFilePath());
sastAnalysisDetail.setCreateTime(DateTime.now());
sastAnalysisDetail.setUpdateTime(DateTime.now());
sastAnalysisDetails.add(sastAnalysisDetail);
BizKnowledgeBug bizKnowledgeBug = bizKnowledgeBugs.stream().filter(bug -> bug.getBugType().equals(sastBugDTO.getBugType())).findFirst().get();
BeanUtils.copyProperties(bizKnowledgeBug, sastAnalysisDetail);
sastAnalysisDetail.setProjectId(projectId);
sastAnalysisDetail.setTaskId(taskId);
sastAnalysisDetail.setFilePath(fullPath + sastBugDTO.getFilePath());
sastAnalysisDetail.setCreateTime(DateTime.now());
sastAnalysisDetail.setUpdateTime(DateTime.now());
sastAnalysisDetails.add(sastAnalysisDetail);
sastAnalysisDetailMapper.insertList(sastAnalysisDetails);
}
}
sastAnalysisDetailMapper.insertList(sastAnalysisDetails);
log.info("SAST结果入库完成, projectId: {}, taskId: {}", sastResult, projectId, taskId);
// 更新状态
projectService.updateTaskAndProjectStatus(taskId, 1, getDirectoryFilesNum(fullPath),getDirectoryFileSize(fullPath));
}
public List<SastBugDTO> execute(Map<String, String> paramMap) {
@ -180,5 +188,46 @@ public class SastServiceImpl implements SastService {
public SastAnalysisDetail findById(Long id) {
return sastAnalysisDetailMapper.selectByPrimaryKey(id);
}
/**
* 获取目录下的文件数量
*
* @param path
* @return
*/
private int getDirectoryFilesNum(String path) {
String command = StringUtils.join("cd ", path, " && ls -lR| grep \"^-\" | wc -l");
ShellResult shellResult = ShellUtil.executeAndGetExitStatus(command);
log.info("获取目录文件数返回command:{},result:{} ", command, JSONObject.toJSONString(shellResult));
try {
return Integer.parseInt(shellResult.getOut());
} catch (NumberFormatException e) {
return 0;
}
}
/**
* 获取目录下所有文件的大小
*
* @param path
* @return
*/
private String getDirectoryFileSize(String path) {
String command = StringUtils.join("cd ", path, "&& cd `ls` && du -sh . | awk -F ' ' '{print $1}'");
ShellResult shellResult = ShellUtil.executeAndGetExitStatus(command);
log.info("获取目录文件大小返回command:{},result:{} ", command, JSONObject.toJSONString(shellResult));
try {
if (shellResult.getExitStatus() == 0) {
return shellResult.getOut();
} else {
return "0k";
}
} catch (NumberFormatException e) {
return "0k";
}
}
}

View File

@ -33,11 +33,13 @@ public class VulnerabilityServiceImpl implements VulnerabilityService {
@Override
public PageInfo<ProjectVulnerabilityDetailDTO> vulnerabilityList(Long projectId, PageVO pageVO) {
Long componentId = componentParseRecordMapper.getByProjectId(projectId).getId();
PageHelper.startPage(pageVO.getPageNum(), pageVO.getPageSize());
ProjectVulnerabilityDetail projectVulnerabilityDetail = new ProjectVulnerabilityDetail();
projectVulnerabilityDetail.setProjectId(projectId);
projectVulnerabilityDetail.setComponentId(componentId);
List<ProjectVulnerabilityDetail> details = projectVulnerabilityDetailMapper.select(projectVulnerabilityDetail);
PageInfo<ProjectVulnerabilityDetail> doPageInfo = new PageInfo<>(details);

View File

@ -1,5 +1,6 @@
package net.educoder.quality.service.postgres;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.github.pagehelper.PageHelper;
@ -7,18 +8,14 @@ import com.github.pagehelper.PageInfo;
import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
import net.educoder.quality.common.constant.PgIssueConstant;
import net.educoder.quality.common.constant.QualityConstants;
import net.educoder.quality.common.enums.*;
import net.educoder.quality.common.exception.BusinessException;
import net.educoder.quality.common.util.WordUtil;
import net.educoder.quality.dto.*;
import net.educoder.quality.entity.mysql.ComponentParseRecord;
import net.educoder.quality.entity.mysql.ComponentParseRecordDetail;
import net.educoder.quality.entity.mysql.ProjectVulnerabilityDetail;
import net.educoder.quality.entity.mysql.*;
import net.educoder.quality.entity.postgres.*;
import net.educoder.quality.mapper.mysql.ComponentParseRecordDetailMapper;
import net.educoder.quality.mapper.mysql.ComponentParseRecordMapper;
import net.educoder.quality.mapper.mysql.ProjectVulnerabilityDetailMapper;
import net.educoder.quality.mapper.mysql.ProjectsMapper;
import net.educoder.quality.mapper.mysql.*;
import net.educoder.quality.mapper.postgres.PgCeActivityMapper;
import net.educoder.quality.mapper.postgres.PgProjectMapper;
import net.educoder.quality.protobuf.DbIssues;
@ -28,9 +25,12 @@ import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFTable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -65,6 +65,12 @@ public class PgProjectsService {
@Autowired
private ComponentParseRecordMapper componentParseRecordMapper;
@Autowired
private SastAnalysisDetailMapper sastMapper;
@Value("${openi.domain}")
private String openIDomain;
public PgProjects getProjectsByProjectName(String projectName) {
PgProjects project = projectMapper.findByName(projectName);
if (project == null) {
@ -367,6 +373,44 @@ public class PgProjectsService {
return resultDTO;
}
public SastBugResultDTO getSastBug(String taskId) {
SastAnalysisDetail find = new SastAnalysisDetail();
find.setTaskId(taskId);
List<SastAnalysisDetail> sastBugList = sastMapper.select(find);
SastBugResultDTO result = new SastBugResultDTO();
result.setTotal((long)sastBugList.size());
for (SastAnalysisDetail detail : sastBugList) {
switch (detail.getBugLevel()) {
case "0":
result.setDeadly(result.getDeadly() + 1);
result.setCritical(result.getCritical() + 1);
break;
case "1":
result.setSeverity(result.getSeverity() + 1);
result.setHigh(result.getHigh() + 1);
break;
case "2":
result.setOrdinary(result.getOrdinary() + 1);
result.setMiddle(result.getMiddle() + 1);
break;
case "3":
result.setHint(result.getHint() + 1);
result.setLow(result.getLow() + 1);
break;
case "4":
result.setForce(result.getForce() + 1);
break;
case "5":
result.setProposal(result.getProposal() + 1);
break;
default:
break;
}
}
return result;
}
/**
* 获取规则(漏洞)详情
@ -427,8 +471,11 @@ public class PgProjectsService {
* @param projectName
* @return
*/
public ProjectMetricsDTO getProjectMetrics(String projectName) {
Integer bugNumber = projectMapper.findIssueCountByProjectNameAndIssueType(projectName, IssueTypeEnum.BUG.getValue());
public ProjectMetricsDTO getProjectMetrics(String projectName,String taskId) {
SastAnalysisDetail find = new SastAnalysisDetail();
find.setTaskId(taskId);
List<SastAnalysisDetail> sastBugList = sastMapper.select(find);
Integer bugNumber = sastBugList.size();
Integer vulnerabilityNumber = projectMapper.findIssueCountByProjectNameAndIssueType(projectName, IssueTypeEnum.VULNERABILITY.getValue());
Integer codeNumber = projectMapper.findByCodeNumber(projectName);
@ -438,39 +485,32 @@ public class PgProjectsService {
/**
* 获取项目缺陷信息
*
* @param projectName
* @param projects
* @return
*/
public ProjectBugDTO getProjectBugInfo(String projectName) {
public ProjectBugDTO getProjectBugInfo(Projects projects,String taskId) {
ProjectBugDTO projectBugDTO = new ProjectBugDTO();
Integer codeNumber = projectMapper.findByCodeNumber(projectName);
Integer bugNumber = projectMapper.findIssueCountByProjectNameAndIssueType(projectName, IssueTypeEnum.BUG.getValue());
SastAnalysisDetail find = new SastAnalysisDetail();
find.setTaskId(taskId);
List<SastAnalysisDetail> sastBugList = sastMapper.select(find);
projectBugDTO.setCodeNumber(codeNumber);
Integer bugNumber = sastBugList.size();
projectBugDTO.setBugNumber(bugNumber);
PgProjectMeasures projectMeasures = projectMapper.findProjectMeasureByMetricId(projectName, 5);
getAndSetLanguage(projectBugDTO, projects.getCreator(), projects.getRepository());
String[] split = projectMeasures.getTextValue().split(";");
List<KeyValuePair> left = new ArrayList<>(split.length);
projectBugDTO.setLeft(left);
for (String languageAndLineNumber : split) {
String[] array = languageAndLineNumber.split("=");
left.add(new KeyValuePair(array[0], array[1]));
}
List<KeyValuePair> right = new ArrayList<>(split.length);
List<KeyValuePair> right = new ArrayList<>();
projectBugDTO.setRight(right);
BugResultDTO bug = getBug(projectName);
right.add(new KeyValuePair("严重", String.valueOf(bug.getCritical())));
right.add(new KeyValuePair("高危", String.valueOf(bug.getHigh())));
right.add(new KeyValuePair("中危", String.valueOf(bug.getMiddle())));
right.add(new KeyValuePair("低危", String.valueOf(bug.getLow())));
right.add(new KeyValuePair("未知", "0"));
SastBugResultDTO bug = getSastBug(taskId);
right.add(new KeyValuePair("致命", String.valueOf(bug.getDeadly())));
right.add(new KeyValuePair("严重", String.valueOf(bug.getSeverity())));
right.add(new KeyValuePair("一般", String.valueOf(bug.getOrdinary())));
right.add(new KeyValuePair("提示", String.valueOf(bug.getHint())));
right.add(new KeyValuePair("强制", String.valueOf(bug.getForce())));
right.add(new KeyValuePair("建议", String.valueOf(bug.getProposal())));
return projectBugDTO;
}
@ -525,4 +565,46 @@ public class PgProjectsService {
return number == null ? 0 : number.intValue();
}
private void getAndSetLanguage(ProjectBugDTO projectBugDTO, String repoOwner, String repoName){
try {
List<KeyValuePair> keyValuePairs = new ArrayList<>();
String response = HttpUtil.get(openIDomain + String.format(QualityConstants.GET_LANGUAGE_API, repoOwner, repoName));
JSONObject result = JSONObject.parseObject(response);
int repoSize = result.getIntValue("repo_size");
projectBugDTO.setCodeNumber(repoSize);
projectBugDTO.setLeft(keyValuePairs);
JSONArray langs = result.getJSONArray("langs");
if(langs != null) {
for (Object lang : langs) {
JSONObject obj = (JSONObject)lang;
BigDecimal percentage = BigDecimal.valueOf(obj.getDoubleValue("Percentage")).multiply(BigDecimal.valueOf(repoSize));
keyValuePairs.add(new KeyValuePair(obj.getString("Language"), setUnit(percentage)));
}
}
}catch (Exception e){
log.error("getLanguage error", e);
projectBugDTO.setLeft(new ArrayList<>());
}
}
private static String setUnit(BigDecimal number){
if (number.divide(BigDecimal.valueOf(1024 * 1024 * 1024)).setScale(0, RoundingMode.HALF_UP).doubleValue() > 0) {
return number.divide(BigDecimal.valueOf(1024 * 1024 * 1024)).setScale(0, RoundingMode.HALF_UP).toString() + "G";
}else if(number.divide(BigDecimal.valueOf(1024 * 1024)).setScale(0, RoundingMode.HALF_UP).doubleValue() > 0){
return number.divide(BigDecimal.valueOf(1024 * 1024)).setScale(0, RoundingMode.HALF_UP).toString() + "M";
}else if(number.divide(BigDecimal.valueOf(1024)).setScale(0, RoundingMode.HALF_UP).doubleValue() > 0){
return number.divide(BigDecimal.valueOf(1024)).setScale(0, RoundingMode.HALF_UP).toString() + "K";
}else{
return number.setScale(0,RoundingMode.HALF_UP).toString();
}
}
public static void main(String[] args) {
System.out.println(setUnit(BigDecimal.valueOf(10000)));
}
}

View File

@ -0,0 +1,12 @@
package net.educoder.quality.vo;
import lombok.Data;
import javax.validation.constraints.NotBlank;
@Data
public class DroneDetectionVO extends CommonVO{
@NotBlank(message = "branchName不能为空")
private String branchName;
}

View File

@ -2,6 +2,7 @@
openi:
gitUsername: wangwei
gitPassword: zq123456
domain: http://118.31.13.117:64300
# sonar相关配置
sonar:
# serverUrl: http://117.50.14.123:9000
@ -18,8 +19,8 @@ opensca-cli:
#path: /data/ww/open-cli/opensca-cli
path: /Users/youyongsheng/Downloads/opensca-cli_v1.0.9_Darwin_x86_64/opensca-cli
nil:
jarPath: /Users/weiwang/IdeaProjects/NIL/build/libs/NIL-all.jar
mil: 6
mit: 50
jarPath: /Users/weiwang/IdeaProjects/quality_analysis/web/src/main/resources/nil/nil.jar
mil: 1
mit: 1
filtrationThreshold: 10
verificationThreshold: 70

View File

@ -30,4 +30,5 @@
<result column="line" property="line" jdbcType="INTEGER" />
<result column="file_path" property="filePath" jdbcType="VARCHAR" />
</resultMap>
</mapper>

Binary file not shown.

View File

@ -1,7 +1,9 @@
package net.educoder.quality;
import cn.hutool.core.date.DateUtil;
import com.github.pagehelper.PageInfo;
import net.educoder.quality.common.util.WordUtil;
import net.educoder.quality.dto.ProjectSastBugCenterDTO;
import net.educoder.quality.dto.ReportDetailDTO;
import net.educoder.quality.entity.mysql.Projects;
import net.educoder.quality.entity.mysql.ReportCenter;
@ -12,6 +14,7 @@ import net.educoder.quality.mapper.postgres.PgProjectMapper;
import net.educoder.quality.service.ProjectService;
import net.educoder.quality.service.ReportCenterService;
import net.educoder.quality.service.postgres.PgProjectsService;
import net.educoder.quality.vo.ProjectBugCenterVO;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
@ -53,10 +56,12 @@ public class MapperTest {
@Test
@Transactional
public void testInsert(){
testMapper.insertTest("张三");
// testMapper.insertTest("李四");
int i=1/0;
ProjectBugCenterVO projectBugCenterVO = new ProjectBugCenterVO();
projectBugCenterVO.setPageSize(5);
projectBugCenterVO.setPageNum(2);
PageInfo<ProjectSastBugCenterDTO> res = projectService.projectSastBugCenter(86L, projectBugCenterVO);
System.out.println(res);
// System.out.println(pgProjectsService.getSastBug("475336319468306432"));
}
@Test