nil更新

2023-03-10 12:48:10 +08:00 · 2023-03-10 12:47:21 +08:00
12 changed files with 139 additions and 153 deletions
--- a/web/src/main/java/net/educoder/quality/controller/ProjectController.java
+++ b/web/src/main/java/net/educoder/quality/controller/ProjectController.java
@ -181,8 +181,6 @@ public class ProjectController {
        return R.success(projectBugCenterDTOPageInfo);
    }

-
-
    /**
     * 缺陷列表导出
     *
@ -213,18 +211,4 @@ public class ProjectController {
        return R.success(projectBugCenterCodeDetailDTO);
    }

-
-    /**
-     * 缺陷中心> 代码详情
-     * V2版本
-     *
-     * @param projectId
-     * @return
-     */
-    @GetMapping("/projects/v2/{projectId}/bug/center/codeDetail")
-    public R<ProjectBugCenterCodeDetailV2DTO> projectBugCenterCodeDetailV2(@PathVariable Long projectId, @RequestParam("ruleId") Integer id) {
-        ProjectBugCenterCodeDetailV2DTO codeDetailV2DTO = projectService.projectBugCenterCodeDetailV2(projectId, id);
-        return R.success(codeDetailV2DTO);
-    }
-
 }
--- a/web/src/main/java/net/educoder/quality/dto/ClonePairDTO.java
+++ b/web/src/main/java/net/educoder/quality/dto/ClonePairDTO.java
@ -23,6 +23,6 @@ public class ClonePairDTO {
        this.targetFileStartLine = targetFileStartLine;
        this.targetFileEndLine = targetFileEndLine;
        this.similarity = similarity;
-        this.similarLines = Math.max(Math.min(sourceFileEndLine - sourceFileStartLine, targetFileEndLine - targetFileStartLine), 0);
+        this.similarLines = Math.max(Math.min(sourceFileEndLine - sourceFileStartLine + 1, targetFileEndLine - targetFileStartLine + 1), 0);
    }
 }
--- a/web/src/main/java/net/educoder/quality/dto/FileSourceDTO.java
+++ b/web/src/main/java/net/educoder/quality/dto/FileSourceDTO.java
@ -1,8 +1,6 @@
 package net.educoder.quality.dto;

-import lombok.AllArgsConstructor;
 import lombok.Data;
-import lombok.NoArgsConstructor;

 /**
 * @Author: youys
@ -10,8 +8,6 @@ import lombok.NoArgsConstructor;
 * @Description:
 */
@Data
-@NoArgsConstructor
-@AllArgsConstructor
 public class FileSourceDTO {

    private Integer rowNumber;
--- a/web/src/main/java/net/educoder/quality/dto/ProjectBugCenterCodeDetailV2DTO.java
+++ b/web/src/main/java/net/educoder/quality/dto/ProjectBugCenterCodeDetailV2DTO.java
@ -1,44 +0,0 @@
-package net.educoder.quality.dto;
-
-import lombok.Data;
-
-import java.util.List;
-
-/**
- * @Author: youys
- * @Date: 2023/3/9
- * @Description: 缺陷中心>查看详情 v2版本
- */
-@Data
-public class ProjectBugCenterCodeDetailV2DTO {
-
-    private String filePath;
-    /**
-     * 代码行号
-     */
-    private Integer line;
-    /**
-     * 缺陷名称
-     */
-    private String bugName;
-    /**
-     * 缺陷描述
-     */
-    private String description;
-    /**
-     * 正确代码示例
-     */
-    private String  example;
-    /**
-     * 错误代码示例
-     */
-    private String errorExample;
-    /**
-     * 代码所属错误分类
-     */
-    private String parentBugCatalog;
-    /**
-     * 代码
-     */
-    private List<FileSourceDTO> codes;
-}
--- a/web/src/main/java/net/educoder/quality/service/ProjectService.java
+++ b/web/src/main/java/net/educoder/quality/service/ProjectService.java
@ -146,15 +146,6 @@ public interface ProjectService {
     */
    ProjectBugCenterCodeDetailDTO projectBugCenterCodeDetail(Long projectId, ProjectBugCenterCodeDetailVO centerCodeDetailVO);

-
-    /**
-     * 获取缺陷中心代码详情 V2版本
-     * @param projectId
-     * @param id
-     * @return
-     */
-    ProjectBugCenterCodeDetailV2DTO projectBugCenterCodeDetailV2(Long projectId, Integer id);
-
    /**
     * 许可证总计
     *
@ -267,5 +258,4 @@ public interface ProjectService {
     * @return
     */
    List<ProjectVulnerabilityListDTO> projectVulnerabilityExport(Long projectId, CommonVO commonVO);
-
 }
--- a/web/src/main/java/net/educoder/quality/service/SastService.java
+++ b/web/src/main/java/net/educoder/quality/service/SastService.java
@ -1,12 +1,5 @@
 package net.educoder.quality.service;

-import net.educoder.quality.entity.mysql.SastAnalysisDetail;
-
 public interface SastService {
    void analysis(Long projectId, String taskId);
-
-
-
-    SastAnalysisDetail findById(Long id);
-
 }
--- a/web/src/main/java/net/educoder/quality/service/impl/CloneDetectionServiceImpl.java
+++ b/web/src/main/java/net/educoder/quality/service/impl/CloneDetectionServiceImpl.java
@ -30,7 +30,7 @@ import net.educoder.quality.service.ProjectService;
 import net.educoder.quality.vo.AddCloneDetectionVO;
 import net.educoder.quality.vo.PageVO;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.commons.io.FileUtils;
 import org.aspectj.util.FileUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@ -38,9 +38,13 @@ import org.springframework.transaction.annotation.Transactional;

 import javax.annotation.Resource;
 import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.LineNumberReader;
 import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;

@Slf4j
@Service
@ -60,6 +64,98 @@ public class CloneDetectionServiceImpl implements CloneDetectionService {
    @Resource
    private CloneDetectionMapper cloneDetectionMapper;

+    private static final String[] textFileSuffix = new String[]{
+            "py",
+            "h",
+            "c",
+            "cpp",
+            "cc",
+            "java",
+            "php",
+            "html",
+            "css",
+            "scss",
+            "go",
+            "r",
+            "graphql",
+            "swift",
+            "xml",
+            "yaml",
+            "json",
+            "lua",
+            "scheme",
+            "less",
+            "ini",
+            "coffee",
+            "litcoffee",
+            "js",
+            "cs",
+            "kt",
+            "md",
+            "sql",
+            "m",
+            "mm",
+            "pas",
+            "perl",
+            "ejs",
+            "pl",
+            "rb",
+            "rs",
+            "rust",
+            "sh",
+            "makefile",
+            "circ",
+            "readme",
+            "yml",
+            "sml",
+            "conf",
+            "txt",
+            "gitignore",
+            "in",
+            "cu",
+            "gemfile",
+            "scala",
+            "net",
+            "l",
+            "v",
+            "config",
+            "properties",
+            "log",
+            "htm",
+            "cnf",
+            "hex",
+            "bat",
+            "asm",
+            "bash",
+            "ts",
+            "tsx",
+            "sass",
+            "jsx",
+            "jsp",
+            "gitkeep",
+            "sv",
+            "hql",
+            "y",
+            "jj",
+            "pls",
+            "sol",
+            "ignore",
+            "ctrl",
+            "vue",
+            "tex",
+            "bib",
+            "cls",
+            "bst",
+            "toc",
+            "sty",
+            "g4",
+            "sy",
+            "ipynb",
+            "m",
+            "mm",
+            "groovy"
+    };
+
    @Override
    @Transactional(rollbackFor = Exception.class)
    public void add(Long projectId, AddCloneDetectionVO addCloneDetectionVO) {
@ -155,18 +251,21 @@ public class CloneDetectionServiceImpl implements CloneDetectionService {
                log.error("save clone detail to database failed, source: {}, target: {}", fullPath + clonePairDTO.getSourceFile(), fullPath + clonePairDTO.getTargetFile(), e);
            }
        }
+
+        // 取总文件信息，记录各项值
+        List<File> files = getFiles(fullPath + "/source/" + project.getProjectName());
        CloneDetectionResultProjectLevel cloneDetectionResultProjectLevel = new CloneDetectionResultProjectLevel();
        cloneDetectionResultProjectLevel.setCloneDetectionId(cloneDetectionId);
        cloneDetectionResultProjectLevel.setSimilarityFile(similarSourceLinesMap.size());
-        long totalFile = getFileCount(fullPath + "/source");
+        long totalFile = files.size();
        cloneDetectionResultProjectLevel.setSimilarityFilePercent(Math.min(((double)similarSourceLinesMap.size()) / totalFile, 1));
        cloneDetectionResultProjectLevel.setTotalFile(totalFile);
        cloneDetectionResultProjectLevel.setSimilarityLine(similarLines);
-        long totalLine = getLinesCount(fullPath + "/source/" + project.getProjectName());
+        long totalLine = getLinesCount(files);
        cloneDetectionResultProjectLevel.setSimilarityLinePercent(Math.min(((double)similarLines) / totalLine, 1));
        cloneDetectionResultProjectLevel.setTotalLine(totalLine);
        cloneDetectionResultProjectLevel.setSimilarityCapacity(similarCapacityCount);
-        long totalCapacity = getCapacityCount(fullPath + "/source/" + project.getProjectName());
+        long totalCapacity = getCapacityCount(files);
        cloneDetectionResultProjectLevel.setSimilarityCapacityPercent(Math.min(((double)similarCapacityCount) / totalCapacity, 1));
        cloneDetectionResultProjectLevel.setTotalCapacity(totalCapacity);
        cloneDetectionResultProjectLevel.setTargetProjectUrl(targetProjectUrl);
@ -310,30 +409,39 @@ public class CloneDetectionServiceImpl implements CloneDetectionService {
    /**
     * 统计文件夹下文件个数
     */
-    private int getFileCount(String repoPath) {
-        String cnt = ShellUtil.execute("ls -lR " + repoPath + "| grep \"^-\" | wc -l");
-        return NumberUtils.toInt(cnt, 0);
+    private List<File> getFiles(String repoPath) {
+        Collection<File> files = FileUtils.listFiles(new File(repoPath), textFileSuffix, true);
+        return files.stream().filter(file -> !file.isHidden()).collect(Collectors.toList());
    }

    /**
     * 统计文件夹下文件行数
     */
-    private int getLinesCount(String repoPath) {
-        String cntResult = ShellUtil.execute("cd " + repoPath + " && git ls-files | xargs cat | wc -l");
-        String[] split = cntResult.trim().split("\n");
-        return NumberUtils.toInt(split[split.length - 1].trim(), Integer.MAX_VALUE);
+    private int getLinesCount(List<File> files) {
+        int count = 0;
+
+        for (File file : files) {
+            FileReader in;
+            try {
+                in = new FileReader(file);
+                LineNumberReader reader = new LineNumberReader(in);
+                reader.skip(Long.MAX_VALUE);
+                int lines = reader.getLineNumber();
+                reader.close();
+                count+= lines;
+            } catch (IOException e) {
+            }
+        }
+        return count;
    }

    /**
     * 统计文件夹大小
-     * @param repoPath
+     * @param files
     * @return
     */
-    private long getCapacityCount(String repoPath) {
-        // mac上为String cntResult = ShellUtil.execute("cd " + repoPath + " && du -s -k -I \"\\.git\" | awk '{print $1}'");
-        String cntResult = ShellUtil.execute("cd " + repoPath + " && du -s -k --exclude=\"\\.git\" | awk '{print $1}'");
-        String[] split = cntResult.trim().split("\n");
-        return NumberUtils.toInt(split[split.length - 1].trim(), Integer.MAX_VALUE) * 1024L;
+    private long getCapacityCount(List<File> files) {
+        return files.stream().mapToLong(File::length).reduce(0, Long::sum);
    }

 }
--- a/web/src/main/java/net/educoder/quality/service/impl/ProjectServiceImpl.java
+++ b/web/src/main/java/net/educoder/quality/service/impl/ProjectServiceImpl.java
@ -27,7 +27,6 @@ import net.educoder.quality.service.postgres.PgProjectsService;
 import net.educoder.quality.task.SonarDetectionRunnable;
 import net.educoder.quality.vo.*;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@ -38,8 +37,6 @@ import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;

 import javax.annotation.Resource;
-import java.io.File;
-import java.io.IOException;
 import java.util.*;

 /**
@ -458,9 +455,9 @@ public class ProjectServiceImpl implements ProjectService {
        BugResultDTO bug = pgProjectsService.getBug(projectName);

        PgProjectMeasures projectMeasures = pgProjectsService.getProjectMeasureByMetricId(projectName, 1);
-        if (projectMeasures != null) {
+        if(projectMeasures != null) {
            detectResult.setCodeNumber(projectMeasures.getValue().intValue());
-        } else {
+        }else{
            detectResult.setCodeNumber(0);
        }

@ -605,40 +602,6 @@ public class ProjectServiceImpl implements ProjectService {
        return projectBugCenterCodeDetailDTO;
    }

-    @Override
-    public ProjectBugCenterCodeDetailV2DTO projectBugCenterCodeDetailV2(Long projectId, Integer id){
-
-        SastAnalysisDetail sastAnalysisDetail = sastService.findById(Long.valueOf(id));
-        if(sastAnalysisDetail == null){
-            throw new BusinessException(ErrorCodeEnum.PARAM_ERROR);
-        }
-
-        ProjectBugCenterCodeDetailV2DTO codeDetailV2DTO = new ProjectBugCenterCodeDetailV2DTO();
-        BeanUtils.copyProperties(sastAnalysisDetail, codeDetailV2DTO);
-
-        String filePath = sastAnalysisDetail.getFilePath();
-
-        File file = new File(filePath);
-        if(file.exists()){
-            try {
-                List<String> codeStr = FileUtils.readLines(file, "UTF-8");
-
-                List<FileSourceDTO> codes = new ArrayList();
-                for (int i = 0; i < codeStr.size(); i++) {
-                    codes.add(new FileSourceDTO(i+1, codeStr.get(i)));
-                }
-                codeDetailV2DTO.setCodes(codes);
-            } catch (IOException e) {
-                log.error("projectBugCenterCodeDetailV2 读文件异常", e);
-                codeDetailV2DTO.setCodes(new ArrayList<>());
-            }
-        }else{
-            codeDetailV2DTO.setCodes(new ArrayList<>());
-        }
-
-        return codeDetailV2DTO;
-    }
-
    @Override
    public LicenceOverviewDTO projectLicenceOverview(Long projectId) {
        //TODO:暂用假数据
@ -899,7 +862,7 @@ public class ProjectServiceImpl implements ProjectService {
        sonarScannerParam.setLanguage(contains ? QualityConstants.C : QualityConstants.OTHER);

        boolean useNet = detectionTemplate.getLanguage().toLowerCase().contains(QualityConstants.NET);
-        if (useNet) {
+        if (useNet){
            sonarScannerParam.setLanguage(QualityConstants.NET);
        }
        sonarScannerParam.setWorkspace(workspace);
@ -984,17 +947,17 @@ public class ProjectServiceImpl implements ProjectService {

    /**
     * 处理giturl
+     * @param gitUrl  http://118.31.13.117:64300/wangwei/jwebssh/analysis Or http://118.31.13.117:64300/wangwei/jwebssh/analysis/
     *
-     * @param gitUrl http://118.31.13.117:64300/wangwei/jwebssh/analysis Or http://118.31.13.117:64300/wangwei/jwebssh/analysis/
     * @return
     */
-    private String processGitUrl(String gitUrl) {
+    private String processGitUrl(String gitUrl){
        String router = gitUrl.substring(gitUrl.lastIndexOf("/"));
        if ("/".equals(router) || StringUtils.isBlank(router)) {
            String newGitUrl = gitUrl.substring(0, gitUrl.length() - 1);
            router = newGitUrl.substring(newGitUrl.lastIndexOf("/"));
-            return newGitUrl.replaceAll(router, "") + ".git";
+            return newGitUrl.replaceAll(router,"") + ".git";
        }
-        return gitUrl.replaceAll(router, "") + ".git";
+        return gitUrl.replaceAll(router,"") + ".git";
    }
 }
--- a/web/src/main/java/net/educoder/quality/service/impl/SastServiceImpl.java
+++ b/web/src/main/java/net/educoder/quality/service/impl/SastServiceImpl.java
@ -176,9 +176,5 @@ public class SastServiceImpl implements SastService {
        return sastBugDTOS;
    }

-    @Override
-    public SastAnalysisDetail findById(Long id) {
-        return sastAnalysisDetailMapper.selectByPrimaryKey(id);
-    }
 }

--- a/web/src/main/resources/application-user.yml
+++ b/web/src/main/resources/application-user.yml
@ -18,8 +18,8 @@ opensca-cli:
  #path: /data/ww/open-cli/opensca-cli
  path: /Users/youyongsheng/Downloads/opensca-cli_v1.0.9_Darwin_x86_64/opensca-cli
 nil:
-  jarPath: /Users/weiwang/IdeaProjects/NIL/build/libs/NIL-all.jar
-  mil: 6
-  mit: 50
+  jarPath: /Users/weiwang/IdeaProjects/quality_analysis/web/src/main/resources/nil/nil.jar
+  mil: 1
+  mit: 1
  filtrationThreshold: 10
  verificationThreshold: 70
--- a/web/src/main/resources/application.yml
+++ b/web/src/main/resources/application.yml
@ -10,9 +10,9 @@ spring:
    # mysql
    master:
      driver-class-name: com.mysql.cj.jdbc.Driver
-      url: jdbc:mysql://rm-bp13v5020p7828r5rso.mysql.rds.aliyuncs.com:3306/quality_analysis?useUnicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false
-      username: testeducoder
-      password: TEST@123
+      url: jdbc:mysql://localhost/quality_analysis?useUnicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false
+      username: root
+      password: 12345678
      type: com.alibaba.druid.pool.DruidDataSource
      druid:
        initial-size: 20
--- a/web/src/main/resources/nil/nil.jar
+++ b/web/src/main/resources/nil/nil.jar
Author	SHA1	Message	Date
weishao	69f7a292da	nil更新	2023-03-10 12:48:10 +08:00
weishao	3658950778	nil更新	2023-03-10 12:47:21 +08:00