nil更新

web/src/main/java/net/educoder/quality/dto/ClonePairDTO.java

@@ -23,6 +23,6 @@ public class ClonePairDTO {
         this.targetFileStartLine = targetFileStartLine;
         this.targetFileEndLine = targetFileEndLine;
         this.similarity = similarity;
-        this.similarLines = Math.max(Math.min(sourceFileEndLine - sourceFileStartLine, targetFileEndLine - targetFileStartLine), 0);
+        this.similarLines = Math.max(Math.min(sourceFileEndLine - sourceFileStartLine + 1, targetFileEndLine - targetFileStartLine + 1), 0);
     }
 }

web/src/main/java/net/educoder/quality/service/impl/CloneDetectionServiceImpl.java

@@ -30,7 +30,7 @@ import net.educoder.quality.service.ProjectService;
 import net.educoder.quality.vo.AddCloneDetectionVO;
 import net.educoder.quality.vo.PageVO;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.commons.io.FileUtils;
 import org.aspectj.util.FileUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -38,9 +38,13 @@ import org.springframework.transaction.annotation.Transactional;
 
 import javax.annotation.Resource;
 import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.LineNumberReader;
 import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 @Slf4j
 @Service
@@ -60,6 +64,98 @@ public class CloneDetectionServiceImpl implements CloneDetectionService {
     @Resource
     private CloneDetectionMapper cloneDetectionMapper;
 
+    private static final String[] textFileSuffix = new String[]{
+            "py",
+            "h",
+            "c",
+            "cpp",
+            "cc",
+            "java",
+            "php",
+            "html",
+            "css",
+            "scss",
+            "go",
+            "r",
+            "graphql",
+            "swift",
+            "xml",
+            "yaml",
+            "json",
+            "lua",
+            "scheme",
+            "less",
+            "ini",
+            "coffee",
+            "litcoffee",
+            "js",
+            "cs",
+            "kt",
+            "md",
+            "sql",
+            "m",
+            "mm",
+            "pas",
+            "perl",
+            "ejs",
+            "pl",
+            "rb",
+            "rs",
+            "rust",
+            "sh",
+            "makefile",
+            "circ",
+            "readme",
+            "yml",
+            "sml",
+            "conf",
+            "txt",
+            "gitignore",
+            "in",
+            "cu",
+            "gemfile",
+            "scala",
+            "net",
+            "l",
+            "v",
+            "config",
+            "properties",
+            "log",
+            "htm",
+            "cnf",
+            "hex",
+            "bat",
+            "asm",
+            "bash",
+            "ts",
+            "tsx",
+            "sass",
+            "jsx",
+            "jsp",
+            "gitkeep",
+            "sv",
+            "hql",
+            "y",
+            "jj",
+            "pls",
+            "sol",
+            "ignore",
+            "ctrl",
+            "vue",
+            "tex",
+            "bib",
+            "cls",
+            "bst",
+            "toc",
+            "sty",
+            "g4",
+            "sy",
+            "ipynb",
+            "m",
+            "mm",
+            "groovy"
+    };
+
     @Override
     @Transactional(rollbackFor = Exception.class)
     public void add(Long projectId, AddCloneDetectionVO addCloneDetectionVO) {
@@ -155,18 +251,21 @@ public class CloneDetectionServiceImpl implements CloneDetectionService {
                 log.error("save clone detail to database failed, source: {}, target: {}", fullPath + clonePairDTO.getSourceFile(), fullPath + clonePairDTO.getTargetFile(), e);
             }
         }
+
+        // 取总文件信息，记录各项值
+        List<File> files = getFiles(fullPath + "/source/" + project.getProjectName());
         CloneDetectionResultProjectLevel cloneDetectionResultProjectLevel = new CloneDetectionResultProjectLevel();
         cloneDetectionResultProjectLevel.setCloneDetectionId(cloneDetectionId);
         cloneDetectionResultProjectLevel.setSimilarityFile(similarSourceLinesMap.size());
-        long totalFile = getFileCount(fullPath + "/source");
+        long totalFile = files.size();
         cloneDetectionResultProjectLevel.setSimilarityFilePercent(Math.min(((double)similarSourceLinesMap.size()) / totalFile, 1));
         cloneDetectionResultProjectLevel.setTotalFile(totalFile);
         cloneDetectionResultProjectLevel.setSimilarityLine(similarLines);
-        long totalLine = getLinesCount(fullPath + "/source/" + project.getProjectName());
+        long totalLine = getLinesCount(files);
         cloneDetectionResultProjectLevel.setSimilarityLinePercent(Math.min(((double)similarLines) / totalLine, 1));
         cloneDetectionResultProjectLevel.setTotalLine(totalLine);
         cloneDetectionResultProjectLevel.setSimilarityCapacity(similarCapacityCount);
-        long totalCapacity = getCapacityCount(fullPath + "/source/" + project.getProjectName());
+        long totalCapacity = getCapacityCount(files);
         cloneDetectionResultProjectLevel.setSimilarityCapacityPercent(Math.min(((double)similarCapacityCount) / totalCapacity, 1));
         cloneDetectionResultProjectLevel.setTotalCapacity(totalCapacity);
         cloneDetectionResultProjectLevel.setTargetProjectUrl(targetProjectUrl);
@@ -310,30 +409,39 @@ public class CloneDetectionServiceImpl implements CloneDetectionService {
     /**
      * 统计文件夹下文件个数
      */
-    private int getFileCount(String repoPath) {
-        String cnt = ShellUtil.execute("ls -lR " + repoPath + "| grep \"^-\" | wc -l");
-        return NumberUtils.toInt(cnt, 0);
+    private List<File> getFiles(String repoPath) {
+        Collection<File> files = FileUtils.listFiles(new File(repoPath), textFileSuffix, true);
+        return files.stream().filter(file -> !file.isHidden()).collect(Collectors.toList());
     }
 
     /**
      * 统计文件夹下文件行数
      */
-    private int getLinesCount(String repoPath) {
-        String cntResult = ShellUtil.execute("cd " + repoPath + " && git ls-files | xargs cat | wc -l");
-        String[] split = cntResult.trim().split("\n");
-        return NumberUtils.toInt(split[split.length - 1].trim(), Integer.MAX_VALUE);
+    private int getLinesCount(List<File> files) {
+        int count = 0;
+
+        for (File file : files) {
+            FileReader in;
+            try {
+                in = new FileReader(file);
+                LineNumberReader reader = new LineNumberReader(in);
+                reader.skip(Long.MAX_VALUE);
+                int lines = reader.getLineNumber();
+                reader.close();
+                count+= lines;
+            } catch (IOException e) {
+            }
+        }
+        return count;
     }
 
     /**
      * 统计文件夹大小
-     * @param repoPath
+     * @param files
      * @return
      */
-    private long getCapacityCount(String repoPath) {
-        // mac上为String cntResult = ShellUtil.execute("cd " + repoPath + " && du -s -k -I \"\\.git\" | awk '{print $1}'");
-        String cntResult = ShellUtil.execute("cd " + repoPath + " && du -s -k --exclude=\"\\.git\" | awk '{print $1}'");
-        String[] split = cntResult.trim().split("\n");
-        return NumberUtils.toInt(split[split.length - 1].trim(), Integer.MAX_VALUE) * 1024L;
+    private long getCapacityCount(List<File> files) {
+        return files.stream().mapToLong(File::length).reduce(0, Long::sum);
     }
 
 }

web/src/main/resources/application-user.yml

@@ -18,8 +18,8 @@ opensca-cli:
   #path: /data/ww/open-cli/opensca-cli
   path: /Users/youyongsheng/Downloads/opensca-cli_v1.0.9_Darwin_x86_64/opensca-cli
 nil:
-  jarPath: /Users/weiwang/IdeaProjects/NIL/build/libs/NIL-all.jar
-  mil: 6
-  mit: 50
+  jarPath: /Users/weiwang/IdeaProjects/quality_analysis/web/src/main/resources/nil/nil.jar
+  mil: 1
+  mit: 1
   filtrationThreshold: 10
   verificationThreshold: 70

web/src/main/resources/application.yml

@@ -10,9 +10,9 @@ spring:
     # mysql
     master:
       driver-class-name: com.mysql.cj.jdbc.Driver
-      url: jdbc:mysql://rm-bp13v5020p7828r5rso.mysql.rds.aliyuncs.com:3306/quality_analysis?useUnicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false
-      username: testeducoder
-      password: TEST@123
+      url: jdbc:mysql://localhost/quality_analysis?useUnicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false
+      username: root
+      password: 12345678
       type: com.alibaba.druid.pool.DruidDataSource
       druid:
         initial-size: 20