ADD file via upload

1.csv

@@ -0,0 +1,707 @@
+memory_limit,cpu_limit,lower_cpu_limit,request_limit
+1,2,0.3,0.2
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+2.5,2,0.5,0.8
+2,2,0.5,0.7
+1,2,0.3,0.4
+1,2,0.3,0.4
+2,2,0.3,0.3
+1,2,0.3,0.4
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.5
+1,2,0.3,0.4
+1,1,0.3,0.4
+2,2,0.5,0.5
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.3
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+2,2,0.4,0.8
+1,2,0.3,0.4
+2.5,2,0.4,0.7
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,1,0.3,0.4
+1,2,0.3,0.4
+3,2,0.5,1.5
+1,2,0.3,0.4
+4,2,0.5,1
+3,2,0.5,1.5
+2,2,0.3,0.5
+1,1,0.3,0.4
+1,2,0.3,0.4
+2,2,0.3,0.5
+1,2,0.4,0.3
+2.5,2,0.5,0.7
+2,2,0.4,0.5
+1,2,0.3,0.4
+3,2,0.4,0.7
+1,2,0.4,0.5
+2,2,0.5,1
+1,2,0.4,0.5
+1,2,0.3,0.4
+1,2,0.4,0.5
+1,2,0.3,0.4
+2,2,0.5,0.5
+1,2,0.4,0.5
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.4,0.5
+2,2,0.4,0.5
+2,2,0.4,0.5
+8,4,0.3,0.4
+1,2,0.4,0.3
+1,2,0.3,0.4
+1,2,0.4,0.5
+2,2,0.4,0.5
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,2,0.4,0.5
+1,2,0.3,0.4
+3,3,0.5,1.5
+1,2,0.4,0.3
+1,2,0.4,0.5
+1,2,0.4,0.5
+1,2,0.4,0.5
+1,2,0.4,0.5
+1,2,0.4,0.5
+1,2,0.4,0.3
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,2,0.4,0.5
+2,2,0.4,0.5
+1,2,0.3,0.3
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.4,0.3
+1,2,0.4,0.5
+1,2,0.3,0.5
+1,2,0.3,0.4
+1,1,0.3,0.4
+1,2,0.4,0.5
+1,2,0.3,0.4
+1,2,0.3,0.2
+1,1,0.3,0.4
+2,2,0.4,0.5
+1,2,0.3,0.4
+1,2,0.4,0.3
+1,2,0.4,0.5
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.4,0.5
+1,2,0.4,0.5
+2,2,0.3,0.4
+10,2,0.5,1
+2,2,0.4,0.7
+1,1,0.3,0.4
+1,2,0.4,0.3
+1,1,0.3,0.4
+1,1,0.3,0.4
+4,3,0.5,1
+1,1,0.3,0.4
+2,2,0.4,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.5,0.5
+1,1,0.3,0.4
+2,2,0.4,0.6
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.4,0.5
+1,1,0.3,0.4
+1,2,0.4,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+3,2,0.9,1
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+6.4,3,2.5,5.9
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.4,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+4,2,0.4,1.7
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.5,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.3,0.4
+2,2,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.4,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.4,0.5
+2,2,0.4,0.5
+1,1,0.3,0.4
+3,2,0.9,1
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.4,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.4,0.5
+2,2,0.4,0.5
+2,2,0.4,0.5
+2,2,0.4,0.5
+6.4,2,0.9,1.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.4,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+3,2,0.4,0.7
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+4,2,0.3,0.5
+1,1,0.3,0.4
+2,2,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+4,2,0.3,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+3,2,0.9,1
+2,1,0.3,0.4
+8,4,0.5,1
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.3,0.3
+2,2,0.3,0.3
+2,2,0.4,0.5
+1,1,0.3,0.4
+8,4,0.5,1
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.2,0.2
+1,1,0.3,0.4
+2,2,0.4,0.5
+2,2,0.4,0.5
+2,2,0.4,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+3,1,0.4,2
+16,8,4,8
+1,1,0.3,0.4
+1,1,0.3,0.4
+10,2,0.5,2
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+5,4,1,1
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.4,0.3
+1,2,0.4,0.3
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.2,0.2
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+0.5,1,0.3,0.4
+0.5,1,0.3,0.4
+0.5,1,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+0.5,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+4,2,0.4,1.7
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.4,0.6
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.4,0.5
+3,2,0.9,1
+1,1,0.3,0.4
+1,2,0.4,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.4,0.5
+1,2,0.4,0.5
+1,1,0.3,0.4
+0.5,0.5,0.3,0.3
+2,1,0.5,1
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.3,0.3
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,2,0.3,0.4
+0.5,1,0.2,0.2
+1,2,0.3,0.4
+1,1,0.3,0.4
+2,2,0.3,0.3
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.5,0.7
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.4,0.5
+4,2,1,2
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,2,0.4,0.5
+2,2,0.4,0.8
+1,1,0.3,0.4
+4,2,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+0.5,1,0.2,0.2
+0.5,1,0.2,0.2
+1,1,0.3,0.4
+2,2,0.3,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+3,2,0.9,1
+1,2,0.3,0.4
+1,2,0.3,0.4
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.4,0.5
+2,1,0.3,0.4
+1,2,0.3,0.4
+1,1,0.3,0.4
+2.5,2,0.5,0.8
+1,1,0.3,0.4
+2,2,1,1
+2,2,0.3,0.3
+2,2,0.3,0.3
+1,1,0.3,0.4
+2,2,0.4,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+0.5,1,0.2,0.2
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,1,1
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.4,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+0.5,1,0.2,0.2
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.5,0.5
+2,2,0.5,0.5
+1,1,0.3,0.4
+1,2,0.3,0.3
+1,1,0.3,0.4
+3,1,0.4,2
+1,1,0.3,0.4
+4,2,1,1
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+0.5,1,0.2,0.2
+1,2,0.3,0.4
+4,2,2,4
+2,2,0.4,0.5
+4,2,2,4
+1,2,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.4,0.3
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.5,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.4,0.3
+1,1,0.3,0.4
+1,2,0.4,0.3
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.4,0.5
+1,1,0.3,0.4
+5,4,1,1
+2,2,0.3,0.3
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.4,0.5
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,1,0.3,0.4
+4,2,1,2
+1,1,0.3,0.4
+1,2,0.4,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,1,0.3,0.4
+0.5,1,0.2,0.2
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,1,0.3,0.4
+4,2,1,2
+2,2,0.3,0.3
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.3,0.3
+0.5,1,0.2,0.2
+1,1,0.3,0.4
+1,1,0.3,0.4
+0.5,1,0.2,0.2
+0.5,1,0.2,0.2
+1,1,0.3,0.4
+0.5,1,0.2,0.2
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,1,0.3,1
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+0.5,1,0.2,0.2
+2,2,0.3,0.4
+2,2,0.5,0.5
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+3,2,0.9,1
+1,1,0.3,0.4
+1,2,0.4,0.5
+2,2,0.3,0.3
+1,2,0.4,0.3
+3,2,0.3,0.4
+4,2,1,1
+6,2,0.9,0.4
+1,2,0.3,0.4
+1,1,0.3,0.4
+1,2,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+1,1,0.3,0.4
+2,2,0.4,0.5
+4.9,2,2,4.9
+1,1,0.3,0.4
+2,2,0.3,0.3
+1,1,0.3,0.4
+1,1,0.3,0.4
+4,2,0.5,0.5
+1,1,0.3,0.4
+4,2,2,4
+1,2,0.3,0.2
+1,1,0.2,0.2
+1,1,0.2,0.2
+2,2,0.3,0.3
+1,1,0.2,0.2
+1,2,0.3,0.2
+1,1,0.2,0.2
+1,1,0.2,0.2
+1,1,0.2,0.2
+1,1,0.2,0.2
+1,1,0.2,0.2
+0.5,1,0.2,0.1
+0.5,1,0.2,0.1
+1,1,0.2,0.2
+0.5,1,0.2,0.1
+1,1,0.2,0.2
+0.5,1,0.2,0.1
+1,1,0.2,0.2
+1,2,0.3,0.2
+0.5,1,0.2,0.1
+2,2,0.3,0.3
+0.5,1,0.2,0.1
+0.5,1,0.2,0.1
+3,2,0.3,0.4
+0.5,1,0.2,0.1
+0.5,1,0.2,0.1
+2,2,0.3,0.3
+1,1,0.2,0.2
+1,1,0.2,0.2
+1,2,0.3,0.2
+0.5,1,0.2,0.1
+0.5,1,0.2,0.1
+1,1,0.2,0.2
+0.5,1,0.2,0.1
+0.5,1,0.2,0.1
+1,1,0.2,0.2
+0.5,1,0.2,0.1
+1,1,0.2,0.2
+0.5,1,0.2,0.1
+0.5,1,0.2,0.1
+1,1,0.2,0.2
+4,2,0.3,0.5
+1,1,0.2,0.2

src/main/kotlin/jp/ac/osaka_u/sdl/nil/NILConfig.kt

@@ -12,7 +12,7 @@ data class NILConfig(
     val verificationThreshold: Int = 70,
     val outputFileName: String = "result.csv",
     val threads: Int = 0,
-    val lang: Language = Language.JAVA,
+    val lang: Language = Language.COMMON,
     val isForBigCloneEval: Boolean = false,
     val isForMutationInjectionFramework: Boolean = false,
 )
@@ -27,7 +27,7 @@ fun parseArgs(args: Array<String>): NILConfig {
     var verificationThreshold = 70
     var outputFileName: String? = null
     var threads = 0
-    var lang = Language.JAVA
+    var lang = Language.COMMON
     var isForBigCloneEval = false
     var isForMutationInjectionFramework = false
 
@@ -81,11 +81,11 @@ fun String.toLangOrException(): Language =
     when (this.toLowerCase()) {
         "java" -> Language.JAVA
         "c" -> Language.C
-        "cpp" -> Language.CPP
+        "cpp", "c++" -> Language.CPP
-        "cs", "csharp" -> Language.CS
+        "cs","c#", "csharp" -> Language.CS
-        "py", "python" -> Language.PYTHON
+        "py","python2", "python3", "python" -> Language.PYTHON
         "kt", "kotlin" -> Language.KOTLIN
-        else -> throw InvalidOptionException("Language $this is invalid.")
+        else -> Language.COMMON
     }
 
 class InvalidOptionException(private val option: String) : RuntimeException() {
@@ -107,5 +107,5 @@ class InvalidOptionException(private val option: String) : RuntimeException() {
 }
 
 enum class Language {
-    JAVA, CPP, C, CS, PYTHON, KOTLIN
+    COMMON, JAVA, CPP, C, CS, PYTHON, KOTLIN
 }

src/main/kotlin/jp/ac/osaka_u/sdl/nil/NILMain.kt

@@ -56,6 +56,7 @@ class NILMain(private val config: NILConfig) {
                         config.verificationThreshold
                     )
                 if (tokenSequences.size - startIndex - 1 > 0) {
+                    // 对当前位置后的每一个token sequence，执行克隆检测的步骤，从索引中定位->过滤->验证
                     Flowable.range(startIndex + 1, tokenSequences.size - startIndex - 1)
                             .parallelIfSpecified(config.threads)
                             .runOn(Schedulers.computation())

src/main/kotlin/jp/ac/osaka_u/sdl/nil/entity/InvertedIndex.kt

@@ -16,7 +16,8 @@ class InvertedIndex private constructor() {
             val invertedIndex = InvertedIndex()
             val endIndex = min(startIndex + partitionSize, tokenSequences.size)
             for (index in startIndex until endIndex) {
-                val nGrams = tokenSequences[index].toNgrams(gramSize)
+                val nGrams = tokenSequences[index].toNgrams(gramSize)  // 当前token sequence对应的ngrams
+                // 索引中添加元素：<ngram, ngramInfo(出现在哪个序列中,序列中一共有多少ngram)>
                 nGrams.forEach { invertedIndex.hashTable.getOrPut(it) { mutableListOf() }.add(index to nGrams.size) }
             }
             return invertedIndex

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/cloneDetection/CustomCloneDetection.kt

@@ -16,6 +16,7 @@ class CustomCloneDetection(
     private val th: Int
 ) : CloneDetection {
     override fun execCustom(id: Id): Flowable<TokenSequencesSimInfo> {
+        // 待对比的token sequence
         val nGrams = tokenSequences[id].toNgrams(gramSize)
         return locatingPhase.locate(nGrams, id)
             .filter { filteringPhase.filter(nGrams.size, it) }

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/cloneDetection/Filtration.kt

@@ -3,5 +3,9 @@ package jp.ac.osaka_u.sdl.nil.usecase.cloneDetection
 import jp.ac.osaka_u.sdl.nil.entity.NGramInfo
 
 interface Filtration {
+    /**
+     * @param nGramSize 当前判断序列的长度
+     * @param cloneCandidate 可疑克隆源：<序列信息，交集大小>
+     */
     fun filter(nGramSize: Int, cloneCandidate: Map.Entry<NGramInfo, Int>): Boolean
 }

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/cloneDetection/NGramBasedFiltration.kt

@@ -3,7 +3,7 @@ package jp.ac.osaka_u.sdl.nil.usecase.cloneDetection
 import jp.ac.osaka_u.sdl.nil.entity.NGramInfo
 
 class NGramBasedFiltration(private val threshold: Int) : Filtration, NGramSimilarity {
-    // 计算交集ngram的数量，取大于阈值的
+    // 计算交集ngram的数量，计算相对两个对比序列的长度，取相似度大于阈值的
     override fun filter(nGramSize: Int, cloneCandidate: Map.Entry<NGramInfo, Int>): Boolean =
         calcSimilarity(nGramSize, cloneCandidate.key.size, cloneCandidate.value) >= threshold
 }

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/cloneDetection/NGramBasedLocation.kt

@@ -10,8 +10,8 @@ class NGramBasedLocation(private val invertedIndex: InvertedIndex) : Location {
     override fun locate(nGrams: NGrams, index: Int): Flowable<Map.Entry<NGramInfo, Int>> =
         nGrams.flatMap { invertedIndex[it] }  // ngram的每一个在倒排索引中出现的位置的列表
             .groupingBy { it }
-            .eachCount() // 每个tokenSequences中出现了多少个ngram与当前nGrams的交集
+            .eachCount() // <每个序列，交集ngram数量>
             .asSequence()
             .toFlowable()
-            .filter { (nGramInfo, _) -> index > nGramInfo.id }  // 代表tokenSequence的NGramInfo，交集的长度
+            .filter { (nGramInfo, _) -> index > nGramInfo.id }  // 取小于当前id的序列
 }

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/preprocess/PreprocessFactory.kt

@@ -2,6 +2,7 @@ package jp.ac.osaka_u.sdl.nil.usecase.preprocess
 
 import jp.ac.osaka_u.sdl.nil.Language
 import jp.ac.osaka_u.sdl.nil.NILConfig
+import jp.ac.osaka_u.sdl.nil.usecase.preprocess.common.CommonPreprocess
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.cpp.CPPPreprocess
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.cpp.CPreprocess
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.cs.CSharpPreprocess
@@ -19,6 +20,7 @@ class PreprocessFactory {
                 Language.CS -> CSharpPreprocess(config)
                 Language.PYTHON -> PythonPreprocess(config)
                 Language.KOTLIN -> KotlinPreprocess(config)
+                else -> {CommonPreprocess(config)}
             }
     }
 }

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/preprocess/common/CommonPreprocess.kt

@@ -0,0 +1,114 @@
+package jp.ac.osaka_u.sdl.nil.usecase.preprocess.common
+
+import io.reactivex.rxjava3.core.Flowable
+import io.reactivex.rxjava3.kotlin.toFlowable
+import jp.ac.osaka_u.sdl.nil.NILConfig
+import jp.ac.osaka_u.sdl.nil.entity.CodeBlock
+import jp.ac.osaka_u.sdl.nil.usecase.preprocess.Preprocess
+import java.io.File
+
+class CommonPreprocess(private val config: NILConfig) : Preprocess(config.threads) {
+    private val textFileSuffix = setOf(
+        "py",
+        "h",
+        "c",
+        "cpp",
+        "cc",
+        "java",
+        "php",
+        "html",
+        "css",
+        "scss",
+        "go",
+        "r",
+        "graphql",
+        "swift",
+        "xml",
+        "yaml",
+        "json",
+        "lua",
+        "scheme",
+        "less",
+        "ini",
+        "coffee",
+        "litcoffee",
+        "js",
+        "cs",
+        "kt",
+        "md",
+        "sql",
+        "m",
+        "mm",
+        "pas",
+        "perl",
+        "ejs",
+        "pl",
+        "rb",
+        "rs",
+        "rust",
+        "sh",
+        "makefile",
+        "circ",
+        "readme",
+        "yml",
+        "sml",
+        "conf",
+        "txt",
+        "gitignore",
+        "in",
+        "cu",
+        "gemfile",
+        "scala",
+        "net",
+        "l",
+        "v",
+        "config",
+        "properties",
+        "log",
+        "htm",
+        "cnf",
+        "hex",
+        "bat",
+        "asm",
+        "bash",
+        "ts",
+        "tsx",
+        "sass",
+        "jsx",
+        "jsp",
+        "gitkeep",
+        "sv",
+        "hql",
+        "y",
+        "jj",
+        "pls",
+        "sol",
+        "ignore",
+        "ctrl",
+        "vue",
+        "tex",
+        "bib",
+        "cls",
+        "bst",
+        "toc",
+        "sty",
+        "g4",
+        "sy",
+        "ipynb",
+        "m",
+        "mm",
+        "groovy"
+    )
+
+    override fun collectSourceFiles(dir: File): Flowable<File> =
+        dir.walk()
+            .filter { it.isFile }
+            .filterNot { it.absolutePath.contains("/.") }
+            .filter { textFileSuffix.contains(it.extension) }
+            .toFlowable()
+
+    override fun collectBlocks(srcFile: File): Flowable<CodeBlock> =
+        Flowable.just(srcFile)
+            .flatMap(CommonTransformer(config)::extractBlocks)
+
+}

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/preprocess/common/CommonTransformer.kt

@@ -0,0 +1,57 @@
+package jp.ac.osaka_u.sdl.nil.usecase.preprocess.common
+
+import io.reactivex.rxjava3.core.BackpressureStrategy
+import io.reactivex.rxjava3.core.Flowable
+import io.reactivex.rxjava3.core.Observable
+import jp.ac.osaka_u.sdl.nil.NILConfig
+import jp.ac.osaka_u.sdl.nil.entity.CodeBlock
+import jp.ac.osaka_u.sdl.nil.usecase.preprocess.SymbolSeparator
+import java.io.File
+import java.io.FileReader
+import java.io.StreamTokenizer
+import java.math.BigDecimal
+
+class CommonTransformer(private val config: NILConfig) {
+
+    fun extractBlocks(sourceFile: File): Flowable<CodeBlock> =
+        Observable.create<CodeBlock> { emitter ->
+            val fileName = sourceFile.canonicalPath
+            val tokens: MutableList<String> = ArrayList()
+            var endLine = 1
+
+            // 读取文件内容，整个文件作为一个token序列
+            val rd = FileReader(sourceFile)
+            rd.use {
+                val st = StreamTokenizer(rd)
+                st.parseNumbers()
+                st.eolIsSignificant(true)
+                var token = st.nextToken()
+                while (token != StreamTokenizer.TT_EOF) {
+                    token = st.nextToken()
+                    when (token) {
+                        StreamTokenizer.TT_NUMBER -> {
+                            val num = st.nval
+                            val bd = BigDecimal(num.toString())
+                            tokens.add(bd.stripTrailingZeros().toPlainString())
+                        }
+
+                        StreamTokenizer.TT_WORD -> {
+                            val word = st.sval
+                            tokens.add(word)
+                        }
+
+                        StreamTokenizer.TT_EOL -> endLine++
+
+                        else -> {}
+                    }
+                }
+            }
+
+            // 整个token序列加入
+            if (tokens.size >= config.minToken && endLine >= config.minLine) {
+                emitter.onNext(CodeBlock(fileName, 1, endLine, SymbolSeparator.separate(tokens)))
+            }
+
+            emitter.onComplete()
+        }.toFlowable(BackpressureStrategy.BUFFER)
+}

技术路线大模型编写提示词.txt

@@ -0,0 +1,16 @@
+该项技术重点解决课程知识图谱自动化生成问题，涵盖知识抽取、知识融合、知识推理、知识加工以及知识呈现等多项技术。
+a）知识抽取
+知识抽取是构建知识图谱的第一步，主要从各种类型的数据源中提取出实体（概念）、属性以及实体间的相互关系。
+b）知识融合
+在获得新知识后，需要对其进行整合，以消除矛盾和歧义。比如，某些实体可能有多种表达，某个特定称谓可能对应于不同的实体等。这个过程称为知识融合，其目的是将不同来源的知识进行整合，形成一个统一、一致的知识库。
+c）知识推理
+知识推理主要用于发现新的事实、关系、公理和规则等。知识推理可以通过多种方法实现，包括基于逻辑规则的推理、基于图结构的推理、基于分布式表示学习的推理、基于神经网络的推理以及混合推理等。
+具体来说，知识推理的基础任务主要包括知识补全、知识纠错和推理问答等。以知识补全为例，通过算法可以补全知识图谱中缺失的属性或者关系，提升知识图谱的完备性。比如，可以利用归纳推理的方法，根据以往的经验分析和先验知识构建概率模型，对推理假设进行验证或推测。另外，基于图结构的推理可以利用图算法来挖掘实体间的关系，进而发现新的知识。
+d）知识加工
+对于经过融合的新知识，还需要进行质量评估，才能将合格的部分加入到知识库中，以确保知识库的质量。
+这个过程可以包括人工参与甄别，也可以利用自动化算法进行筛选。新增数据后，可以进行知识推理、拓展现有知识，得到新知识。
+e）知识图谱展示
+知识图谱展示是将知识图谱中的实体、关系、属性等以图形化的方式展现出来，帮助用户更直观地理解和应用知识。
+
+结合这个评价，对上述内容进行扩展，用三段话进行描述，业务场景为基础业务能力实训保障系统：
+描述不能太口语化，技术路线应该是聚焦为啥用这个技术，有什么优势，解决什么场景问题，而不是介绍技术概念。

聊天记忆.html

@@ -0,0 +1,69 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>智能助手</title>
+    <script>
+        let conversationId = ""; // 初始化conversation_id
+    
+        // 定义API请求函数
+        async function sendRequest(type, inputMarkdown) {
+            // 构造请求体，包含conversation_id
+            const requestBody = {
+                query: inputMarkdown,
+                inputs: {"description": "生成杨辉三角", type: type},
+                response_mode: "blocking",
+                conversation_id: conversationId,
+                user: "user_id", // 请替换为实际的用户ID
+                files: []
+            };
+    
+            const response = await fetch('https://ai-data.educoder.net/v1/chat-messages', {
+                method: "POST",
+                headers: {
+                    'Content-Type': 'application/json',
+                    // 请替换为实际有效的认证令牌
+                    'Authorization': 'Bearer app-Z8HFAfducmnrr3zO9961jqgB'
+                },
+                body: JSON.stringify(requestBody)
+            });
+    
+            if (!response.ok) {
+                throw new Error(`HTTP error! Status: ${response.status}`);
+            }
+    
+            const data = await response.json();
+            if (data && data.answer) {
+                document.getElementById('outputMarkdown').value = data.answer;
+            } else {
+                document.getElementById('outputMarkdown').value = "未获取到有效回答";
+            }
+    
+            // 从响应中提取conversation_id并保存
+            if (data && data.conversation_id) {
+                conversationId = data.conversation_id;
+            }
+        }
+    
+        // 定义按钮点击事件
+        function onButtonClick(type) {
+            const inputMarkdown = document.getElementById('inputMarkdown').value;
+            sendRequest(type, inputMarkdown);
+        }
+    
+        function onSubmit() {
+            const inputMarkdown = document.getElementById('inputMarkdown').value;
+            sendRequest(2, inputMarkdown); // 假设type=2为普通问答
+        }
+    </script>
+</head>
+<body>
+    <h1>智能助手</h1>
+    <textarea id="inputMarkdown" rows="4" cols="50" placeholder="请输入您的问题..."></textarea><br>
+    <button onclick="onButtonClick(0)">智能审题</button>
+    <button onclick="onButtonClick(1)">代码纠错</button>
+    <button onclick="onSubmit()">提交问题</button>
+    <br>
+    <textarea id="outputMarkdown" rows="4" cols="50" readonly></textarea>
+</body>
+</html>