tokenize

1.csv

@@ -1,707 +0,0 @@
-memory_limit,cpu_limit,lower_cpu_limit,request_limit
-1,2,0.3,0.2
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-2.5,2,0.5,0.8
-2,2,0.5,0.7
-1,2,0.3,0.4
-1,2,0.3,0.4
-2,2,0.3,0.3
-1,2,0.3,0.4
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.5
-1,2,0.3,0.4
-1,1,0.3,0.4
-2,2,0.5,0.5
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.3
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-2,2,0.4,0.8
-1,2,0.3,0.4
-2.5,2,0.4,0.7
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,1,0.3,0.4
-1,2,0.3,0.4
-3,2,0.5,1.5
-1,2,0.3,0.4
-4,2,0.5,1
-3,2,0.5,1.5
-2,2,0.3,0.5
-1,1,0.3,0.4
-1,2,0.3,0.4
-2,2,0.3,0.5
-1,2,0.4,0.3
-2.5,2,0.5,0.7
-2,2,0.4,0.5
-1,2,0.3,0.4
-3,2,0.4,0.7
-1,2,0.4,0.5
-2,2,0.5,1
-1,2,0.4,0.5
-1,2,0.3,0.4
-1,2,0.4,0.5
-1,2,0.3,0.4
-2,2,0.5,0.5
-1,2,0.4,0.5
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.4,0.5
-2,2,0.4,0.5
-2,2,0.4,0.5
-8,4,0.3,0.4
-1,2,0.4,0.3
-1,2,0.3,0.4
-1,2,0.4,0.5
-2,2,0.4,0.5
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,2,0.4,0.5
-1,2,0.3,0.4
-3,3,0.5,1.5
-1,2,0.4,0.3
-1,2,0.4,0.5
-1,2,0.4,0.5
-1,2,0.4,0.5
-1,2,0.4,0.5
-1,2,0.4,0.5
-1,2,0.4,0.3
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,2,0.4,0.5
-2,2,0.4,0.5
-1,2,0.3,0.3
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.4,0.3
-1,2,0.4,0.5
-1,2,0.3,0.5
-1,2,0.3,0.4
-1,1,0.3,0.4
-1,2,0.4,0.5
-1,2,0.3,0.4
-1,2,0.3,0.2
-1,1,0.3,0.4
-2,2,0.4,0.5
-1,2,0.3,0.4
-1,2,0.4,0.3
-1,2,0.4,0.5
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.4,0.5
-1,2,0.4,0.5
-2,2,0.3,0.4
-10,2,0.5,1
-2,2,0.4,0.7
-1,1,0.3,0.4
-1,2,0.4,0.3
-1,1,0.3,0.4
-1,1,0.3,0.4
-4,3,0.5,1
-1,1,0.3,0.4
-2,2,0.4,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.5,0.5
-1,1,0.3,0.4
-2,2,0.4,0.6
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.4,0.5
-1,1,0.3,0.4
-1,2,0.4,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-3,2,0.9,1
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-6.4,3,2.5,5.9
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.4,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-4,2,0.4,1.7
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.5,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.3,0.4
-2,2,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.4,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.4,0.5
-2,2,0.4,0.5
-1,1,0.3,0.4
-3,2,0.9,1
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.4,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.4,0.5
-2,2,0.4,0.5
-2,2,0.4,0.5
-2,2,0.4,0.5
-6.4,2,0.9,1.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.4,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-3,2,0.4,0.7
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-4,2,0.3,0.5
-1,1,0.3,0.4
-2,2,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-4,2,0.3,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-3,2,0.9,1
-2,1,0.3,0.4
-8,4,0.5,1
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.3,0.3
-2,2,0.3,0.3
-2,2,0.4,0.5
-1,1,0.3,0.4
-8,4,0.5,1
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.2,0.2
-1,1,0.3,0.4
-2,2,0.4,0.5
-2,2,0.4,0.5
-2,2,0.4,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-3,1,0.4,2
-16,8,4,8
-1,1,0.3,0.4
-1,1,0.3,0.4
-10,2,0.5,2
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-5,4,1,1
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.4,0.3
-1,2,0.4,0.3
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.2,0.2
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-0.5,1,0.3,0.4
-0.5,1,0.3,0.4
-0.5,1,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-0.5,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-4,2,0.4,1.7
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.4,0.6
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.4,0.5
-3,2,0.9,1
-1,1,0.3,0.4
-1,2,0.4,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.4,0.5
-1,2,0.4,0.5
-1,1,0.3,0.4
-0.5,0.5,0.3,0.3
-2,1,0.5,1
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.3,0.3
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,2,0.3,0.4
-0.5,1,0.2,0.2
-1,2,0.3,0.4
-1,1,0.3,0.4
-2,2,0.3,0.3
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.5,0.7
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.4,0.5
-4,2,1,2
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,2,0.4,0.5
-2,2,0.4,0.8
-1,1,0.3,0.4
-4,2,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-0.5,1,0.2,0.2
-0.5,1,0.2,0.2
-1,1,0.3,0.4
-2,2,0.3,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-3,2,0.9,1
-1,2,0.3,0.4
-1,2,0.3,0.4
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.4,0.5
-2,1,0.3,0.4
-1,2,0.3,0.4
-1,1,0.3,0.4
-2.5,2,0.5,0.8
-1,1,0.3,0.4
-2,2,1,1
-2,2,0.3,0.3
-2,2,0.3,0.3
-1,1,0.3,0.4
-2,2,0.4,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-0.5,1,0.2,0.2
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,1,1
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.4,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-0.5,1,0.2,0.2
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.5,0.5
-2,2,0.5,0.5
-1,1,0.3,0.4
-1,2,0.3,0.3
-1,1,0.3,0.4
-3,1,0.4,2
-1,1,0.3,0.4
-4,2,1,1
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-0.5,1,0.2,0.2
-1,2,0.3,0.4
-4,2,2,4
-2,2,0.4,0.5
-4,2,2,4
-1,2,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.4,0.3
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.5,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.4,0.3
-1,1,0.3,0.4
-1,2,0.4,0.3
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.4,0.5
-1,1,0.3,0.4
-5,4,1,1
-2,2,0.3,0.3
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.4,0.5
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,1,0.3,0.4
-4,2,1,2
-1,1,0.3,0.4
-1,2,0.4,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,1,0.3,0.4
-0.5,1,0.2,0.2
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,1,0.3,0.4
-4,2,1,2
-2,2,0.3,0.3
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.3,0.3
-0.5,1,0.2,0.2
-1,1,0.3,0.4
-1,1,0.3,0.4
-0.5,1,0.2,0.2
-0.5,1,0.2,0.2
-1,1,0.3,0.4
-0.5,1,0.2,0.2
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,1,0.3,1
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-0.5,1,0.2,0.2
-2,2,0.3,0.4
-2,2,0.5,0.5
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-3,2,0.9,1
-1,1,0.3,0.4
-1,2,0.4,0.5
-2,2,0.3,0.3
-1,2,0.4,0.3
-3,2,0.3,0.4
-4,2,1,1
-6,2,0.9,0.4
-1,2,0.3,0.4
-1,1,0.3,0.4
-1,2,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-1,1,0.3,0.4
-2,2,0.4,0.5
-4.9,2,2,4.9
-1,1,0.3,0.4
-2,2,0.3,0.3
-1,1,0.3,0.4
-1,1,0.3,0.4
-4,2,0.5,0.5
-1,1,0.3,0.4
-4,2,2,4
-1,2,0.3,0.2
-1,1,0.2,0.2
-1,1,0.2,0.2
-2,2,0.3,0.3
-1,1,0.2,0.2
-1,2,0.3,0.2
-1,1,0.2,0.2
-1,1,0.2,0.2
-1,1,0.2,0.2
-1,1,0.2,0.2
-1,1,0.2,0.2
-0.5,1,0.2,0.1
-0.5,1,0.2,0.1
-1,1,0.2,0.2
-0.5,1,0.2,0.1
-1,1,0.2,0.2
-0.5,1,0.2,0.1
-1,1,0.2,0.2
-1,2,0.3,0.2
-0.5,1,0.2,0.1
-2,2,0.3,0.3
-0.5,1,0.2,0.1
-0.5,1,0.2,0.1
-3,2,0.3,0.4
-0.5,1,0.2,0.1
-0.5,1,0.2,0.1
-2,2,0.3,0.3
-1,1,0.2,0.2
-1,1,0.2,0.2
-1,2,0.3,0.2
-0.5,1,0.2,0.1
-0.5,1,0.2,0.1
-1,1,0.2,0.2
-0.5,1,0.2,0.1
-0.5,1,0.2,0.1
-1,1,0.2,0.2
-0.5,1,0.2,0.1
-1,1,0.2,0.2
-0.5,1,0.2,0.1
-0.5,1,0.2,0.1
-1,1,0.2,0.2
-4,2,0.3,0.5
-1,1,0.2,0.2

src/main/java/jp/ac/osaka_u/sdl/nil/parser/utils/TokenizeUtil.java

@@ -0,0 +1,67 @@
+package jp.ac.osaka_u.sdl.nil.parser.utils;
+
+import jp.ac.osaka_u.sdl.nil.parser.utils.entity.BlockMsg;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.StreamTokenizer;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+
+public class TokenizeUtil {
+    /**
+     * 对文件内容简单的解析为令牌，取到hashcode list
+     * @param file
+     * @return
+     */
+    public static BlockMsg simpleTokenize(File file){
+        FileReader rd = null;
+        try {
+            rd = new FileReader(file);
+            StreamTokenizer st = new StreamTokenizer(rd);
+            st.parseNumbers();
+            st.wordChars('_', '_');
+            st.eolIsSignificant(false);
+            st.ordinaryChars(0, ' ');
+            st.slashSlashComments(true);
+            st.slashStarComments(true);
+            int token = st.nextToken();
+            List<Integer> res = new ArrayList<>();
+            int line = 0;
+            while (token != StreamTokenizer.TT_EOF) {
+                token = st.nextToken();
+                switch (token) {
+                    case StreamTokenizer.TT_NUMBER:
+                        double num = st.nval;
+                        BigDecimal bd = new BigDecimal(String.valueOf(num));
+                        res.add(bd.stripTrailingZeros().toPlainString().hashCode());
+                        break;
+                    case StreamTokenizer.TT_WORD:
+                        String word = st.sval;
+                        res.add(word.hashCode());
+                        break;
+                    case StreamTokenizer.TT_EOL:
+                        line++;
+                        break;
+                    default:
+                        break;
+                }
+            }
+            BlockMsg blockMsg = new BlockMsg(1, line, res);
+            return blockMsg;
+        }catch (IOException e){
+            e.printStackTrace();
+        }finally {
+            if (rd!=null){
+                try {
+                    rd.close();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+        return null;
+    }
+}

src/main/java/jp/ac/osaka_u/sdl/nil/parser/utils/entity/BlockMsg.java

@@ -0,0 +1,50 @@
+package jp.ac.osaka_u.sdl.nil.parser.utils.entity;
+
+import java.util.List;
+
+public class BlockMsg {
+    private int beginLine;
+    private int endLine;
+    private List<Integer> tokens;
+
+    public BlockMsg() {}
+
+    public BlockMsg(int beginLine, int endLine, List<Integer> tokens) {
+        this.beginLine = beginLine;
+        this.endLine = endLine;
+        this.tokens = tokens;
+    }
+
+    public int getBeginLine() {
+        return beginLine;
+    }
+
+    public void setBeginLine(int beginLine) {
+        this.beginLine = beginLine;
+    }
+
+    public int getEndLine() {
+        return endLine;
+    }
+
+    public void setEndLine(int endLine) {
+        this.endLine = endLine;
+    }
+
+    public List<Integer> getTokens() {
+        return tokens;
+    }
+
+    public void setTokens(List<Integer> tokens) {
+        this.tokens = tokens;
+    }
+
+    @Override
+    public String toString() {
+        return "BlockMsg{" +
+                "beginLine=" + beginLine +
+                ", endLine=" + endLine +
+                ", tokens=" + tokens +
+                '}';
+    }
+}

src/main/kotlin/jp/ac/osaka_u/sdl/nil/NILConfig.kt

@@ -12,7 +12,7 @@ data class NILConfig(
     val verificationThreshold: Int = 70,
     val outputFileName: String = "result.csv",
     val threads: Int = 0,
-    val lang: Language = Language.COMMON,
+    val lang: Language = Language.JAVA,
     val isForBigCloneEval: Boolean = false,
     val isForMutationInjectionFramework: Boolean = false,
 )
@@ -27,7 +27,7 @@ fun parseArgs(args: Array<String>): NILConfig {
     var verificationThreshold = 70
     var outputFileName: String? = null
     var threads = 0
-    var lang = Language.COMMON
+    var lang = Language.JAVA
     var isForBigCloneEval = false
     var isForMutationInjectionFramework = false
 
@@ -81,11 +81,12 @@ fun String.toLangOrException(): Language =
     when (this.toLowerCase()) {
         "java" -> Language.JAVA
         "c" -> Language.C
-        "cpp", "c++" -> Language.CPP
-        "cs","c#", "csharp" -> Language.CS
-        "py","python2", "python3", "python" -> Language.PYTHON
+        "cpp" -> Language.CPP
+        "cs", "csharp" -> Language.CS
+        "py", "python" -> Language.PYTHON
         "kt", "kotlin" -> Language.KOTLIN
-        else -> Language.COMMON
+        "cur", "currency" -> Language.CURRENCY
+        else -> throw InvalidOptionException("Language $this is invalid.")
     }
 
 class InvalidOptionException(private val option: String) : RuntimeException() {
@@ -107,5 +108,5 @@ class InvalidOptionException(private val option: String) : RuntimeException() {
 }
 
 enum class Language {
-    COMMON, JAVA, CPP, C, CS, PYTHON, KOTLIN
+    JAVA, CPP, C, CS, PYTHON, KOTLIN, CURRENCY
 }

src/main/kotlin/jp/ac/osaka_u/sdl/nil/NILMain.kt

@@ -56,7 +56,6 @@ class NILMain(private val config: NILConfig) {
                         config.verificationThreshold
                     )
                 if (tokenSequences.size - startIndex - 1 > 0) {
-                    // 对当前位置后的每一个token sequence，执行克隆检测的步骤，从索引中定位->过滤->验证
                     Flowable.range(startIndex + 1, tokenSequences.size - startIndex - 1)
                             .parallelIfSpecified(config.threads)
                             .runOn(Schedulers.computation())

src/main/kotlin/jp/ac/osaka_u/sdl/nil/entity/InvertedIndex.kt

@@ -16,8 +16,7 @@ class InvertedIndex private constructor() {
             val invertedIndex = InvertedIndex()
             val endIndex = min(startIndex + partitionSize, tokenSequences.size)
             for (index in startIndex until endIndex) {
-                val nGrams = tokenSequences[index].toNgrams(gramSize)  // 当前token sequence对应的ngrams
-                // 索引中添加元素：<ngram, ngramInfo(出现在哪个序列中,序列中一共有多少ngram)>
+                val nGrams = tokenSequences[index].toNgrams(gramSize)
                 nGrams.forEach { invertedIndex.hashTable.getOrPut(it) { mutableListOf() }.add(index to nGrams.size) }
             }
             return invertedIndex

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/cloneDetection/CustomCloneDetection.kt

@@ -16,7 +16,6 @@ class CustomCloneDetection(
     private val th: Int
 ) : CloneDetection {
     override fun execCustom(id: Id): Flowable<TokenSequencesSimInfo> {
-        // 待对比的token sequence
         val nGrams = tokenSequences[id].toNgrams(gramSize)
         return locatingPhase.locate(nGrams, id)
             .filter { filteringPhase.filter(nGrams.size, it) }

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/cloneDetection/Filtration.kt

@@ -3,9 +3,5 @@ package jp.ac.osaka_u.sdl.nil.usecase.cloneDetection
 import jp.ac.osaka_u.sdl.nil.entity.NGramInfo
 
 interface Filtration {
-    /**
-     * @param nGramSize 当前判断序列的长度
-     * @param cloneCandidate 可疑克隆源：<序列信息，交集大小>
-     */
     fun filter(nGramSize: Int, cloneCandidate: Map.Entry<NGramInfo, Int>): Boolean
 }

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/cloneDetection/NGramBasedFiltration.kt

@@ -3,7 +3,7 @@ package jp.ac.osaka_u.sdl.nil.usecase.cloneDetection
 import jp.ac.osaka_u.sdl.nil.entity.NGramInfo
 
 class NGramBasedFiltration(private val threshold: Int) : Filtration, NGramSimilarity {
-    // 计算交集ngram的数量，计算相对两个对比序列的长度，取相似度大于阈值的
+    // 计算交集ngram的数量，取大于阈值的
     override fun filter(nGramSize: Int, cloneCandidate: Map.Entry<NGramInfo, Int>): Boolean =
         calcSimilarity(nGramSize, cloneCandidate.key.size, cloneCandidate.value) >= threshold
 }

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/cloneDetection/NGramBasedLocation.kt

@@ -10,8 +10,8 @@ class NGramBasedLocation(private val invertedIndex: InvertedIndex) : Location {
     override fun locate(nGrams: NGrams, index: Int): Flowable<Map.Entry<NGramInfo, Int>> =
         nGrams.flatMap { invertedIndex[it] }  // ngram的每一个在倒排索引中出现的位置的列表
             .groupingBy { it }
-            .eachCount() // <每个序列，交集ngram数量>
+            .eachCount() // 每个tokenSequences中出现了多少个ngram与当前nGrams的交集
             .asSequence()
             .toFlowable()
-            .filter { (nGramInfo, _) -> index > nGramInfo.id }  // 取小于当前id的序列
+            .filter { (nGramInfo, _) -> index > nGramInfo.id }  // 代表tokenSequence的NGramInfo，交集的长度
 }

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/preprocess/PreprocessFactory.kt

@@ -2,10 +2,10 @@ package jp.ac.osaka_u.sdl.nil.usecase.preprocess
 
 import jp.ac.osaka_u.sdl.nil.Language
 import jp.ac.osaka_u.sdl.nil.NILConfig
-import jp.ac.osaka_u.sdl.nil.usecase.preprocess.common.CommonPreprocess
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.cpp.CPPPreprocess
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.cpp.CPreprocess
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.cs.CSharpPreprocess
+import jp.ac.osaka_u.sdl.nil.usecase.preprocess.currency.CurrencyPreprocess
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.java.JavaPreprocess
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.kotlin.KotlinPreprocess
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.python.PythonPreprocess
@@ -20,7 +20,7 @@ class PreprocessFactory {
                 Language.CS -> CSharpPreprocess(config)
                 Language.PYTHON -> PythonPreprocess(config)
                 Language.KOTLIN -> KotlinPreprocess(config)
-                else -> {CommonPreprocess(config)}
+                Language.CURRENCY -> CurrencyPreprocess(config)
             }
     }
 }

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/preprocess/common/CommonPreprocess.kt

@@ -1,114 +0,0 @@
-package jp.ac.osaka_u.sdl.nil.usecase.preprocess.common
-
-import io.reactivex.rxjava3.core.Flowable
-import io.reactivex.rxjava3.kotlin.toFlowable
-import jp.ac.osaka_u.sdl.nil.NILConfig
-import jp.ac.osaka_u.sdl.nil.entity.CodeBlock
-import jp.ac.osaka_u.sdl.nil.usecase.preprocess.Preprocess
-import java.io.File
-
-class CommonPreprocess(private val config: NILConfig) : Preprocess(config.threads) {
-    private val textFileSuffix = setOf(
-        "py",
-        "h",
-        "c",
-        "cpp",
-        "cc",
-        "java",
-        "php",
-        "html",
-        "css",
-        "scss",
-        "go",
-        "r",
-        "graphql",
-        "swift",
-        "xml",
-        "yaml",
-        "json",
-        "lua",
-        "scheme",
-        "less",
-        "ini",
-        "coffee",
-        "litcoffee",
-        "js",
-        "cs",
-        "kt",
-        "md",
-        "sql",
-        "m",
-        "mm",
-        "pas",
-        "perl",
-        "ejs",
-        "pl",
-        "rb",
-        "rs",
-        "rust",
-        "sh",
-        "makefile",
-        "circ",
-        "readme",
-        "yml",
-        "sml",
-        "conf",
-        "txt",
-        "gitignore",
-        "in",
-        "cu",
-        "gemfile",
-        "scala",
-        "net",
-        "l",
-        "v",
-        "config",
-        "properties",
-        "log",
-        "htm",
-        "cnf",
-        "hex",
-        "bat",
-        "asm",
-        "bash",
-        "ts",
-        "tsx",
-        "sass",
-        "jsx",
-        "jsp",
-        "gitkeep",
-        "sv",
-        "hql",
-        "y",
-        "jj",
-        "pls",
-        "sol",
-        "ignore",
-        "ctrl",
-        "vue",
-        "tex",
-        "bib",
-        "cls",
-        "bst",
-        "toc",
-        "sty",
-        "g4",
-        "sy",
-        "ipynb",
-        "m",
-        "mm",
-        "groovy"
-    )
-
-    override fun collectSourceFiles(dir: File): Flowable<File> =
-        dir.walk()
-            .filter { it.isFile }
-            .filterNot { it.absolutePath.contains("/.") }
-            .filter { textFileSuffix.contains(it.extension) }
-            .toFlowable()
-
-    override fun collectBlocks(srcFile: File): Flowable<CodeBlock> =
-        Flowable.just(srcFile)
-            .flatMap(CommonTransformer(config)::extractBlocks)
-
-}

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/preprocess/common/CommonTransformer.kt

@@ -1,57 +0,0 @@
-package jp.ac.osaka_u.sdl.nil.usecase.preprocess.common
-
-import io.reactivex.rxjava3.core.BackpressureStrategy
-import io.reactivex.rxjava3.core.Flowable
-import io.reactivex.rxjava3.core.Observable
-import jp.ac.osaka_u.sdl.nil.NILConfig
-import jp.ac.osaka_u.sdl.nil.entity.CodeBlock
-import jp.ac.osaka_u.sdl.nil.usecase.preprocess.SymbolSeparator
-import java.io.File
-import java.io.FileReader
-import java.io.StreamTokenizer
-import java.math.BigDecimal
-
-class CommonTransformer(private val config: NILConfig) {
-
-    fun extractBlocks(sourceFile: File): Flowable<CodeBlock> =
-        Observable.create<CodeBlock> { emitter ->
-            val fileName = sourceFile.canonicalPath
-            val tokens: MutableList<String> = ArrayList()
-            var endLine = 1
-
-            // 读取文件内容，整个文件作为一个token序列
-            val rd = FileReader(sourceFile)
-            rd.use {
-                val st = StreamTokenizer(rd)
-                st.parseNumbers()
-                st.eolIsSignificant(true)
-                var token = st.nextToken()
-                while (token != StreamTokenizer.TT_EOF) {
-                    token = st.nextToken()
-                    when (token) {
-                        StreamTokenizer.TT_NUMBER -> {
-                            val num = st.nval
-                            val bd = BigDecimal(num.toString())
-                            tokens.add(bd.stripTrailingZeros().toPlainString())
-                        }
-
-                        StreamTokenizer.TT_WORD -> {
-                            val word = st.sval
-                            tokens.add(word)
-                        }
-
-                        StreamTokenizer.TT_EOL -> endLine++
-
-                        else -> {}
-                    }
-                }
-            }
-
-            // 整个token序列加入
-            if (tokens.size >= config.minToken && endLine >= config.minLine) {
-                emitter.onNext(CodeBlock(fileName, 1, endLine, SymbolSeparator.separate(tokens)))
-            }
-
-            emitter.onComplete()
-        }.toFlowable(BackpressureStrategy.BUFFER)
-}

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/preprocess/currency/CurrencyPreprocess.kt

@@ -0,0 +1,19 @@
+package jp.ac.osaka_u.sdl.nil.usecase.preprocess.currency
+
+import io.reactivex.rxjava3.core.Flowable
+import io.reactivex.rxjava3.kotlin.toFlowable
+import jp.ac.osaka_u.sdl.nil.NILConfig
+import jp.ac.osaka_u.sdl.nil.entity.CodeBlock
+import jp.ac.osaka_u.sdl.nil.usecase.preprocess.Preprocess
+import java.io.File
+
+class CurrencyPreprocess(private val config: NILConfig) : Preprocess(config.threads) {
+    override fun collectSourceFiles(dir: File): Flowable<File> =
+        dir.walk()
+            .filter { it.isFile }
+            .toFlowable()
+
+    override fun collectBlocks(srcFile: File): Flowable<CodeBlock> =
+        Flowable.just(srcFile)
+            .flatMap(CurrencyTransformer(config)::extractBlocks)
+}

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/preprocess/currency/CurrencyTransformer.kt

@@ -0,0 +1,23 @@
+package jp.ac.osaka_u.sdl.nil.usecase.preprocess.currency
+
+import io.reactivex.rxjava3.core.BackpressureStrategy
+import io.reactivex.rxjava3.core.Flowable
+import io.reactivex.rxjava3.core.Observable
+import jp.ac.osaka_u.sdl.nil.NILConfig
+import jp.ac.osaka_u.sdl.nil.entity.CodeBlock
+import jp.ac.osaka_u.sdl.nil.parser.utils.TokenizeUtil
+import jp.ac.osaka_u.sdl.nil.parser.utils.entity.BlockMsg
+
+import java.io.File
+
+class CurrencyTransformer(private val config: NILConfig) {
+    fun extractBlocks(sourceFile: File): Flowable<CodeBlock> =
+        Observable.create<CodeBlock> { emitter ->
+            val blockMsg: BlockMsg = TokenizeUtil.simpleTokenize(sourceFile);
+            val tokens: List<Int> = blockMsg.tokens
+            if (tokens.size >= config.minToken) {
+                emitter.onNext(CodeBlock(sourceFile.canonicalPath, blockMsg.beginLine, blockMsg.endLine,tokens))
+            }
+            emitter.onComplete()
+        }.toFlowable(BackpressureStrategy.BUFFER)
+}

src/test/kotlin/jp/ac/osaka_u/sdl/nil/NILMainTest.kt

@@ -71,6 +71,18 @@ internal class NILMainTest {
         outputFile.delete()
     }
 
+    @Test
+    fun testCur() {
+        val config =
+            parseArgs(arrayOf("-s", "./src/test/resources/examples", "-bce", "-t", "1", "-p", "1", "-l", "cur"))
+        NILMain(config).run()
+
+        val outputFile = File(config.outputFileName)
+        println(outputFile.readText().lines().sorted())
+
+        outputFile.delete()
+    }
+
     @Test
     fun testKotlin() {
         val config =

src/test/resources/examples/FizzBuzzT.py

@@ -0,0 +1,24 @@
+def main1():
+    # Comment
+    for i in range(1, 101):
+        if i % 15 == 0:
+            print("FizzBuzzT")
+        elif i % 3 == 0:
+            print("Fizz")
+        elif i % 5 == 0:
+            print("Buzz")
+        else:
+            print("TIASO")
+
+def main2():
+    for i in range(1, 101):
+        # Comment
+        if i % 15 == 0:
+            print("FizzBuzzT")
+        elif i % 3 == 0:
+            print("Fizz")
+        elif i % 5 == 0:
+            print("Buzz")
+        else:
+            print(i)
+

技术路线大模型编写提示词.txt

@@ -1,16 +0,0 @@
-该项技术重点解决课程知识图谱自动化生成问题，涵盖知识抽取、知识融合、知识推理、知识加工以及知识呈现等多项技术。
-a）知识抽取
-知识抽取是构建知识图谱的第一步，主要从各种类型的数据源中提取出实体（概念）、属性以及实体间的相互关系。
-b）知识融合
-在获得新知识后，需要对其进行整合，以消除矛盾和歧义。比如，某些实体可能有多种表达，某个特定称谓可能对应于不同的实体等。这个过程称为知识融合，其目的是将不同来源的知识进行整合，形成一个统一、一致的知识库。
-c）知识推理
-知识推理主要用于发现新的事实、关系、公理和规则等。知识推理可以通过多种方法实现，包括基于逻辑规则的推理、基于图结构的推理、基于分布式表示学习的推理、基于神经网络的推理以及混合推理等。
-具体来说，知识推理的基础任务主要包括知识补全、知识纠错和推理问答等。以知识补全为例，通过算法可以补全知识图谱中缺失的属性或者关系，提升知识图谱的完备性。比如，可以利用归纳推理的方法，根据以往的经验分析和先验知识构建概率模型，对推理假设进行验证或推测。另外，基于图结构的推理可以利用图算法来挖掘实体间的关系，进而发现新的知识。
-d）知识加工
-对于经过融合的新知识，还需要进行质量评估，才能将合格的部分加入到知识库中，以确保知识库的质量。
-这个过程可以包括人工参与甄别，也可以利用自动化算法进行筛选。新增数据后，可以进行知识推理、拓展现有知识，得到新知识。
-e）知识图谱展示
-知识图谱展示是将知识图谱中的实体、关系、属性等以图形化的方式展现出来，帮助用户更直观地理解和应用知识。
-
-结合这个评价，对上述内容进行扩展，用三段话进行描述，业务场景为基础业务能力实训保障系统：
-描述不能太口语化，技术路线应该是聚焦为啥用这个技术，有什么优势，解决什么场景问题，而不是介绍技术概念。

聊天记忆.html

@@ -1,69 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <title>智能助手</title>
-    <script>
-        let conversationId = ""; // 初始化conversation_id
-    
-        // 定义API请求函数
-        async function sendRequest(type, inputMarkdown) {
-            // 构造请求体，包含conversation_id
-            const requestBody = {
-                query: inputMarkdown,
-                inputs: {"description": "生成杨辉三角", type: type},
-                response_mode: "blocking",
-                conversation_id: conversationId,
-                user: "user_id", // 请替换为实际的用户ID
-                files: []
-            };
-    
-            const response = await fetch('https://ai-data.educoder.net/v1/chat-messages', {
-                method: "POST",
-                headers: {
-                    'Content-Type': 'application/json',
-                    // 请替换为实际有效的认证令牌
-                    'Authorization': 'Bearer app-Z8HFAfducmnrr3zO9961jqgB'
-                },
-                body: JSON.stringify(requestBody)
-            });
-    
-            if (!response.ok) {
-                throw new Error(`HTTP error! Status: ${response.status}`);
-            }
-    
-            const data = await response.json();
-            if (data && data.answer) {
-                document.getElementById('outputMarkdown').value = data.answer;
-            } else {
-                document.getElementById('outputMarkdown').value = "未获取到有效回答";
-            }
-    
-            // 从响应中提取conversation_id并保存
-            if (data && data.conversation_id) {
-                conversationId = data.conversation_id;
-            }
-        }
-    
-        // 定义按钮点击事件
-        function onButtonClick(type) {
-            const inputMarkdown = document.getElementById('inputMarkdown').value;
-            sendRequest(type, inputMarkdown);
-        }
-    
-        function onSubmit() {
-            const inputMarkdown = document.getElementById('inputMarkdown').value;
-            sendRequest(2, inputMarkdown); // 假设type=2为普通问答
-        }
-    </script>
-</head>
-<body>
-    <h1>智能助手</h1>
-    <textarea id="inputMarkdown" rows="4" cols="50" placeholder="请输入您的问题..."></textarea><br>
-    <button onclick="onButtonClick(0)">智能审题</button>
-    <button onclick="onButtonClick(1)">代码纠错</button>
-    <button onclick="onSubmit()">提交问题</button>
-    <br>
-    <textarea id="outputMarkdown" rows="4" cols="50" readonly></textarea>
-</body>
-</html>