tokenize

src/main/java/jp/ac/osaka_u/sdl/nil/parser/utils/TokenizeUtil.java

@@ -0,0 +1,67 @@
+package jp.ac.osaka_u.sdl.nil.parser.utils;
+
+import jp.ac.osaka_u.sdl.nil.parser.utils.entity.BlockMsg;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.StreamTokenizer;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+
+public class TokenizeUtil {
+    /**
+     * 对文件内容简单的解析为令牌，取到hashcode list
+     * @param file
+     * @return
+     */
+    public static BlockMsg simpleTokenize(File file){
+        FileReader rd = null;
+        try {
+            rd = new FileReader(file);
+            StreamTokenizer st = new StreamTokenizer(rd);
+            st.parseNumbers();
+            st.wordChars('_', '_');
+            st.eolIsSignificant(false);
+            st.ordinaryChars(0, ' ');
+            st.slashSlashComments(true);
+            st.slashStarComments(true);
+            int token = st.nextToken();
+            List<Integer> res = new ArrayList<>();
+            int line = 0;
+            while (token != StreamTokenizer.TT_EOF) {
+                token = st.nextToken();
+                switch (token) {
+                    case StreamTokenizer.TT_NUMBER:
+                        double num = st.nval;
+                        BigDecimal bd = new BigDecimal(String.valueOf(num));
+                        res.add(bd.stripTrailingZeros().toPlainString().hashCode());
+                        break;
+                    case StreamTokenizer.TT_WORD:
+                        String word = st.sval;
+                        res.add(word.hashCode());
+                        break;
+                    case StreamTokenizer.TT_EOL:
+                        line++;
+                        break;
+                    default:
+                        break;
+                }
+            }
+            BlockMsg blockMsg = new BlockMsg(1, line, res);
+            return blockMsg;
+        }catch (IOException e){
+            e.printStackTrace();
+        }finally {
+            if (rd!=null){
+                try {
+                    rd.close();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+        return null;
+    }
+}

src/main/java/jp/ac/osaka_u/sdl/nil/parser/utils/entity/BlockMsg.java

@@ -0,0 +1,50 @@
+package jp.ac.osaka_u.sdl.nil.parser.utils.entity;
+
+import java.util.List;
+
+public class BlockMsg {
+    private int beginLine;
+    private int endLine;
+    private List<Integer> tokens;
+
+    public BlockMsg() {}
+
+    public BlockMsg(int beginLine, int endLine, List<Integer> tokens) {
+        this.beginLine = beginLine;
+        this.endLine = endLine;
+        this.tokens = tokens;
+    }
+
+    public int getBeginLine() {
+        return beginLine;
+    }
+
+    public void setBeginLine(int beginLine) {
+        this.beginLine = beginLine;
+    }
+
+    public int getEndLine() {
+        return endLine;
+    }
+
+    public void setEndLine(int endLine) {
+        this.endLine = endLine;
+    }
+
+    public List<Integer> getTokens() {
+        return tokens;
+    }
+
+    public void setTokens(List<Integer> tokens) {
+        this.tokens = tokens;
+    }
+
+    @Override
+    public String toString() {
+        return "BlockMsg{" +
+                "beginLine=" + beginLine +
+                ", endLine=" + endLine +
+                ", tokens=" + tokens +
+                '}';
+    }
+}

src/main/kotlin/jp/ac/osaka_u/sdl/nil/NILConfig.kt

@@ -85,6 +85,7 @@ fun String.toLangOrException(): Language =
         "cs", "csharp" -> Language.CS
         "py", "python" -> Language.PYTHON
         "kt", "kotlin" -> Language.KOTLIN
+        "cur", "currency" -> Language.CURRENCY
         else -> throw InvalidOptionException("Language $this is invalid.")
     }
 
@@ -107,5 +108,5 @@ class InvalidOptionException(private val option: String) : RuntimeException() {
 }
 
 enum class Language {
-    JAVA, CPP, C, CS, PYTHON, KOTLIN
+    JAVA, CPP, C, CS, PYTHON, KOTLIN, CURRENCY
 }

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/preprocess/PreprocessFactory.kt

@@ -5,6 +5,7 @@ import jp.ac.osaka_u.sdl.nil.NILConfig
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.cpp.CPPPreprocess
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.cpp.CPreprocess
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.cs.CSharpPreprocess
+import jp.ac.osaka_u.sdl.nil.usecase.preprocess.currency.CurrencyPreprocess
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.java.JavaPreprocess
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.kotlin.KotlinPreprocess
 import jp.ac.osaka_u.sdl.nil.usecase.preprocess.python.PythonPreprocess
@@ -19,6 +20,7 @@ class PreprocessFactory {
                 Language.CS -> CSharpPreprocess(config)
                 Language.PYTHON -> PythonPreprocess(config)
                 Language.KOTLIN -> KotlinPreprocess(config)
+                Language.CURRENCY -> CurrencyPreprocess(config)
             }
     }
 }

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/preprocess/currency/CurrencyPreprocess.kt

@@ -0,0 +1,19 @@
+package jp.ac.osaka_u.sdl.nil.usecase.preprocess.currency
+
+import io.reactivex.rxjava3.core.Flowable
+import io.reactivex.rxjava3.kotlin.toFlowable
+import jp.ac.osaka_u.sdl.nil.NILConfig
+import jp.ac.osaka_u.sdl.nil.entity.CodeBlock
+import jp.ac.osaka_u.sdl.nil.usecase.preprocess.Preprocess
+import java.io.File
+
+class CurrencyPreprocess(private val config: NILConfig) : Preprocess(config.threads) {
+    override fun collectSourceFiles(dir: File): Flowable<File> =
+        dir.walk()
+            .filter { it.isFile }
+            .toFlowable()
+
+    override fun collectBlocks(srcFile: File): Flowable<CodeBlock> =
+        Flowable.just(srcFile)
+            .flatMap(CurrencyTransformer(config)::extractBlocks)
+}

src/main/kotlin/jp/ac/osaka_u/sdl/nil/usecase/preprocess/currency/CurrencyTransformer.kt

@@ -0,0 +1,23 @@
+package jp.ac.osaka_u.sdl.nil.usecase.preprocess.currency
+
+import io.reactivex.rxjava3.core.BackpressureStrategy
+import io.reactivex.rxjava3.core.Flowable
+import io.reactivex.rxjava3.core.Observable
+import jp.ac.osaka_u.sdl.nil.NILConfig
+import jp.ac.osaka_u.sdl.nil.entity.CodeBlock
+import jp.ac.osaka_u.sdl.nil.parser.utils.TokenizeUtil
+import jp.ac.osaka_u.sdl.nil.parser.utils.entity.BlockMsg
+
+import java.io.File
+
+class CurrencyTransformer(private val config: NILConfig) {
+    fun extractBlocks(sourceFile: File): Flowable<CodeBlock> =
+        Observable.create<CodeBlock> { emitter ->
+            val blockMsg: BlockMsg = TokenizeUtil.simpleTokenize(sourceFile);
+            val tokens: List<Int> = blockMsg.tokens
+            if (tokens.size >= config.minToken) {
+                emitter.onNext(CodeBlock(sourceFile.canonicalPath, blockMsg.beginLine, blockMsg.endLine,tokens))
+            }
+            emitter.onComplete()
+        }.toFlowable(BackpressureStrategy.BUFFER)
+}

src/test/kotlin/jp/ac/osaka_u/sdl/nil/NILMainTest.kt

@@ -71,6 +71,18 @@ internal class NILMainTest {
         outputFile.delete()
     }
 
+    @Test
+    fun testCur() {
+        val config =
+            parseArgs(arrayOf("-s", "./src/test/resources/examples", "-bce", "-t", "1", "-p", "1", "-l", "cur"))
+        NILMain(config).run()
+
+        val outputFile = File(config.outputFileName)
+        println(outputFile.readText().lines().sorted())
+
+        outputFile.delete()
+    }
+
     @Test
     fun testKotlin() {
         val config =

src/test/resources/examples/FizzBuzzT.py

@@ -0,0 +1,24 @@
+def main1():
+    # Comment
+    for i in range(1, 101):
+        if i % 15 == 0:
+            print("FizzBuzzT")
+        elif i % 3 == 0:
+            print("Fizz")
+        elif i % 5 == 0:
+            print("Buzz")
+        else:
+            print("TIASO")
+
+def main2():
+    for i in range(1, 101):
+        # Comment
+        if i % 15 == 0:
+            print("FizzBuzzT")
+        elif i % 3 == 0:
+            print("Fizz")
+        elif i % 5 == 0:
+            print("Buzz")
+        else:
+            print(i)
+