From db166f7b5c14294c1628def851a7e0a48ba7ebf4 Mon Sep 17 00:00:00 2001
From: XudongXie <xiexudong23@otcaix.iscas.ac.cn>
Date: Fri, 16 Aug 2024 22:35:28 +0800
Subject: [PATCH] lexer

---
 sqlTranslate/.idea/compiler.xml               | 13 +++
 sqlTranslate/.idea/encodings.xml              |  7 ++
 sqlTranslate/.idea/jarRepositories.xml        | 20 +++++
 sqlTranslate/.idea/workspace.xml              | 78 +++++++++++++++++-
 .../src/main/java/Lexer/OracleLexer.java      | 79 +++++++++++++++++++
 sqlTranslate/src/main/java/Lexer/Token.java   | 31 ++++++++
 sqlTranslate/src/main/java/Main.java          | 16 ++++
 .../src/main/java/org/example/App.java        |  7 --
 .../src/test/java/org/example/AppTest.java    | 13 ---
 9 files changed, 240 insertions(+), 24 deletions(-)
 create mode 100644 sqlTranslate/.idea/compiler.xml
 create mode 100644 sqlTranslate/.idea/encodings.xml
 create mode 100644 sqlTranslate/.idea/jarRepositories.xml
 create mode 100644 sqlTranslate/src/main/java/Lexer/OracleLexer.java
 create mode 100644 sqlTranslate/src/main/java/Lexer/Token.java
 create mode 100644 sqlTranslate/src/main/java/Main.java
 delete mode 100644 sqlTranslate/src/main/java/org/example/App.java
 delete mode 100644 sqlTranslate/src/test/java/org/example/AppTest.java

diff --git a/sqlTranslate/.idea/compiler.xml b/sqlTranslate/.idea/compiler.xml
new file mode 100644
index 00000000..8e6776c3
--- /dev/null
+++ b/sqlTranslate/.idea/compiler.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CompilerConfiguration">
+    <annotationProcessing>
+      <profile name="Maven default annotation processors profile" enabled="true">
+        <sourceOutputDir name="target/generated-sources/annotations" />
+        <sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
+        <outputRelativeToContentRoot value="true" />
+        <module name="sqlTranslate" />
+      </profile>
+    </annotationProcessing>
+  </component>
+</project>
\ No newline at end of file
diff --git a/sqlTranslate/.idea/encodings.xml b/sqlTranslate/.idea/encodings.xml
new file mode 100644
index 00000000..aa00ffab
--- /dev/null
+++ b/sqlTranslate/.idea/encodings.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Encoding">
+    <file url="file://$PROJECT_DIR$/src/main/java" charset="UTF-8" />
+    <file url="file://$PROJECT_DIR$/src/main/resources" charset="UTF-8" />
+  </component>
+</project>
\ No newline at end of file
diff --git a/sqlTranslate/.idea/jarRepositories.xml b/sqlTranslate/.idea/jarRepositories.xml
new file mode 100644
index 00000000..712ab9d9
--- /dev/null
+++ b/sqlTranslate/.idea/jarRepositories.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="RemoteRepositoriesConfiguration">
+    <remote-repository>
+      <option name="id" value="central" />
+      <option name="name" value="Central Repository" />
+      <option name="url" value="https://repo.maven.apache.org/maven2" />
+    </remote-repository>
+    <remote-repository>
+      <option name="id" value="central" />
+      <option name="name" value="Maven Central repository" />
+      <option name="url" value="https://repo1.maven.org/maven2" />
+    </remote-repository>
+    <remote-repository>
+      <option name="id" value="jboss.community" />
+      <option name="name" value="JBoss Community repository" />
+      <option name="url" value="https://repository.jboss.org/nexus/content/repositories/public/" />
+    </remote-repository>
+  </component>
+</project>
\ No newline at end of file
diff --git a/sqlTranslate/.idea/workspace.xml b/sqlTranslate/.idea/workspace.xml
index 8a06abb0..67d9ae73 100644
--- a/sqlTranslate/.idea/workspace.xml
+++ b/sqlTranslate/.idea/workspace.xml
@@ -4,14 +4,32 @@
     <option name="autoReloadType" value="SELECTIVE" />
   </component>
   <component name="ChangeListManager">
-    <list default="true" id="d8d12963-4777-48e3-9669-309cf5e41d60" name="更改" comment="">
-      <change afterPath="$PROJECT_DIR$/pom.xml" afterDir="false" />
+    <list default="true" id="d8d12963-4777-48e3-9669-309cf5e41d60" name="更改" comment="init">
+      <change afterPath="$PROJECT_DIR$/.idea/compiler.xml" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/.idea/encodings.xml" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/.idea/jarRepositories.xml" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/src/main/java/Lexer/OracleLexer.java" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/src/main/java/Lexer/Token.java" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/src/main/java/Main.java" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/src/main/java/org/example/App.java" beforeDir="false" />
+      <change beforePath="$PROJECT_DIR$/src/test/java/org/example/AppTest.java" beforeDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
     <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
     <option name="LAST_RESOLUTION" value="IGNORE" />
   </component>
+  <component name="FileTemplateManagerImpl">
+    <option name="RECENT_TEMPLATES">
+      <list>
+        <option value="Class" />
+      </list>
+    </option>
+  </component>
+  <component name="MarkdownSettingsMigration">
+    <option name="stateVersion" value="1" />
+  </component>
   <component name="MavenImportPreferences">
     <option name="importingSettings">
       <MavenImportingSettings>
@@ -20,6 +38,9 @@
     </option>
   </component>
   <component name="ProjectId" id="2kMwWhUohyqt8bZ1bSi1kwgGkSK" />
+  <component name="ProjectLevelVcsManager">
+    <ConfirmationsSetting value="2" id="Add" />
+  </component>
   <component name="ProjectViewState">
     <option name="hideEmptyMiddlePackages" value="true" />
     <option name="showLibraryContents" value="true" />
@@ -27,10 +48,11 @@
   <component name="PropertiesComponent"><![CDATA[{
   "keyToString": {
     "RunOnceActivity.OpenProjectViewOnStart": "true",
-    "RunOnceActivity.ShowReadmeOnStart": "true"
+    "RunOnceActivity.ShowReadmeOnStart": "true",
+    "SHARE_PROJECT_CONFIGURATION_FILES": "true"
   }
 }]]></component>
-  <component name="RunManager">
+  <component name="RunManager" selected="应用程序.Main">
     <configuration name="App" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
       <option name="MAIN_CLASS_NAME" value="org.example.App" />
       <module name="sqlTranslate" />
@@ -44,9 +66,34 @@
         <option name="Make" enabled="true" />
       </method>
     </configuration>
+    <configuration name="Main" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
+      <option name="MAIN_CLASS_NAME" value="Main" />
+      <module name="sqlTranslate" />
+      <method v="2">
+        <option name="Make" enabled="true" />
+      </method>
+    </configuration>
+    <configuration name="AppTest.shouldAnswerWithTrue" type="JUnit" factoryName="JUnit" temporary="true" nameIsGenerated="true">
+      <module name="sqlTranslate" />
+      <extension name="coverage">
+        <pattern>
+          <option name="PATTERN" value="org.example.*" />
+          <option name="ENABLED" value="true" />
+        </pattern>
+      </extension>
+      <option name="PACKAGE_NAME" value="org.example" />
+      <option name="MAIN_CLASS_NAME" value="org.example.AppTest" />
+      <option name="METHOD_NAME" value="shouldAnswerWithTrue" />
+      <option name="TEST_OBJECT" value="method" />
+      <method v="2">
+        <option name="Make" enabled="true" />
+      </method>
+    </configuration>
     <recent_temporary>
       <list>
+        <item itemvalue="应用程序.Main" />
         <item itemvalue="应用程序.App" />
+        <item itemvalue="JUnit.AppTest.shouldAnswerWithTrue" />
       </list>
     </recent_temporary>
   </component>
@@ -59,6 +106,29 @@
       <option name="presentableId" value="Default" />
       <updated>1723106789329</updated>
     </task>
+    <task id="LOCAL-00001" summary="init">
+      <created>1723106890550</created>
+      <option name="number" value="00001" />
+      <option name="presentableId" value="LOCAL-00001" />
+      <option name="project" value="LOCAL" />
+      <updated>1723106890550</updated>
+    </task>
+    <option name="localTasksCounter" value="2" />
     <servers />
   </component>
+  <component name="Vcs.Log.Tabs.Properties">
+    <option name="TAB_STATES">
+      <map>
+        <entry key="MAIN">
+          <value>
+            <State />
+          </value>
+        </entry>
+      </map>
+    </option>
+  </component>
+  <component name="VcsManagerConfiguration">
+    <MESSAGE value="init" />
+    <option name="LAST_COMMIT_MESSAGE" value="init" />
+  </component>
 </project>
\ No newline at end of file
diff --git a/sqlTranslate/src/main/java/Lexer/OracleLexer.java b/sqlTranslate/src/main/java/Lexer/OracleLexer.java
new file mode 100644
index 00000000..005e683b
--- /dev/null
+++ b/sqlTranslate/src/main/java/Lexer/OracleLexer.java
@@ -0,0 +1,79 @@
+package Lexer;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+
+public class OracleLexer {
+    public static final String[] keywords = {"SELECT", "FROM", "WHERE", "AND", "OR", "INSERT", "UPDATE", "DELETE", "now()"};
+    private static final Pattern TOKEN_PATTERN = Pattern.compile(
+            "(now\\(\\))|" +                    // now() function
+            "(\\b[A-Za-z_][A-Za-z0-9_]*\\b)|" + // Keywords and identifiers
+                    "(\\d+\\.?\\d*)|" +                 // Numbers (integer or decimal)
+                    "(\"[^\"]*\")|" +                   // Double-quoted strings
+                    "('([^']|\\\\')*)'|" +              // Single-quoted strings
+                    "([;,()])|" +                     // Symbols: ;, (, ), ,
+                    "([=<>+\\-*/<<>>])|" +              // Operators, including shift operators
+                    "(\\s+)"                            // Whitespace
+    );
+
+    private final List<Token> tokens = new ArrayList<>();
+    private final String input;
+    private int position = 0;
+
+    public OracleLexer(String input) {
+        this.input = input;
+    }
+
+    public List<Token> tokenize() {
+        Matcher matcher = TOKEN_PATTERN.matcher(input);
+        while (matcher.find()) {
+            String tokenValue = matcher.group();
+            Token token = createToken(tokenValue);
+            if (token != null) { // Skip whitespace
+                tokens.add(token);
+                // Update position based on the length of the matched group
+                position += tokenValue.length();
+            } else {
+                // If the token is null, we skip it (whitespace)
+                position += tokenValue.length();
+            }
+        }
+        tokens.add(new Token(Token.TokenType.EOF, ""));
+        return tokens;
+    }
+
+    private Token createToken(String tokenValue) {
+        if (tokenValue.matches("\\d+(\\.\\d+)?")) {
+            return new Token(Token.TokenType.NUMBER, tokenValue);
+        } else if (tokenValue.matches("\"[^\"]*\"")) {
+            return new Token(Token.TokenType.STRING, tokenValue);
+        } else if (tokenValue.matches("'([^']|\\\\')*'")) {
+            return new Token(Token.TokenType.STRING, tokenValue);
+        } else if (tokenValue.matches("([;(),])")) {
+            return new Token(Token.TokenType.SYMBOL, tokenValue);
+        } else if (tokenValue.matches("[=<>+\\-*/<<>>]")) {
+            return new Token(Token.TokenType.OPERATOR, tokenValue);
+        } else if (tokenValue.matches("now\\(\\)")) {
+            // Handle the now() function as a single token
+            return new Token(Token.TokenType.KEYWORD, tokenValue);
+        } else if (isKeyword(tokenValue)) {
+            return new Token(Token.TokenType.KEYWORD, tokenValue);
+        } else if (tokenValue.matches("[a-zA-Z_][a-zA-Z0-9_]*")) {
+            return new Token(Token.TokenType.IDENTIFIER, tokenValue);
+        }
+        return null;
+    }
+
+    private boolean isKeyword(String tokenValue) {
+        for (String keyword : keywords) {
+            if (keyword.equalsIgnoreCase(tokenValue)) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
diff --git a/sqlTranslate/src/main/java/Lexer/Token.java b/sqlTranslate/src/main/java/Lexer/Token.java
new file mode 100644
index 00000000..b8319f0d
--- /dev/null
+++ b/sqlTranslate/src/main/java/Lexer/Token.java
@@ -0,0 +1,31 @@
+package Lexer;
+
+public class Token {
+    public enum TokenType {
+        KEYWORD, IDENTIFIER, NUMBER, OPERATOR, STRING, SYMBOL, EOF
+    }
+
+    private final TokenType type;
+    private final String value;
+
+    public Token(TokenType type, String value) {
+        this.type = type;
+        this.value = value;
+    }
+
+    public TokenType getType() {
+        return type;
+    }
+
+    public String getValue() {
+        return value;
+    }
+
+    @Override
+    public String toString() {
+        return "Token{" +
+                "type=" + type +
+                ", value='" + value + '\'' +
+                '}';
+    }
+}
diff --git a/sqlTranslate/src/main/java/Main.java b/sqlTranslate/src/main/java/Main.java
new file mode 100644
index 00000000..b68a4f3d
--- /dev/null
+++ b/sqlTranslate/src/main/java/Main.java
@@ -0,0 +1,16 @@
+import Lexer.OracleLexer;
+import Lexer.Token;
+
+import java.util.List;
+
+public class Main {
+    public static void main(String[] args) {
+        String sql = "SELECT now() FROM table WHERE name = 'John Doe';";
+        OracleLexer lexer = new OracleLexer(sql);
+        List<Token> tokens = lexer.tokenize();
+
+        for (Token token : tokens) {
+            System.out.println(token);
+        }
+    }
+}
diff --git a/sqlTranslate/src/main/java/org/example/App.java b/sqlTranslate/src/main/java/org/example/App.java
deleted file mode 100644
index e803e22b..00000000
--- a/sqlTranslate/src/main/java/org/example/App.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package org.example;
-
-public class App {
-  public static void main(String[] args) {
-    System.out.println("Hello World!");
-  }
-}
diff --git a/sqlTranslate/src/test/java/org/example/AppTest.java b/sqlTranslate/src/test/java/org/example/AppTest.java
deleted file mode 100644
index 6fb6e67d..00000000
--- a/sqlTranslate/src/test/java/org/example/AppTest.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.example;
-
-import org.junit.jupiter.api.Test;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-public class AppTest {
-    
-  @Test
-  public void shouldAnswerWithTrue() {
-    assertTrue(true);
-  }
-}