forked from opengaussexamples/examples
lexer
This commit is contained in:
XudongXie
parent
d123cbfa82
commit
db166f7b5c
|
|
@ -0,0 +1,13 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="CompilerConfiguration">
|
||||
<annotationProcessing>
|
||||
<profile name="Maven default annotation processors profile" enabled="true">
|
||||
<sourceOutputDir name="target/generated-sources/annotations" />
|
||||
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
|
||||
<outputRelativeToContentRoot value="true" />
|
||||
<module name="sqlTranslate" />
|
||||
</profile>
|
||||
</annotationProcessing>
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="Encoding">
|
||||
<file url="file://$PROJECT_DIR$/src/main/java" charset="UTF-8" />
|
||||
<file url="file://$PROJECT_DIR$/src/main/resources" charset="UTF-8" />
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="RemoteRepositoriesConfiguration">
|
||||
<remote-repository>
|
||||
<option name="id" value="central" />
|
||||
<option name="name" value="Central Repository" />
|
||||
<option name="url" value="https://repo.maven.apache.org/maven2" />
|
||||
</remote-repository>
|
||||
<remote-repository>
|
||||
<option name="id" value="central" />
|
||||
<option name="name" value="Maven Central repository" />
|
||||
<option name="url" value="https://repo1.maven.org/maven2" />
|
||||
</remote-repository>
|
||||
<remote-repository>
|
||||
<option name="id" value="jboss.community" />
|
||||
<option name="name" value="JBoss Community repository" />
|
||||
<option name="url" value="https://repository.jboss.org/nexus/content/repositories/public/" />
|
||||
</remote-repository>
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -4,14 +4,32 @@
|
|||
<option name="autoReloadType" value="SELECTIVE" />
|
||||
</component>
|
||||
<component name="ChangeListManager">
|
||||
<list default="true" id="d8d12963-4777-48e3-9669-309cf5e41d60" name="更改" comment="">
|
||||
<change afterPath="$PROJECT_DIR$/pom.xml" afterDir="false" />
|
||||
<list default="true" id="d8d12963-4777-48e3-9669-309cf5e41d60" name="更改" comment="init">
|
||||
<change afterPath="$PROJECT_DIR$/.idea/compiler.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/encodings.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/jarRepositories.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/src/main/java/Lexer/OracleLexer.java" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/src/main/java/Lexer/Token.java" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/src/main/java/Main.java" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/src/main/java/org/example/App.java" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/src/test/java/org/example/AppTest.java" beforeDir="false" />
|
||||
</list>
|
||||
<option name="SHOW_DIALOG" value="false" />
|
||||
<option name="HIGHLIGHT_CONFLICTS" value="true" />
|
||||
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
|
||||
<option name="LAST_RESOLUTION" value="IGNORE" />
|
||||
</component>
|
||||
<component name="FileTemplateManagerImpl">
|
||||
<option name="RECENT_TEMPLATES">
|
||||
<list>
|
||||
<option value="Class" />
|
||||
</list>
|
||||
</option>
|
||||
</component>
|
||||
<component name="MarkdownSettingsMigration">
|
||||
<option name="stateVersion" value="1" />
|
||||
</component>
|
||||
<component name="MavenImportPreferences">
|
||||
<option name="importingSettings">
|
||||
<MavenImportingSettings>
|
||||
|
|
@ -20,6 +38,9 @@
|
|||
</option>
|
||||
</component>
|
||||
<component name="ProjectId" id="2kMwWhUohyqt8bZ1bSi1kwgGkSK" />
|
||||
<component name="ProjectLevelVcsManager">
|
||||
<ConfirmationsSetting value="2" id="Add" />
|
||||
</component>
|
||||
<component name="ProjectViewState">
|
||||
<option name="hideEmptyMiddlePackages" value="true" />
|
||||
<option name="showLibraryContents" value="true" />
|
||||
|
|
@ -27,10 +48,11 @@
|
|||
<component name="PropertiesComponent"><![CDATA[{
|
||||
"keyToString": {
|
||||
"RunOnceActivity.OpenProjectViewOnStart": "true",
|
||||
"RunOnceActivity.ShowReadmeOnStart": "true"
|
||||
"RunOnceActivity.ShowReadmeOnStart": "true",
|
||||
"SHARE_PROJECT_CONFIGURATION_FILES": "true"
|
||||
}
|
||||
}]]></component>
|
||||
<component name="RunManager">
|
||||
<component name="RunManager" selected="应用程序.Main">
|
||||
<configuration name="App" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
|
||||
<option name="MAIN_CLASS_NAME" value="org.example.App" />
|
||||
<module name="sqlTranslate" />
|
||||
|
|
@ -44,9 +66,34 @@
|
|||
<option name="Make" enabled="true" />
|
||||
</method>
|
||||
</configuration>
|
||||
<configuration name="Main" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
|
||||
<option name="MAIN_CLASS_NAME" value="Main" />
|
||||
<module name="sqlTranslate" />
|
||||
<method v="2">
|
||||
<option name="Make" enabled="true" />
|
||||
</method>
|
||||
</configuration>
|
||||
<configuration name="AppTest.shouldAnswerWithTrue" type="JUnit" factoryName="JUnit" temporary="true" nameIsGenerated="true">
|
||||
<module name="sqlTranslate" />
|
||||
<extension name="coverage">
|
||||
<pattern>
|
||||
<option name="PATTERN" value="org.example.*" />
|
||||
<option name="ENABLED" value="true" />
|
||||
</pattern>
|
||||
</extension>
|
||||
<option name="PACKAGE_NAME" value="org.example" />
|
||||
<option name="MAIN_CLASS_NAME" value="org.example.AppTest" />
|
||||
<option name="METHOD_NAME" value="shouldAnswerWithTrue" />
|
||||
<option name="TEST_OBJECT" value="method" />
|
||||
<method v="2">
|
||||
<option name="Make" enabled="true" />
|
||||
</method>
|
||||
</configuration>
|
||||
<recent_temporary>
|
||||
<list>
|
||||
<item itemvalue="应用程序.Main" />
|
||||
<item itemvalue="应用程序.App" />
|
||||
<item itemvalue="JUnit.AppTest.shouldAnswerWithTrue" />
|
||||
</list>
|
||||
</recent_temporary>
|
||||
</component>
|
||||
|
|
@ -59,6 +106,29 @@
|
|||
<option name="presentableId" value="Default" />
|
||||
<updated>1723106789329</updated>
|
||||
</task>
|
||||
<task id="LOCAL-00001" summary="init">
|
||||
<created>1723106890550</created>
|
||||
<option name="number" value="00001" />
|
||||
<option name="presentableId" value="LOCAL-00001" />
|
||||
<option name="project" value="LOCAL" />
|
||||
<updated>1723106890550</updated>
|
||||
</task>
|
||||
<option name="localTasksCounter" value="2" />
|
||||
<servers />
|
||||
</component>
|
||||
<component name="Vcs.Log.Tabs.Properties">
|
||||
<option name="TAB_STATES">
|
||||
<map>
|
||||
<entry key="MAIN">
|
||||
<value>
|
||||
<State />
|
||||
</value>
|
||||
</entry>
|
||||
</map>
|
||||
</option>
|
||||
</component>
|
||||
<component name="VcsManagerConfiguration">
|
||||
<MESSAGE value="init" />
|
||||
<option name="LAST_COMMIT_MESSAGE" value="init" />
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
package Lexer;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
||||
public class OracleLexer {
|
||||
public static final String[] keywords = {"SELECT", "FROM", "WHERE", "AND", "OR", "INSERT", "UPDATE", "DELETE", "now()"};
|
||||
private static final Pattern TOKEN_PATTERN = Pattern.compile(
|
||||
"(now\\(\\))|" + // now() function
|
||||
"(\\b[A-Za-z_][A-Za-z0-9_]*\\b)|" + // Keywords and identifiers
|
||||
"(\\d+\\.?\\d*)|" + // Numbers (integer or decimal)
|
||||
"(\"[^\"]*\")|" + // Double-quoted strings
|
||||
"('([^']|\\\\')*)'|" + // Single-quoted strings
|
||||
"([;,()])|" + // Symbols: ;, (, ), ,
|
||||
"([=<>+\\-*/<<>>])|" + // Operators, including shift operators
|
||||
"(\\s+)" // Whitespace
|
||||
);
|
||||
|
||||
private final List<Token> tokens = new ArrayList<>();
|
||||
private final String input;
|
||||
private int position = 0;
|
||||
|
||||
public OracleLexer(String input) {
|
||||
this.input = input;
|
||||
}
|
||||
|
||||
public List<Token> tokenize() {
|
||||
Matcher matcher = TOKEN_PATTERN.matcher(input);
|
||||
while (matcher.find()) {
|
||||
String tokenValue = matcher.group();
|
||||
Token token = createToken(tokenValue);
|
||||
if (token != null) { // Skip whitespace
|
||||
tokens.add(token);
|
||||
// Update position based on the length of the matched group
|
||||
position += tokenValue.length();
|
||||
} else {
|
||||
// If the token is null, we skip it (whitespace)
|
||||
position += tokenValue.length();
|
||||
}
|
||||
}
|
||||
tokens.add(new Token(Token.TokenType.EOF, ""));
|
||||
return tokens;
|
||||
}
|
||||
|
||||
private Token createToken(String tokenValue) {
|
||||
if (tokenValue.matches("\\d+(\\.\\d+)?")) {
|
||||
return new Token(Token.TokenType.NUMBER, tokenValue);
|
||||
} else if (tokenValue.matches("\"[^\"]*\"")) {
|
||||
return new Token(Token.TokenType.STRING, tokenValue);
|
||||
} else if (tokenValue.matches("'([^']|\\\\')*'")) {
|
||||
return new Token(Token.TokenType.STRING, tokenValue);
|
||||
} else if (tokenValue.matches("([;(),])")) {
|
||||
return new Token(Token.TokenType.SYMBOL, tokenValue);
|
||||
} else if (tokenValue.matches("[=<>+\\-*/<<>>]")) {
|
||||
return new Token(Token.TokenType.OPERATOR, tokenValue);
|
||||
} else if (tokenValue.matches("now\\(\\)")) {
|
||||
// Handle the now() function as a single token
|
||||
return new Token(Token.TokenType.KEYWORD, tokenValue);
|
||||
} else if (isKeyword(tokenValue)) {
|
||||
return new Token(Token.TokenType.KEYWORD, tokenValue);
|
||||
} else if (tokenValue.matches("[a-zA-Z_][a-zA-Z0-9_]*")) {
|
||||
return new Token(Token.TokenType.IDENTIFIER, tokenValue);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean isKeyword(String tokenValue) {
|
||||
for (String keyword : keywords) {
|
||||
if (keyword.equalsIgnoreCase(tokenValue)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package Lexer;
|
||||
|
||||
public class Token {
|
||||
public enum TokenType {
|
||||
KEYWORD, IDENTIFIER, NUMBER, OPERATOR, STRING, SYMBOL, EOF
|
||||
}
|
||||
|
||||
private final TokenType type;
|
||||
private final String value;
|
||||
|
||||
public Token(TokenType type, String value) {
|
||||
this.type = type;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public TokenType getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Token{" +
|
||||
"type=" + type +
|
||||
", value='" + value + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
import Lexer.OracleLexer;
|
||||
import Lexer.Token;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) {
|
||||
String sql = "SELECT now() FROM table WHERE name = 'John Doe';";
|
||||
OracleLexer lexer = new OracleLexer(sql);
|
||||
List<Token> tokens = lexer.tokenize();
|
||||
|
||||
for (Token token : tokens) {
|
||||
System.out.println(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
package org.example;
|
||||
|
||||
public class App {
|
||||
public static void main(String[] args) {
|
||||
System.out.println("Hello World!");
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
package org.example;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
public class AppTest {
|
||||
|
||||
@Test
|
||||
public void shouldAnswerWithTrue() {
|
||||
assertTrue(true);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue