jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!288 [I1WJ7P] Remove snake yaml dependency and related code in base-jdb and hana connector 

Merge pull request !288 from Yize_Li/remove-snakeyaml-dependency
This commit is contained in:
i-robot 2020-09-25 20:58:53 +08:00 committed by Gitee
parent 8b2e35b9ad c3a63b0767
commit 53f333ff0c
27 changed files with 667 additions and 891 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hetu-hana/src/main/java/io/hetu/core/plugin/hana/HanaConfig.java
View File

@ -16,7 +16,6 @@ package io.hetu.core.plugin.hana;
import io.airlift.configuration.Config;
import io.airlift.configuration.ConfigDescription;
import io.airlift.log.Logger;
import java.util.Locale;
@ -28,8 +27,6 @@ import java.util.Locale;
*/
public class HanaConfig
{
private final Logger logger = Logger.get(HanaConfig.class);
private boolean isAutoCommit = true;
private int communicationTimeout = HanaConstants.DEFAULT_COMMUNICATION_TIMEOUT;
@ -52,9 +49,7 @@ public class HanaConfig
private boolean isQueryPushDownEnabled = true;
private String sqlConfigFilePath = "";
private boolean isDefaultPath = true;
private String hanaSqlVersion = "DEFAULT";
private boolean isByPassDsTzSetting = true;
@ -228,23 +223,14 @@ public class HanaConfig
*
* @return String the usable file path
*/
public String getSqlConfigFilePath()
public String getHanaSqlVersion()
{
return this.sqlConfigFilePath;
}
/**
* get is the Sql rewrite configuration properties file is DefaultPath
*
* @return String the usable file path
*/
public boolean isDefaultPath()
{
return this.isDefaultPath;
return this.hanaSqlVersion;
}
/**
* Is ignore data source's time zone,just use onquery's timezone
*
* @param isByPassTz is bypass the hana time zone setting
* @return HanaConfig
*/
@ -258,6 +244,7 @@ public class HanaConfig
/**
* is bypass the data source's timezone setting.
*
* @return boolean
*/
public boolean isByPassDataSourceTimeZone()
@ -267,6 +254,7 @@ public class HanaConfig
/**
* set data source's time zone key
*
* @param timeZoneKey time zone key
* @return HanaConfig
*/
@ -280,6 +268,7 @@ public class HanaConfig
/**
* get datasource's time zone
*
* @return time zone key
*/
public String getDataSourceTimeZoneKey()

13
hetu-hana/src/main/java/io/hetu/core/plugin/hana/rewrite/HanaSqlQueryWriter.java
View File

@ -23,7 +23,8 @@ import io.hetu.core.plugin.hana.rewrite.functioncall.DateAddFunctionCallRewrite;
import io.hetu.core.plugin.hana.rewrite.functioncall.DateTimeFunctionCallRewriter;
import io.hetu.core.plugin.hana.rewrite.functioncall.HanaUnsupportedFunctionCallRewriter;
import io.hetu.core.plugin.hana.rewrite.functioncall.VarbinaryLiteralFunctionCallRewriter;
import io.prestosql.configmanager.ConfigConstants;
import io.prestosql.configmanager.ConfigSupplier;
import io.prestosql.configmanager.DefaultUdfRewriteConfigSupplier;
import io.prestosql.spi.sql.expression.Operators;
import io.prestosql.spi.sql.expression.QualifiedName;
import io.prestosql.spi.sql.expression.Time;
@ -82,10 +83,14 @@ public class HanaSqlQueryWriter
private void functionCallManagerHandle(HanaConfig hanaConfig)
{
// use the default function result string builder in the HanaConfigUdfRewriter
DefaultConnectorConfigFunctionRewriter connectorConfigFunctionRewriter = new DefaultConnectorConfigFunctionRewriter(HanaConstants.CONNECTOR_NAME, hanaConfig.getSqlConfigFilePath());
// add inner config udf, use the default function result string builder in the HanaConfigUdfRewriter
ConfigSupplier configSupplier = new DefaultUdfRewriteConfigSupplier(UdfFunctionRewriteConstants.DEFAULT_VERSION_UDF_REWRITE_PATTERNS);
DefaultConnectorConfigFunctionRewriter connectorConfigFunctionRewriter =
new DefaultConnectorConfigFunctionRewriter(HanaConstants.CONNECTOR_NAME, configSupplier);
// use the default function Signature Builder in the HanaFunctionRewriterManager
hanaFunctionRewriterManager = FunctionWriterManagerGroup.getFunctionWriterManagerInstance(HanaConstants.CONNECTOR_NAME, ConfigConstants.DEFAULT_VERSION_NAME, getInjectFunctionCallRewritersDefault(hanaConfig), connectorConfigFunctionRewriter);
hanaFunctionRewriterManager = FunctionWriterManagerGroup.newFunctionWriterManagerInstance(HanaConstants.CONNECTOR_NAME,
hanaConfig.getHanaSqlVersion(), getInjectFunctionCallRewritersDefault(hanaConfig), connectorConfigFunctionRewriter);
}
private Map<String, FunctionCallRewriter> getInjectFunctionCallRewritersDefault(HanaConfig hanaConfig)

92
hetu-hana/src/main/java/io/hetu/core/plugin/hana/rewrite/UdfFunctionRewriteConstants.java Normal file
View File

@ -0,0 +1,92 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hana.rewrite;
import com.google.common.collect.ImmutableMap;
import java.util.Map;
public class UdfFunctionRewriteConstants
{
private UdfFunctionRewriteConstants()
{
}
/**
* udf rewrite pattern map
*/
public static final Map<String, String> DEFAULT_VERSION_UDF_REWRITE_PATTERNS =
new ImmutableMap.Builder<String, String>()
//aggregate functions
.put("CORR($1,$2)", "CORR($1, $2)")
.put("STDDEV($1)", "STDDEV($1)")
.put("VARIANCE($1)", "VAR($1)")
// window rank functions
.put("RANK()", "RANK()")
.put("DENSE_RANK()", "DENSE_RANK()")
.put("ROW_NUMBER()", "ROW_NUMBER()")
.put("PERCENT_RANK()", "PERCENT_RANK()")
.put("CUME_DIST()", "CUME_DIST()")
// math functions
.put("ABS($1)", "ABS($1)")
.put("ACOS($1)", "ACOS($1)")
.put("ASIN($1)", "ASIN($1)")
.put("ATAN($1)", "ATAN($1)")
.put("ATAN2($1,$2)", "ATAN2($1, $2)")
.put("CEIL($1)", "CEIL($1)")
.put("CEILING($1)", "CEIL($1)")
.put("COS($1)", "COS($1)")
.put("EXP($1)", "EXP($1)")
.put("FLOOR($1)", "FLOOR($1)")
.put("LN($1)", "LN($1)")
.put("LOG10($1)", "LOG(10, $1)")
.put("LOG2($1)", "LOG(2, $1)")
.put("LOG($1,$2)", "LOG($1, $2)")
.put("MOD($1,$2)", "MOD($1, $2)")
.put("POW($1,$2)", "POW($1, $2)")
.put("POWER($1,$2)", "POWER($1, $2)")
.put("RAND()", "RAND()")
.put("RANDOM()", "RAND()")
.put("ROUND($1)", "ROUND($1)")
.put("ROUND($1,$2)", "ROUND($1, $2)")
.put("SIGN($1)", "SIGN($1)")
.put("SIN($1)", "SIN($1)")
.put("SQRT($1)", "SQRT($1)")
.put("TAN($1)", "TAN($1)")
//character functions
.put("CONCAT($1,$2)", "CONCAT($1, $2)")
.put("LENGTH($1)", "LENGTH($1)")
.put("LOWER($1)", "LOWER($1)")
.put("LPAD($1,$2,$3)", "LPAD($1, $2, $3)")
.put("LTRIM($1)", "LTRIM($1)")
.put("REPLACE($1,$2)", "REPLACE($1, $2, '')")
.put("REPLACE($1,$2,$3)", "REPLACE($1, $2, $3)")
.put("RPAD($1,$2,$3)", "RPAD($1, $2, $3)")
.put("RTRIM($1)", "RTRIM($1)")
.put("STRPOS($1,$2)", "LOCATE($1, $2)")
.put("SUBSTR($1,$2,$3)", "SUBSTR($1, $2, $3)")
.put("POSITION($1,$2)", "LOCATE($2, $1)")
.put("TRIM($1)", "TRIM($1)")
.put("UPPER($1)", "UPPER($1)")
//date functions
.put("YEAR($1)", "EXTRACT(YEAR FROM $1)")
.put("MONTH($1)", "EXTRACT(MONTH FROM $1)")
.put("DAY($1)", "EXTRACT(DAY FROM $1)")
.put("HOUR($1)", "EXTRACT(HOUR FROM $1)")
.put("MINUTE($1)", "EXTRACT(MINUTE FROM $1)")
.put("SECOND($1)", "EXTRACT(SECOND FROM $1)")
.put("DAY_OF_WEEK($1)", "WEEKDAY($1)")
.build();
}

65
hetu-hana/src/main/resources/Hana-default-configurations.yml
View File

@ -1,65 +0,0 @@
rewrite_functions:
#aggregate functions
CORR($1,$2) : CORR($1, $2)
#COUNT($1): COUNT($1)
#MIN($1) : MIN($1)
#MAX($1) : MAX($1)
#SUM($1) : SUM($1)
#AVG($1) : AVG($1)
STDDEV($1) : STDDEV($1)
VARIANCE($1): VAR($1)
#window rank functions
RANK(): RANK()
DENSE_RANK(): DENSE_RANK()
ROW_NUMBER(): ROW_NUMBER()
PERCENT_RANK(): PERCENT_RANK()
CUME_DIST(): CUME_DIST()
#math functions
ABS($1) : ABS($1)
ACOS($1) : ACOS($1)
ASIN($1) : ASIN($1)
ATAN($1) : ATAN($1)
ATAN2($1,$2) : ATAN2($1, $2)
CEIL($1) : CEIL($1)
CEILING($1) : CEIL($1)
COS($1) : COS($1)
EXP($1) : EXP($1)
FLOOR($1) : FLOOR($1)
LN($1) : LN($1)
LOG10($1) : LOG(10, $1)
LOG2($1) : LOG(2, $1)
LOG($1,$2) : LOG($1, $2)
MOD($1,$2) : MOD($1, $2)
POW($1,$2) : POW($1, $2)
POWER($1,$2) : POWER($1, $2)
RAND() : RAND()
RANDOM() : RAND()
ROUND($1) : ROUND($1)
ROUND($1,$2) : ROUND($1, $2)
SIGN($1) : SIGN($1)
SIN($1) : SIN($1)
SQRT($1) : SQRT($1)
TAN($1) : TAN($1)
#character functions
CONCAT($1,$2): CONCAT($1, $2)
LENGTH($1) : LENGTH($1)
LOWER($1) : LOWER($1)
LPAD($1,$2,$3) : LPAD($1, $2, $3)
LTRIM($1) : LTRIM($1)
REPLACE($1,$2) : REPLACE($1, $2, '')
REPLACE($1,$2,$3) : REPLACE($1, $2, $3)
RPAD($1,$2,$3) : RPAD($1, $2, $3)
RTRIM($1) : RTRIM($1)
STRPOS($1,$2) : LOCATE($1, $2)
SUBSTR($1,$2,$3) : SUBSTR($1, $2, $3)
POSITION($1,$2) : LOCATE($2, $1)
TRIM($1) : TRIM($1)
UPPER($1) : UPPER($1)
#date functions
YEAR($1) : EXTRACT(YEAR FROM $1)
MONTH($1) : EXTRACT(MONTH FROM $1)
DAY($1) : EXTRACT(DAY FROM $1)
HOUR($1) : EXTRACT(HOUR FROM $1)
MINUTE($1) : EXTRACT(MINUTE FROM $1)
SECOND($1) : EXTRACT(SECOND FROM $1)
DAY_OF_WEEK($1) : WEEKDAY($1)

3
hetu-hana/src/test/java/io/hetu/core/plugin/hana/TestHanaConfig.java
View File

@ -105,8 +105,5 @@ public class TestHanaConfig
boolean isQueryPushDown = config.isQueryPushDownEnabled();
assertEquals(isQueryPushDown, true);
String defaultFilePath = config.getSqlConfigFilePath();
assertEquals(defaultFilePath, "");
}
}

21
hetu-hana/src/test/java/io/hetu/core/plugin/hana/TestHanaSqlQueryWriter.java
View File

@ -17,10 +17,7 @@ package io.hetu.core.plugin.hana;
import com.google.common.collect.ImmutableList;
import io.airlift.log.Logger;
import io.hetu.core.plugin.hana.rewrite.HanaSqlQueryWriter;
import io.prestosql.configmanager.ConfigConstants;
import io.prestosql.configmanager.ConfigManager;
import io.prestosql.configmanager.ConfigUtil;
import io.prestosql.configmanager.ConfigVersionFileHandler;
import io.hetu.core.plugin.hana.rewrite.UdfFunctionRewriteConstants;
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
import io.prestosql.plugin.jdbc.ConnectionFactory;
import io.prestosql.plugin.jdbc.DriverConnectionFactory;
@ -103,7 +100,6 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@ -142,8 +138,6 @@ public class TestHanaSqlQueryWriter
private HanaConfig hanaConfig = new HanaConfig();
private ConfigManager configManager;
/**
* Create TestHanaSqlQueryWriter
*/
@ -938,7 +932,7 @@ public class TestHanaSqlQueryWriter
{
LOGGER.info("Testing config function call rewrite");
Map<String, String> propertiesMap = loadConfigMapInTest();
Map<String, String> propertiesMap = UdfFunctionRewriteConstants.DEFAULT_VERSION_UDF_REWRITE_PATTERNS;
// config functions
for (Map.Entry<String, String> entry : propertiesMap.entrySet()) {
String key = entry.getKey();
@ -969,17 +963,6 @@ public class TestHanaSqlQueryWriter
}
}
private Map<String, String> loadConfigMapInTest()
{
String versionName = "ut";
ConfigVersionFileHandler configVersionFileHandler = new ConfigVersionFileHandler();
String defaultConfigFileName = ConfigUtil.buildFileNameFromCoNameAndVerName(HanaConstants.CONNECTOR_NAME, versionName);
configVersionFileHandler.addVersionConfigFile(defaultConfigFileName, hanaConfig.getSqlConfigFilePath());
this.configManager = ConfigManager.newInstance(configVersionFileHandler, HanaConstants.CONNECTOR_NAME);
Map<String, String> proMap = this.configManager.getConfigItemsMap(HanaConstants.CONNECTOR_NAME, versionName, ConfigConstants.CONFIG_UDF_MODULE_NAME);
return new HashMap<>(proMap);
}
@Test
public void testDataAddFunctions()
{

65
hetu-hana/src/test/resources/Hana-ut-configurations.yml
View File

@ -1,65 +0,0 @@
rewrite_functions:
#aggregate functions
CORR($1,$2) : CORR($1, $2)
#COUNT($1): COUNT($1)
#MIN($1) : MIN($1)
#MAX($1) : MAX($1)
#SUM($1) : SUM($1)
#AVG($1) : AVG($1)
STDDEV($1) : STDDEV($1)
VARIANCE($1): VAR($1)
#window rank functions
RANK(): RANK()
DENSE_RANK(): DENSE_RANK()
ROW_NUMBER(): ROW_NUMBER()
PERCENT_RANK(): PERCENT_RANK()
CUME_DIST(): CUME_DIST()
#math functions
ABS($1) : ABS($1)
ACOS($1) : ACOS($1)
ASIN($1) : ASIN($1)
ATAN($1) : ATAN($1)
ATAN2($1,$2) : ATAN2($1, $2)
CEIL($1) : CEIL($1)
CEILING($1) : CEIL($1)
COS($1) : COS($1)
EXP($1) : EXP($1)
FLOOR($1) : FLOOR($1)
LN($1) : LN($1)
LOG10($1) : LOG(10, $1)
LOG2($1) : LOG(2, $1)
LOG($1,$2) : LOG($1, $2)
MOD($1,$2) : MOD($1, $2)
POW($1,$2) : POW($1, $2)
POWER($1,$2) : POWER($1, $2)
RAND() : RAND()
RANDOM() : RAND()
ROUND($1) : ROUND($1)
ROUND($1,$2) : ROUND($1, $2)
SIGN($1) : SIGN($1)
SIN($1) : SIN($1)
SQRT($1) : SQRT($1)
TAN($1) : TAN($1)
#character functions
CONCAT($1,$2): CONCAT($1, $2)
LENGTH($1) : LENGTH($1)
LOWER($1) : LOWER($1)
LPAD($1,$2,$3) : LPAD($1, $2, $3)
LTRIM($1) : LTRIM($1)
REPLACE($1,$2) : REPLACE($1, $2, '')
REPLACE($1,$2,$3) : REPLACE($1, $2, $3)
RPAD($1,$2,$3) : RPAD($1, $2, $3)
RTRIM($1) : RTRIM($1)
STRPOS($1,$2) : LOCATE($1, $2)
SUBSTR($1,$2,$3) : SUBSTR($1, $2, $3)
POSITION($1,$2) : LOCATE($2, $1)
TRIM($1) : TRIM($1)
UPPER($1) : UPPER($1)
#date functions
YEAR($1) : EXTRACT(YEAR FROM $1)
MONTH($1) : EXTRACT(MONTH FROM $1)
DAY($1) : EXTRACT(DAY FROM $1)
HOUR($1) : EXTRACT(HOUR FROM $1)
MINUTE($1) : EXTRACT(MINUTE FROM $1)
SECOND($1) : EXTRACT(SECOND FROM $1)
DAY_OF_WEEK($1) : WEEKDAY($1)

7
pom.xml
View File

@ -863,13 +863,6 @@
<version>5.1.47</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.yaml/snakeyaml -->
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.25</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>

6
presto-base-jdbc/pom.xml
View File

@ -18,12 +18,6 @@
</properties>
<dependencies>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</dependency>
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-plugin-toolkit</artifactId>

256
presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigManager.java
View File

@ -1,256 +0,0 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.configmanager;
import io.airlift.log.Logger;
import io.prestosql.configmanager.fileloader.ConfigFileParser;
import io.prestosql.configmanager.fileloader.ConfigFileParserGroup;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import static java.util.Objects.requireNonNull;
/**
* Config Dynamic Loader
* It use the ConfigFileParserGroup to manager the group of specific filenames
*
* @since 2019-12-26
*/
public class ConfigManager
{
private static volatile ConfigManager instance;
private final Logger logger = Logger.get(ConfigManager.class);
private ConfigVersionFileHandler configVersionFileHandler;
private String connectorName;
/**
* the constructor
*/
private ConfigManager(ConfigVersionFileHandler configVersionFileHandler, String connectorName)
{
requireNonNull(configVersionFileHandler);
requireNonNull(connectorName);
this.connectorName = connectorName;
this.configVersionFileHandler = configVersionFileHandler;
Map<String, String> configFileMap = configVersionFileHandler.getFileMap();
addConfigFiles(configFileMap);
threadStart();
}
private void threadStart()
{
RefreshThread refreshThread = new RefreshThread();
refreshThread.setDaemon(true);
refreshThread.setName(connectorName + "'s ConnectorConfigFunctionRewriteThread");
refreshThread.start();
}
/**
* add a config file to dynamic loader
* filename:filepath
*
* @param fileMap the config file paths's map
*/
private void addConfigFiles(Map<String, String> fileMap)
{
requireNonNull(fileMap, " file path requires no null in addConfigFile");
for (Map.Entry<String, String> entry : fileMap.entrySet()) {
addConfigFile(entry.getKey(), entry.getValue());
}
}
/**
* add a config file to dynamic loader
*
* @param fileName fileName
* @param filePath the config file path
*/
public void addConfigFile(String fileName, String filePath)
{
requireNonNull(fileName);
requireNonNull(filePath);
ConfigFileParserGroup.addConfigFileParserInstance(fileName, filePath);
}
/**
* get a config items from a specific config [connector name]-[version]-[module:...:property's name]
*
* @param connectorName connectorName
* @param versionName config version name
* @param configModuleNames Config Module Names include property item name
* @return Optional of the property's String values, it can be empty
*/
public Optional<String> getConfigPropertyValue(String connectorName, String versionName, String... configModuleNames)
{
requireNonNull(connectorName);
requireNonNull(versionName);
requireNonNull(configModuleNames);
String configSearchPattern = ConfigUtil.buildConfigSearchPattern(configModuleNames);
String fileName = ConfigUtil.buildFileNameFromCoNameAndVerName(connectorName, versionName);
Optional<Object> optionalO = searchConfigValues(fileName, configSearchPattern);
if (optionalO.isPresent()) {
try {
return Optional.of((String) optionalO.get());
}
catch (ClassCastException cce) {
logger.error("error config cast to string... ");
return Optional.empty();
}
}
else {
return Optional.empty();
}
}
private Optional<Object> searchConfigValues(String configFileName, String pattern)
{
Optional<ConfigFileParser> configFileParserOp = ConfigFileParserGroup.getConfigFileParserInstance(configFileName);
if (configFileParserOp.isPresent()) {
ConfigFileParser configFileParser = configFileParserOp.get();
Map<String, Object> yamlMap = configFileParser.loadConfigModuleMap();
String[] patternNames = pattern.split(ConfigConstants.PATTERN_SPLIT);
for (int i = 0; i < patternNames.length - 1; i++) {
if (yamlMap.containsKey(patternNames[i])) {
try {
yamlMap = (Map<String, Object>) yamlMap.get(patternNames[i]);
}
catch (ClassCastException cce) {
logger.error("error config cast to map... ");
return Optional.empty();
}
}
else {
logger.info("no such items in pattern... " + pattern);
return Optional.empty();
}
}
try {
Object values = yamlMap.get(patternNames[patternNames.length - 1]);
if (values != null) {
return Optional.of(values);
}
else {
logger.info("no such items in pattern... " + pattern);
return Optional.empty();
}
}
catch (ClassCastException cce) {
logger.error("error config cast to string... ");
return Optional.empty();
}
}
else {
logger.info("no such file named: " + configFileName);
return Optional.empty();
}
}
/**
* get config map from a specific version and config module name
* once you get the config map of a specific version and config module, you are responsible for ensuring data consistency
*
* @param connectorName connector name
* @param versionName config version name
* @param configModuleNames Config Module Names do not include property item name
* @return config map, it can be empty
*/
@Deprecated
public Map<String, String> getConfigItemsMap(String connectorName, String versionName, String... configModuleNames)
{
requireNonNull(versionName);
requireNonNull(configModuleNames);
String configSearchPattern = ConfigUtil.buildConfigSearchPattern(configModuleNames);
String fileName = ConfigUtil.buildFileNameFromCoNameAndVerName(connectorName, versionName);
Optional<Object> optionalO = searchConfigValues(fileName, configSearchPattern);
if (optionalO.isPresent()) {
try {
return (Map<String, String>) optionalO.get();
}
catch (ClassCastException cce) {
logger.error("error config cast to string... ");
return Collections.emptyMap();
}
}
else {
return Collections.emptyMap();
}
}
/**
* remove a config file from the dynamic loader
*
* @param configFileName file name
*/
public void removeConfigFile(String configFileName)
{
requireNonNull(configFileName, " file path requires no null in removeConfigFile");
ConfigFileParserGroup.removeConfigFileParser(configFileName);
}
private class RefreshThread
extends Thread
{
private static final long SLEEP_MILLION_SECOND = 10000;
/**
* When an object implementing interface <code>Runnable</code> is used to create a thread, starting the thread
* causes the object's <code>run</code> method to be called in that separately executing thread.
* <p>
* The general contract of the method <code>run</code> is that it may take any action whatsoever.
*
* @see Thread#run()
*/
@Override
public void run()
{
logger.info("Dynamic loader %s start at %d", Thread.currentThread().getName(), System.currentTimeMillis());
while (true) {
ConfigFileParserGroup.refreshFactoryInstances();
try {
Thread.sleep(SLEEP_MILLION_SECOND);
}
catch (InterruptedException e) {
logger.warn(e.getMessage());
}
}
}
}
/**
* ConfigManager is design as singleton, every connector/application can only construct one ConfigManager instance
*
* @param configVersionFileHandler config Version File Handler
* @param connectorName connector/application's Name
* @return an instance of ConfigManager
*/
public static ConfigManager newInstance(ConfigVersionFileHandler configVersionFileHandler, String connectorName)
{
requireNonNull(configVersionFileHandler);
requireNonNull(connectorName);
if (instance == null) {
synchronized (ConfigManager.class) {
if (instance == null) {
instance = new ConfigManager(configVersionFileHandler, connectorName);
}
}
}
return instance;
}
}

39
presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigConstants.java → presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigSupplier.java
View File

@ -14,33 +14,22 @@
*/
package io.prestosql.configmanager;
/**
* Config Constants
* @since 2019-12-28
*/
public class ConfigConstants
import java.util.Map;
import java.util.Optional;
public interface ConfigSupplier
{
private ConfigConstants()
{
}
/**
* This method reture the configuration key-values stored in a map
*
* @return a map store configuration key-values
*/
Map<String, String> getConfigKeyValueMap();
/**
* config pattern split string
* This method return the configurations value according to a key
*
* @return
*/
public static final String PATTERN_SPLIT = ":";
/**
* udf yaml file module name
*/
public static final String CONFIG_UDF_MODULE_NAME = "rewrite_functions";
/**
* connector config file name mod
*/
public static final String CONNECTOR_INJECT_CONFIG_FILE_NAME = "#1-#2-configurations.yml";
/**
* the default version name
*/
public static final String DEFAULT_VERSION_NAME = "default";
Optional<String> getConfigValue(String key);
}

60
presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigUtil.java
View File

@ -1,60 +0,0 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.configmanager;
/**
* Config Utils
*
* @since 2019-12-31
*/
public class ConfigUtil
{
private ConfigUtil()
{
}
/**
* Build Config Search Pattern
*
* @param propertyNames property module names
* @return Search Pattern
*/
public static String buildConfigSearchPattern(String[] propertyNames)
{
StringBuilder result = new StringBuilder();
for (int i = 0; i < propertyNames.length - 1; i++) {
String name = propertyNames[i];
result.append(name).append(ConfigConstants.PATTERN_SPLIT);
}
result.append(propertyNames[propertyNames.length - 1]);
return result.toString();
}
/**
* version file name builder, build File Name From Connector Name And Version Name
* Every config version of a connector has a config file named 'connector-configurations-version.yaml'
*
* @param connectorName connectorName
* @param versionName versionName
* @return file name connector-version-configurations.yaml
*/
public static String buildFileNameFromCoNameAndVerName(String connectorName, String versionName)
{
String mod = ConfigConstants.CONNECTOR_INJECT_CONFIG_FILE_NAME;
mod = mod.replace("#1", connectorName);
mod = mod.replace("#2", versionName);
return mod;
}
}

68
presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigVersionFileHandler.java
View File

@ -1,68 +0,0 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.configmanager;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import static java.util.Objects.requireNonNull;
/**
* Configuration Version Controller
* It contain the [file's name]:[file's path]
* You must use it to construct a ConfigManager instance
*
* @since 2019-12-26
*/
public final class ConfigVersionFileHandler
{
private Map<String, String> moduleVersionFiles;
public ConfigVersionFileHandler()
{
this.moduleVersionFiles = new ConcurrentHashMap<>(Collections.emptyMap());
}
public Optional<String> getFilePathFromVersion(String versionFileName)
{
requireNonNull(versionFileName);
return Optional.ofNullable(this.moduleVersionFiles.get(versionFileName));
}
public Map<String, String> getFileMap()
{
return new HashMap<>(moduleVersionFiles);
}
public void addConfigVersionFiles(Map<String, String> versionFiles)
{
requireNonNull(versionFiles, " moduleFiles require not null in ConfigVersionController");
this.moduleVersionFiles.putAll(versionFiles);
}
/**
* this method will refresh the config file for a specific version
*
* @param versionFileName versionName
* @param filePath filePath
*/
public void addVersionConfigFile(String versionFileName, String filePath)
{
moduleVersionFiles.put(versionFileName, filePath);
}
}

52
presto-base-jdbc/src/main/java/io/prestosql/configmanager/DefaultUdfRewriteConfigSupplier.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.configmanager;
import com.google.common.collect.ImmutableMap;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import static java.util.Objects.requireNonNull;
public class DefaultUdfRewriteConfigSupplier
implements ConfigSupplier
{
private final Map<String, String> udfConfigMap;
public DefaultUdfRewriteConfigSupplier(Map<String, String> udfConfigMap)
{
requireNonNull(udfConfigMap);
this.udfConfigMap = new HashMap<>(udfConfigMap);
}
@Override
public Map<String, String> getConfigKeyValueMap()
{
return ImmutableMap.copyOf(this.udfConfigMap);
}
@Override
public Optional<String> getConfigValue(String key)
{
if (this.udfConfigMap.containsKey(key)) {
return Optional.of(this.udfConfigMap.get(key));
}
else {
return Optional.empty();
}
}
}

142
presto-base-jdbc/src/main/java/io/prestosql/configmanager/fileloader/ConfigFileParser.java
View File

@ -1,142 +0,0 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.configmanager.fileloader;
import io.airlift.log.Logger;
import org.yaml.snakeyaml.Yaml;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import static java.util.Objects.requireNonNull;
/**
* Config File Parser
*
* @since 2019-12-26
*/
public final class ConfigFileParser
{
private final Logger logger = Logger.get(ConfigFileParser.class);
private String configFilePath;
private String fileName;
private String absFilePathWithFileName;
private Yaml yaml;
private Map<String, Object> yamlMaps;
private boolean isYamlFileInJar;
private long configFileLastModified = -1;
/**
* the constructor
*
* @param fileName fileName
* @param yamlFilePath you can config the different file path
*/
public ConfigFileParser(String fileName, String yamlFilePath)
{
this.configFilePath = requireNonNull(yamlFilePath);
this.fileName = requireNonNull(fileName);
Path path = Paths.get(configFilePath, fileName);
this.absFilePathWithFileName = path.toString();
reloadConfigRewriteMap();
}
public boolean getIsConfigFileInJar()
{
return isYamlFileInJar;
}
public void reloadConfigRewriteMap()
{
InputStream inputStream;
// to find it in jar file properties
// UT and runtime pass way
inputStream = this.getClass().getClassLoader().getResourceAsStream(fileName);
isYamlFileInJar = true;
if (inputStream == null) {
logger.debug("yaml config path error...");
}
else {
// parser the yaml file and save to matcherMap
this.yaml = new Yaml();
try {
this.yamlMaps = new ConcurrentHashMap<>(yaml.load(inputStream));
}
catch (ClassCastException cls) {
logger.info("Error format in the yaml config file!");
this.yamlMaps = new ConcurrentHashMap<>(Collections.emptyMap());
}
try {
inputStream.close();
}
catch (IOException e) {
logger.debug("error closing stream in config load.");
}
}
}
/**
* get a config module map by module name from yaml file
*
* @return the config key value map, it can be empty
*/
public Map<String, Object> loadConfigModuleMap()
{
if (yamlMaps != null) {
return new ConcurrentHashMap<>(this.yamlMaps);
}
else {
return Collections.emptyMap();
}
}
/**
* to get the result if the yaml file is modify or not
*
* @return true for having been modified and false for not
*/
public boolean isConfigFileModified()
{
if (isYamlFileInJar || absFilePathWithFileName == null) {
return false;
}
File file;
file = new File(absFilePathWithFileName);
if (!file.exists()) {
logger.info("Config file: %s path does not exists", absFilePathWithFileName);
return false;
}
boolean isChange = false;
if (file.lastModified() > configFileLastModified) {
logger.info("config file change in %s ", absFilePathWithFileName);
configFileLastModified = file.lastModified();
isChange = true;
}
return isChange;
}
}

127
presto-base-jdbc/src/main/java/io/prestosql/configmanager/fileloader/ConfigFileParserGroup.java
View File

@ -1,127 +0,0 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.configmanager.fileloader;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import static java.util.Objects.requireNonNull;
/**
* Config File Parser Group
* use a file name as the specific identify of a file
*
* @since 2019-12-18
*/
public final class ConfigFileParserGroup
{
private static Map<String, ConfigFileParser> factoryInstances = new ConcurrentHashMap<>(Collections.emptyMap());
/**
* the constructor
*/
private ConfigFileParserGroup()
{
}
/**
* refresh the ConfigFileParser instance
*/
public static void refreshFactoryInstances()
{
synchronized (ConfigFileParserGroup.class) {
for (Map.Entry<String, ConfigFileParser> entry : factoryInstances.entrySet()) {
ConfigFileParser configFileParser = entry.getValue();
if (!configFileParser.getIsConfigFileInJar() && configFileParser.isConfigFileModified()) {
configFileParser.reloadConfigRewriteMap();
}
}
}
}
/**
* remove config file parser by specific file path
*
* @param configFileName config file name
*/
public static void removeConfigFileParser(String configFileName)
{
requireNonNull(configFileName, " file name requires no null...");
synchronized (ConfigFileParserGroup.class) {
factoryInstances.remove(configFileName);
}
}
/**
* is contain file
*
* @param configFileName config file name
* @return if config file is loaded
*/
public static boolean isContainFile(String configFileName)
{
requireNonNull(configFileName, " file name requires no null...");
synchronized (ConfigFileParserGroup.class) {
return factoryInstances.containsKey(configFileName);
}
}
/**
* the instance loader for singleton factory it will create a new instance using the config File path if there does
* not exist one
*
* @param fileName fileName
* @return the single instance for a special class
*/
public static Optional<ConfigFileParser> getConfigFileParserInstance(String fileName)
{
requireNonNull(fileName, " file name requires no null in getInstance");
ConfigFileParser instance = factoryInstances.get(fileName);
if (instance == null) {
synchronized (ConfigFileParserGroup.class) {
if (!factoryInstances.containsKey(fileName)) {
return Optional.empty();
}
else {
instance = factoryInstances.get(fileName);
}
}
}
return Optional.of(instance);
}
/**
* the instance loader for singleton factory it will create a new instance using the config File path if there does
* not exist one
*
* @param fileName file name
* @param configFilePath config file path
*/
public static void addConfigFileParserInstance(String fileName, String configFilePath)
{
requireNonNull(fileName, " file name requires no null in getInstance");
requireNonNull(configFilePath, " file path requires no null in getInstance");
if (!factoryInstances.containsKey(fileName)) {
synchronized (ConfigFileParserGroup.class) {
if (!factoryInstances.containsKey(fileName)) {
ConfigFileParser instance = new ConfigFileParser(fileName, configFilePath);
factoryInstances.put(fileName, instance);
}
}
}
}
}

29
presto-base-jdbc/src/main/java/io/prestosql/sql/builder/functioncall/FunctionWriterManagerGroup.java
View File

@ -18,8 +18,9 @@ import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter;
import io.prestosql.sql.builder.functioncall.functions.config.DefaultConnectorConfigFunctionRewriter;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.Optional;
import static java.util.Objects.requireNonNull;
@ -31,22 +32,40 @@ import static java.util.Objects.requireNonNull;
*/
public class FunctionWriterManagerGroup
{
private static Map<String, FunctionWriterManager> factoryInstances = new ConcurrentHashMap<>(Collections.emptyMap());
private static final Map<String, FunctionWriterManager> factoryInstances = new HashMap<>(Collections.emptyMap());
private FunctionWriterManagerGroup()
{
}
/**
* Get a Function Writer Manager Instance, every version of a connector can have only one FunctionWriterManager instance
* get a Function Writer Manager Instance, every version of a connector can have only one FunctionWriterManager instance
*
* @param connectorName connector name
* @param version the specific identify of a FunctionWriterManager Instance's version
* @param version the data source's sql version
*/
public static Optional<FunctionWriterManager> getFunctionWriterManagerInstance(String connectorName, String version)
{
synchronized (FunctionWriterManagerGroup.class) {
if (factoryInstances.containsKey(version)) {
return Optional.of(factoryInstances.get(version));
}
else {
return Optional.empty();
}
}
}
/**
* New a Function Writer Manager Instance, every version of a connector can have only one FunctionWriterManager instance
*
* @param connectorName connector name
* @param version the data source's sql version
* @param functionCallRewriterMap functionCallRewriterMap
* @param connectorConfigFunctionRewriter Connector Config Function Re-writer
* @return FunctionWriterManager Instance
*/
public static FunctionWriterManager getFunctionWriterManagerInstance(String connectorName, String version, Map<String, FunctionCallRewriter> functionCallRewriterMap, DefaultConnectorConfigFunctionRewriter connectorConfigFunctionRewriter)
public static FunctionWriterManager newFunctionWriterManagerInstance(String connectorName, String version, Map<String, FunctionCallRewriter> functionCallRewriterMap, DefaultConnectorConfigFunctionRewriter connectorConfigFunctionRewriter)
{
requireNonNull(connectorName);
requireNonNull(version);

32
presto-base-jdbc/src/main/java/io/prestosql/sql/builder/functioncall/functions/config/DefaultConnectorConfigFunctionRewriter.java
View File

@ -14,12 +14,10 @@
*/
package io.prestosql.sql.builder.functioncall.functions.config;
import io.prestosql.configmanager.ConfigConstants;
import io.prestosql.configmanager.ConfigManager;
import io.prestosql.configmanager.ConfigUtil;
import io.prestosql.configmanager.ConfigVersionFileHandler;
import io.prestosql.configmanager.ConfigSupplier;
import io.prestosql.sql.builder.functioncall.ConfigFunctionParser;
import io.prestosql.sql.builder.functioncall.FunctionCallArgsPackage;
import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter;
import java.util.Optional;
import java.util.function.BiFunction;
@ -33,6 +31,7 @@ import static java.util.Objects.requireNonNull;
* @since 2019-12-17
*/
public class DefaultConnectorConfigFunctionRewriter
implements FunctionCallRewriter
{
protected static volatile DefaultConnectorConfigFunctionRewriter instance;
@ -42,21 +41,18 @@ public class DefaultConnectorConfigFunctionRewriter
private String connectorName;
private String[] configModuleNames = {ConfigConstants.CONFIG_UDF_MODULE_NAME};
private ConfigVersionFileHandler configVersionFileHandler;
private ConfigManager configManager;
private ConfigSupplier configSupplier;
/**
* the constructor, use an default defined function call args matcher
*
* @param connectorName connectorName
* @param defaultFilePath defaultFilePath
* @param configSupplier default configSupplier
*/
public DefaultConnectorConfigFunctionRewriter(String connectorName, String defaultFilePath)
public DefaultConnectorConfigFunctionRewriter(String connectorName, ConfigSupplier configSupplier)
{
this(connectorName, defaultFilePath, ConfigFunctionParser::baseFunctionArgsToConfigPropertyName, ConfigFunctionParser::baseConfigPropertyValueToFunctionPushDownString);
this(connectorName, configSupplier, ConfigFunctionParser::baseFunctionArgsToConfigPropertyName,
ConfigFunctionParser::baseConfigPropertyValueToFunctionPushDownString);
}
/**
@ -67,15 +63,12 @@ public class DefaultConnectorConfigFunctionRewriter
* @param propertyNameBuilder propertyNameBuilder
* @param resultFunctionStringBuilder argsFunctionStringBuilder
*/
private DefaultConnectorConfigFunctionRewriter(String connectorName, String defaultFilePath, Function<FunctionCallArgsPackage, String> propertyNameBuilder, BiFunction<FunctionCallArgsPackage, String, String> resultFunctionStringBuilder)
private DefaultConnectorConfigFunctionRewriter(String connectorName, ConfigSupplier configSupplier, Function<FunctionCallArgsPackage, String> propertyNameBuilder, BiFunction<FunctionCallArgsPackage, String, String> resultFunctionStringBuilder)
{
this.connectorName = requireNonNull(connectorName, "versionName is null");
this.propertyNameBuilder = requireNonNull(propertyNameBuilder, "signatureBuilder is null");
this.resultFunctionStringBuilder = requireNonNull(resultFunctionStringBuilder, "argsFunction is null...");
this.configVersionFileHandler = new ConfigVersionFileHandler();
String defaultConfigFileName = ConfigUtil.buildFileNameFromCoNameAndVerName(connectorName, ConfigConstants.DEFAULT_VERSION_NAME);
this.configVersionFileHandler.addVersionConfigFile(defaultConfigFileName, defaultFilePath);
this.configManager = ConfigManager.newInstance(configVersionFileHandler, connectorName);
this.configSupplier = requireNonNull(configSupplier, "configSupplier is null");
}
/**
@ -87,10 +80,7 @@ public class DefaultConnectorConfigFunctionRewriter
public String rewriteFunctionCall(FunctionCallArgsPackage functionCallArgsPackage)
{
String functionPropertyName = propertyNameBuilder.apply(functionCallArgsPackage);
String[] modules = new String[configModuleNames.length + 1];
System.arraycopy(configModuleNames, 0, modules, 0, configModuleNames.length);
modules[modules.length - 1] = functionPropertyName;
Optional<String> propertyValue = this.configManager.getConfigPropertyValue(connectorName, ConfigConstants.DEFAULT_VERSION_NAME, modules);
Optional<String> propertyValue = this.configSupplier.getConfigValue(functionPropertyName);
return propertyValue.map(s -> resultFunctionStringBuilder.apply(functionCallArgsPackage, s)).orElse(null);
}
}

44
presto-base-jdbc/src/test/java/io/prestosql/configmanager/TestDefaultUdfRewriteConfigSupplier.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.configmanager;
import com.google.common.collect.ImmutableMap;
import org.testng.annotations.Test;
import java.util.Map;
import static org.testng.Assert.assertEquals;
public class TestDefaultUdfRewriteConfigSupplier
{
@Test
public void testDefaultUdfRewriteConfigSupplier()
{
Map<String, String> propertiesMap = new ImmutableMap.Builder<String, String>()
.put("k1", "v1")
.put("k2", "v2")
.build();
DefaultUdfRewriteConfigSupplier defaultUdfRewriteConfigSupplier = new DefaultUdfRewriteConfigSupplier(propertiesMap);
Map<String, String> map = defaultUdfRewriteConfigSupplier.getConfigKeyValueMap();
for (Map.Entry<String, String> entry : map.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
assertEquals(propertiesMap.get(key), value);
}
assertEquals(defaultUdfRewriteConfigSupplier.getConfigValue("k1").get(), "v1");
assertEquals(defaultUdfRewriteConfigSupplier.getConfigValue("k2").get(), "v2");
}
}

56
presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/TestBaseFunctionUtil.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.sql.builder.functioncall;
import io.prestosql.spi.sql.expression.QualifiedName;
import io.prestosql.spi.sql.expression.Selection;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static org.testng.Assert.assertEquals;
public class TestBaseFunctionUtil
{
@Test
public void testFormatIdentifier()
{
String col1String = "col1";
Selection selection1 = new Selection("col11");
String col2String = "col2";
Selection selection2 = new Selection("col22");
Map<String, Selection> map = new HashMap<>();
map.put(col1String, selection1);
map.put(col2String, selection2);
Optional<Map<String, Selection>> qualifiedNames = Optional.of(map);
assertEquals(BaseFunctionUtil.formatIdentifier(qualifiedNames, "col1"), "col11");
assertEquals(BaseFunctionUtil.formatIdentifier(qualifiedNames, "col2"), "col22");
}
@Test
public void testFormatQualifiedName()
{
List<String> argsList = new ArrayList<>();
argsList.add("var1");
argsList.add("var2");
QualifiedName qualifiedName = new QualifiedName(argsList);
assertEquals(BaseFunctionUtil.formatQualifiedName(qualifiedName), "var1.var2");
}
}

43
presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/TestConfigFunctionParser.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.sql.builder.functioncall;
import io.prestosql.spi.sql.expression.QualifiedName;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import static org.testng.Assert.assertEquals;
public class TestConfigFunctionParser
{
@Test
public void testConfigFunctionParser()
{
String functionName = "LOG10";
List<String> functionList = new ArrayList<>();
functionList.add(functionName);
List<String> argsList = new ArrayList<>();
argsList.add("var1");
FunctionCallArgsPackage functionCallArgsPackage =
new FunctionCallArgsPackage(new QualifiedName(functionList), false, argsList,
Optional.empty(), Optional.empty(), Optional.empty());
String configPropertyName = ConfigFunctionParser.baseFunctionArgsToConfigPropertyName(functionCallArgsPackage);
assertEquals(ConfigFunctionParser.baseConfigPropertyValueToFunctionPushDownString(
functionCallArgsPackage, UdfPropertiesConstants.Test_UDF_REWRITE_PATTERNS.get(configPropertyName)), "LOG(10, var1)");
}
}

112
presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/TestFunctionWriterManagerGroup.java Normal file
View File

@ -0,0 +1,112 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.sql.builder.functioncall;
import io.prestosql.configmanager.DefaultUdfRewriteConfigSupplier;
import io.prestosql.spi.sql.expression.QualifiedName;
import io.prestosql.sql.builder.functioncall.base.UnsupportedFunctionCallRewriterForUt;
import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter;
import io.prestosql.sql.builder.functioncall.functions.config.DefaultConnectorConfigFunctionRewriter;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static io.prestosql.sql.builder.functioncall.UdfPropertiesConstants.Test_UDF_REWRITE_PATTERNS;
import static org.testng.Assert.assertEquals;
public class TestFunctionWriterManagerGroup
{
private static FunctionWriterManager functionWriterManager1;
@BeforeTest
public void setUp()
{
String connectorName = "jdbc_connector";
String versionName = "default";
Map<String, FunctionCallRewriter> functionCallRewriterMap = new HashMap<>();
functionCallRewriterMap.put(VarbinaryLiteralFunctionCallRewriterForUt.INNER_FUNC_VARBINARY_LITERAL, new VarbinaryLiteralFunctionCallRewriterForUt());
functionCallRewriterMap.put(UnsupportedFunctionCallRewriterForUt.UNSUPPORTED_FUNCTION_NAME_TEST, new UnsupportedFunctionCallRewriterForUt(connectorName));
DefaultUdfRewriteConfigSupplier defaultUdfRewriteConfigSupplier = new DefaultUdfRewriteConfigSupplier(Test_UDF_REWRITE_PATTERNS);
DefaultConnectorConfigFunctionRewriter defaultConnectorConfigFunctionRewriter = new DefaultConnectorConfigFunctionRewriter(connectorName, defaultUdfRewriteConfigSupplier);
functionWriterManager1 = FunctionWriterManagerGroup.newFunctionWriterManagerInstance(connectorName,
versionName, functionCallRewriterMap, defaultConnectorConfigFunctionRewriter);
FunctionWriterManager functionWriterManager2 = FunctionWriterManagerGroup.newFunctionWriterManagerInstance(connectorName,
versionName, functionCallRewriterMap, defaultConnectorConfigFunctionRewriter);
assertEquals(functionWriterManager1.toString(), functionWriterManager2.toString());
}
@Test
public void testFunctionWriterManagerSupportFunctions()
{
String functionName = "LOG10";
List<String> functionList = new ArrayList<>();
functionList.add(functionName);
List<String> argsList = new ArrayList<>();
argsList.add("var1");
assertEquals(functionWriterManager1.getFunctionRewriteResult(
new QualifiedName(functionList), false, argsList, Optional.empty(), Optional.empty(), Optional.empty()), "LOG(10, var1)");
functionName = "$literal$varbinary";
functionList.clear();
functionList.add(functionName);
argsList.clear();
argsList.add("1232");
assertEquals(functionWriterManager1.getFunctionRewriteResult(
new QualifiedName(functionList), false, argsList, Optional.empty(), Optional.empty(), Optional.empty()), "X'1232'");
functionName = "CORR";
functionList.clear();
functionList.add(functionName);
argsList.clear();
argsList.add("var1");
argsList.add("var2");
assertEquals(functionWriterManager1.getFunctionRewriteResult(
new QualifiedName(functionList), false, argsList, Optional.empty(), Optional.empty(), Optional.empty()), "CORR(var1, var2)");
functionName = "LOG10";
functionList.clear();
functionList.add(functionName);
argsList.clear();
argsList.add("var1");
assertEquals(functionWriterManager1.getFunctionRewriteResult(
new QualifiedName(functionList), false, argsList, Optional.empty(), Optional.empty(), Optional.empty()), "LOG(10, var1)");
}
@Test
public void testFunctionWriterManagerUnsupportedFunctions()
{
String functionName = "LO10";
List<String> functionList = new ArrayList<>();
functionList.add(functionName);
List<String> argsList = new ArrayList<>();
argsList.add("var1");
try {
functionWriterManager1.getFunctionRewriteResult(
new QualifiedName(functionList), false, argsList, Optional.empty(), Optional.empty(), Optional.empty());
}
catch (UnsupportedOperationException exception) {
assertEquals(exception.getMessage(), "jdbc_connector Connector does not support function call of LO10");
}
}
}

36
presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/UdfPropertiesConstants.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.sql.builder.functioncall;
import com.google.common.collect.ImmutableMap;
import java.util.Map;
public class UdfPropertiesConstants
{
private UdfPropertiesConstants()
{
}
/**
* udf rewrite pattern map
*/
public static final Map<String, String> Test_UDF_REWRITE_PATTERNS =
new ImmutableMap.Builder<String, String>()
//aggregate functions
.put("CORR($1,$2)", "CORR($1, $2)")
.put("LOG10($1)", "LOG(10, $1)")
.build();
}

42
presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/VarbinaryLiteralFunctionCallRewriterForUt.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.sql.builder.functioncall;
import io.prestosql.spi.type.StandardTypes;
import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter;
import static java.lang.String.format;
/**
* VARBINARY literal expression is rewrite to <code>$literal$varbinary</code> function call.
* This is class is for rewrite inner function call to support hana expression pushdown
*
* @since 2019-09-29
*/
public class VarbinaryLiteralFunctionCallRewriterForUt
implements FunctionCallRewriter
{
/**
* function call name of VARBINARY literal in HeTu inner
*/
public static final String INNER_FUNC_VARBINARY_LITERAL = "$literal$" + StandardTypes.VARBINARY;
@Override
public String rewriteFunctionCall(FunctionCallArgsPackage functionCallArgsPackage)
{
return format("X'%s'", functionCallArgsPackage.getArgumentsList().get(0).replace("'", ""));
}
}

42
presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/base/TestFromBase64CallRewriter.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.sql.builder.functioncall.base;
import io.prestosql.spi.sql.expression.QualifiedName;
import io.prestosql.sql.builder.functioncall.FunctionCallArgsPackage;
import io.prestosql.sql.builder.functioncall.functions.base.FromBase64CallRewriter;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import static org.testng.Assert.assertEquals;
public class TestFromBase64CallRewriter
{
@Test
public void testFromBase64CallRewriter()
{
List<String> list = new ArrayList<>();
list.add("'12A69797965458999E'");
FunctionCallArgsPackage functionCallArgsPackage =
new FunctionCallArgsPackage(new QualifiedName(Collections.emptyList()), false, list,
Optional.empty(), Optional.empty(), Optional.empty());
FromBase64CallRewriter fromBase64CallRewriter = new FromBase64CallRewriter();
assertEquals(fromBase64CallRewriter.rewriteFunctionCall(functionCallArgsPackage), "D7603AF7BF7BF7AE78E7CF7DF4");
}
}

33
presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/base/UnsupportedFunctionCallRewriterForUt.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.sql.builder.functioncall.base;
import io.prestosql.sql.builder.functioncall.functions.base.UnsupportedFunctionCallRewriter;
public class UnsupportedFunctionCallRewriterForUt
extends UnsupportedFunctionCallRewriter
{
public static final String UNSUPPORTED_FUNCTION_NAME_TEST = "unsupported_function";
/**
* the constructor of Unsupported Function Call Re-writer
*
* @param connectorName
*/
public UnsupportedFunctionCallRewriterForUt(String connectorName)
{
super(connectorName);
}
}

48
presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/config/TestDefaultConnectorConfigFunctionRewriter.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.sql.builder.functioncall.config;
import io.prestosql.configmanager.DefaultUdfRewriteConfigSupplier;
import io.prestosql.spi.sql.expression.QualifiedName;
import io.prestosql.sql.builder.functioncall.FunctionCallArgsPackage;
import io.prestosql.sql.builder.functioncall.functions.config.DefaultConnectorConfigFunctionRewriter;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import static io.prestosql.sql.builder.functioncall.UdfPropertiesConstants.Test_UDF_REWRITE_PATTERNS;
import static org.testng.Assert.assertEquals;
public class TestDefaultConnectorConfigFunctionRewriter
{
@Test
public void testDefaultConnectorConfigFunctionRewriter()
{
List<String> functionName = new ArrayList<>();
functionName.add("CORR");
List<String> argsList = new ArrayList<>();
argsList.add("var1");
argsList.add("var2");
FunctionCallArgsPackage functionCallArgsPackage =
new FunctionCallArgsPackage(new QualifiedName(functionName), false, argsList,
Optional.empty(), Optional.empty(), Optional.empty());
String connectorName = "jdbc_connector";
DefaultUdfRewriteConfigSupplier defaultUdfRewriteConfigSupplier = new DefaultUdfRewriteConfigSupplier(Test_UDF_REWRITE_PATTERNS);
DefaultConnectorConfigFunctionRewriter defaultConnectorConfigFunctionRewriter = new DefaultConnectorConfigFunctionRewriter(connectorName, defaultUdfRewriteConfigSupplier);
assertEquals(defaultConnectorConfigFunctionRewriter.rewriteFunctionCall(functionCallArgsPackage), "CORR(var1, var2)");
}
}