diff --git a/hetu-hana/src/main/java/io/hetu/core/plugin/hana/HanaConfig.java b/hetu-hana/src/main/java/io/hetu/core/plugin/hana/HanaConfig.java index cdfd80231..900721357 100644 --- a/hetu-hana/src/main/java/io/hetu/core/plugin/hana/HanaConfig.java +++ b/hetu-hana/src/main/java/io/hetu/core/plugin/hana/HanaConfig.java @@ -16,7 +16,6 @@ package io.hetu.core.plugin.hana; import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; -import io.airlift.log.Logger; import java.util.Locale; @@ -28,8 +27,6 @@ import java.util.Locale; */ public class HanaConfig { - private final Logger logger = Logger.get(HanaConfig.class); - private boolean isAutoCommit = true; private int communicationTimeout = HanaConstants.DEFAULT_COMMUNICATION_TIMEOUT; @@ -52,9 +49,7 @@ public class HanaConfig private boolean isQueryPushDownEnabled = true; - private String sqlConfigFilePath = ""; - - private boolean isDefaultPath = true; + private String hanaSqlVersion = "DEFAULT"; private boolean isByPassDsTzSetting = true; @@ -228,23 +223,14 @@ public class HanaConfig * * @return String the usable file path */ - public String getSqlConfigFilePath() + public String getHanaSqlVersion() { - return this.sqlConfigFilePath; - } - - /** - * get is the Sql rewrite configuration properties file is DefaultPath - * - * @return String the usable file path - */ - public boolean isDefaultPath() - { - return this.isDefaultPath; + return this.hanaSqlVersion; } /** * Is ignore data source's time zone,just use onquery's timezone + * * @param isByPassTz is bypass the hana time zone setting * @return HanaConfig */ @@ -258,6 +244,7 @@ public class HanaConfig /** * is bypass the data source's timezone setting. + * * @return boolean */ public boolean isByPassDataSourceTimeZone() @@ -267,6 +254,7 @@ public class HanaConfig /** * set data source's time zone key + * * @param timeZoneKey time zone key * @return HanaConfig */ @@ -280,6 +268,7 @@ public class HanaConfig /** * get datasource's time zone + * * @return time zone key */ public String getDataSourceTimeZoneKey() diff --git a/hetu-hana/src/main/java/io/hetu/core/plugin/hana/rewrite/HanaSqlQueryWriter.java b/hetu-hana/src/main/java/io/hetu/core/plugin/hana/rewrite/HanaSqlQueryWriter.java index 382ed99db..5d3ad6eb9 100644 --- a/hetu-hana/src/main/java/io/hetu/core/plugin/hana/rewrite/HanaSqlQueryWriter.java +++ b/hetu-hana/src/main/java/io/hetu/core/plugin/hana/rewrite/HanaSqlQueryWriter.java @@ -23,7 +23,8 @@ import io.hetu.core.plugin.hana.rewrite.functioncall.DateAddFunctionCallRewrite; import io.hetu.core.plugin.hana.rewrite.functioncall.DateTimeFunctionCallRewriter; import io.hetu.core.plugin.hana.rewrite.functioncall.HanaUnsupportedFunctionCallRewriter; import io.hetu.core.plugin.hana.rewrite.functioncall.VarbinaryLiteralFunctionCallRewriter; -import io.prestosql.configmanager.ConfigConstants; +import io.prestosql.configmanager.ConfigSupplier; +import io.prestosql.configmanager.DefaultUdfRewriteConfigSupplier; import io.prestosql.spi.sql.expression.Operators; import io.prestosql.spi.sql.expression.QualifiedName; import io.prestosql.spi.sql.expression.Time; @@ -82,10 +83,14 @@ public class HanaSqlQueryWriter private void functionCallManagerHandle(HanaConfig hanaConfig) { - // use the default function result string builder in the HanaConfigUdfRewriter - DefaultConnectorConfigFunctionRewriter connectorConfigFunctionRewriter = new DefaultConnectorConfigFunctionRewriter(HanaConstants.CONNECTOR_NAME, hanaConfig.getSqlConfigFilePath()); + // add inner config udf, use the default function result string builder in the HanaConfigUdfRewriter + ConfigSupplier configSupplier = new DefaultUdfRewriteConfigSupplier(UdfFunctionRewriteConstants.DEFAULT_VERSION_UDF_REWRITE_PATTERNS); + DefaultConnectorConfigFunctionRewriter connectorConfigFunctionRewriter = + new DefaultConnectorConfigFunctionRewriter(HanaConstants.CONNECTOR_NAME, configSupplier); + // use the default function Signature Builder in the HanaFunctionRewriterManager - hanaFunctionRewriterManager = FunctionWriterManagerGroup.getFunctionWriterManagerInstance(HanaConstants.CONNECTOR_NAME, ConfigConstants.DEFAULT_VERSION_NAME, getInjectFunctionCallRewritersDefault(hanaConfig), connectorConfigFunctionRewriter); + hanaFunctionRewriterManager = FunctionWriterManagerGroup.newFunctionWriterManagerInstance(HanaConstants.CONNECTOR_NAME, + hanaConfig.getHanaSqlVersion(), getInjectFunctionCallRewritersDefault(hanaConfig), connectorConfigFunctionRewriter); } private Map getInjectFunctionCallRewritersDefault(HanaConfig hanaConfig) diff --git a/hetu-hana/src/main/java/io/hetu/core/plugin/hana/rewrite/UdfFunctionRewriteConstants.java b/hetu-hana/src/main/java/io/hetu/core/plugin/hana/rewrite/UdfFunctionRewriteConstants.java new file mode 100644 index 000000000..a1153ab47 --- /dev/null +++ b/hetu-hana/src/main/java/io/hetu/core/plugin/hana/rewrite/UdfFunctionRewriteConstants.java @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.hana.rewrite; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +public class UdfFunctionRewriteConstants +{ + private UdfFunctionRewriteConstants() + { + } + + /** + * udf rewrite pattern map + */ + public static final Map DEFAULT_VERSION_UDF_REWRITE_PATTERNS = + new ImmutableMap.Builder() + //aggregate functions + .put("CORR($1,$2)", "CORR($1, $2)") + .put("STDDEV($1)", "STDDEV($1)") + .put("VARIANCE($1)", "VAR($1)") + // window rank functions + .put("RANK()", "RANK()") + .put("DENSE_RANK()", "DENSE_RANK()") + .put("ROW_NUMBER()", "ROW_NUMBER()") + .put("PERCENT_RANK()", "PERCENT_RANK()") + .put("CUME_DIST()", "CUME_DIST()") + // math functions + .put("ABS($1)", "ABS($1)") + .put("ACOS($1)", "ACOS($1)") + .put("ASIN($1)", "ASIN($1)") + .put("ATAN($1)", "ATAN($1)") + .put("ATAN2($1,$2)", "ATAN2($1, $2)") + .put("CEIL($1)", "CEIL($1)") + .put("CEILING($1)", "CEIL($1)") + .put("COS($1)", "COS($1)") + .put("EXP($1)", "EXP($1)") + .put("FLOOR($1)", "FLOOR($1)") + .put("LN($1)", "LN($1)") + .put("LOG10($1)", "LOG(10, $1)") + .put("LOG2($1)", "LOG(2, $1)") + .put("LOG($1,$2)", "LOG($1, $2)") + .put("MOD($1,$2)", "MOD($1, $2)") + .put("POW($1,$2)", "POW($1, $2)") + .put("POWER($1,$2)", "POWER($1, $2)") + .put("RAND()", "RAND()") + .put("RANDOM()", "RAND()") + .put("ROUND($1)", "ROUND($1)") + .put("ROUND($1,$2)", "ROUND($1, $2)") + .put("SIGN($1)", "SIGN($1)") + .put("SIN($1)", "SIN($1)") + .put("SQRT($1)", "SQRT($1)") + .put("TAN($1)", "TAN($1)") + //character functions + .put("CONCAT($1,$2)", "CONCAT($1, $2)") + .put("LENGTH($1)", "LENGTH($1)") + .put("LOWER($1)", "LOWER($1)") + .put("LPAD($1,$2,$3)", "LPAD($1, $2, $3)") + .put("LTRIM($1)", "LTRIM($1)") + .put("REPLACE($1,$2)", "REPLACE($1, $2, '')") + .put("REPLACE($1,$2,$3)", "REPLACE($1, $2, $3)") + .put("RPAD($1,$2,$3)", "RPAD($1, $2, $3)") + .put("RTRIM($1)", "RTRIM($1)") + .put("STRPOS($1,$2)", "LOCATE($1, $2)") + .put("SUBSTR($1,$2,$3)", "SUBSTR($1, $2, $3)") + .put("POSITION($1,$2)", "LOCATE($2, $1)") + .put("TRIM($1)", "TRIM($1)") + .put("UPPER($1)", "UPPER($1)") + //date functions + .put("YEAR($1)", "EXTRACT(YEAR FROM $1)") + .put("MONTH($1)", "EXTRACT(MONTH FROM $1)") + .put("DAY($1)", "EXTRACT(DAY FROM $1)") + .put("HOUR($1)", "EXTRACT(HOUR FROM $1)") + .put("MINUTE($1)", "EXTRACT(MINUTE FROM $1)") + .put("SECOND($1)", "EXTRACT(SECOND FROM $1)") + .put("DAY_OF_WEEK($1)", "WEEKDAY($1)") + .build(); +} diff --git a/hetu-hana/src/main/resources/Hana-default-configurations.yml b/hetu-hana/src/main/resources/Hana-default-configurations.yml deleted file mode 100644 index 7b11ebcf4..000000000 --- a/hetu-hana/src/main/resources/Hana-default-configurations.yml +++ /dev/null @@ -1,65 +0,0 @@ -rewrite_functions: - #aggregate functions - CORR($1,$2) : CORR($1, $2) - #COUNT($1): COUNT($1) - #MIN($1) : MIN($1) - #MAX($1) : MAX($1) - #SUM($1) : SUM($1) - #AVG($1) : AVG($1) - STDDEV($1) : STDDEV($1) - VARIANCE($1): VAR($1) - #window rank functions - RANK(): RANK() - DENSE_RANK(): DENSE_RANK() - ROW_NUMBER(): ROW_NUMBER() - PERCENT_RANK(): PERCENT_RANK() - CUME_DIST(): CUME_DIST() - #math functions - ABS($1) : ABS($1) - ACOS($1) : ACOS($1) - ASIN($1) : ASIN($1) - ATAN($1) : ATAN($1) - ATAN2($1,$2) : ATAN2($1, $2) - CEIL($1) : CEIL($1) - CEILING($1) : CEIL($1) - COS($1) : COS($1) - EXP($1) : EXP($1) - FLOOR($1) : FLOOR($1) - LN($1) : LN($1) - LOG10($1) : LOG(10, $1) - LOG2($1) : LOG(2, $1) - LOG($1,$2) : LOG($1, $2) - MOD($1,$2) : MOD($1, $2) - POW($1,$2) : POW($1, $2) - POWER($1,$2) : POWER($1, $2) - RAND() : RAND() - RANDOM() : RAND() - ROUND($1) : ROUND($1) - ROUND($1,$2) : ROUND($1, $2) - SIGN($1) : SIGN($1) - SIN($1) : SIN($1) - SQRT($1) : SQRT($1) - TAN($1) : TAN($1) - #character functions - CONCAT($1,$2): CONCAT($1, $2) - LENGTH($1) : LENGTH($1) - LOWER($1) : LOWER($1) - LPAD($1,$2,$3) : LPAD($1, $2, $3) - LTRIM($1) : LTRIM($1) - REPLACE($1,$2) : REPLACE($1, $2, '') - REPLACE($1,$2,$3) : REPLACE($1, $2, $3) - RPAD($1,$2,$3) : RPAD($1, $2, $3) - RTRIM($1) : RTRIM($1) - STRPOS($1,$2) : LOCATE($1, $2) - SUBSTR($1,$2,$3) : SUBSTR($1, $2, $3) - POSITION($1,$2) : LOCATE($2, $1) - TRIM($1) : TRIM($1) - UPPER($1) : UPPER($1) - #date functions - YEAR($1) : EXTRACT(YEAR FROM $1) - MONTH($1) : EXTRACT(MONTH FROM $1) - DAY($1) : EXTRACT(DAY FROM $1) - HOUR($1) : EXTRACT(HOUR FROM $1) - MINUTE($1) : EXTRACT(MINUTE FROM $1) - SECOND($1) : EXTRACT(SECOND FROM $1) - DAY_OF_WEEK($1) : WEEKDAY($1) \ No newline at end of file diff --git a/hetu-hana/src/test/java/io/hetu/core/plugin/hana/TestHanaConfig.java b/hetu-hana/src/test/java/io/hetu/core/plugin/hana/TestHanaConfig.java index 2a4280386..11124b0e8 100644 --- a/hetu-hana/src/test/java/io/hetu/core/plugin/hana/TestHanaConfig.java +++ b/hetu-hana/src/test/java/io/hetu/core/plugin/hana/TestHanaConfig.java @@ -105,8 +105,5 @@ public class TestHanaConfig boolean isQueryPushDown = config.isQueryPushDownEnabled(); assertEquals(isQueryPushDown, true); - - String defaultFilePath = config.getSqlConfigFilePath(); - assertEquals(defaultFilePath, ""); } } diff --git a/hetu-hana/src/test/java/io/hetu/core/plugin/hana/TestHanaSqlQueryWriter.java b/hetu-hana/src/test/java/io/hetu/core/plugin/hana/TestHanaSqlQueryWriter.java index 49bb67541..3ed661a45 100644 --- a/hetu-hana/src/test/java/io/hetu/core/plugin/hana/TestHanaSqlQueryWriter.java +++ b/hetu-hana/src/test/java/io/hetu/core/plugin/hana/TestHanaSqlQueryWriter.java @@ -17,10 +17,7 @@ package io.hetu.core.plugin.hana; import com.google.common.collect.ImmutableList; import io.airlift.log.Logger; import io.hetu.core.plugin.hana.rewrite.HanaSqlQueryWriter; -import io.prestosql.configmanager.ConfigConstants; -import io.prestosql.configmanager.ConfigManager; -import io.prestosql.configmanager.ConfigUtil; -import io.prestosql.configmanager.ConfigVersionFileHandler; +import io.hetu.core.plugin.hana.rewrite.UdfFunctionRewriteConstants; import io.prestosql.plugin.jdbc.BaseJdbcConfig; import io.prestosql.plugin.jdbc.ConnectionFactory; import io.prestosql.plugin.jdbc.DriverConnectionFactory; @@ -103,7 +100,6 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -142,8 +138,6 @@ public class TestHanaSqlQueryWriter private HanaConfig hanaConfig = new HanaConfig(); - private ConfigManager configManager; - /** * Create TestHanaSqlQueryWriter */ @@ -938,7 +932,7 @@ public class TestHanaSqlQueryWriter { LOGGER.info("Testing config function call rewrite"); - Map propertiesMap = loadConfigMapInTest(); + Map propertiesMap = UdfFunctionRewriteConstants.DEFAULT_VERSION_UDF_REWRITE_PATTERNS; // config functions for (Map.Entry entry : propertiesMap.entrySet()) { String key = entry.getKey(); @@ -969,17 +963,6 @@ public class TestHanaSqlQueryWriter } } - private Map loadConfigMapInTest() - { - String versionName = "ut"; - ConfigVersionFileHandler configVersionFileHandler = new ConfigVersionFileHandler(); - String defaultConfigFileName = ConfigUtil.buildFileNameFromCoNameAndVerName(HanaConstants.CONNECTOR_NAME, versionName); - configVersionFileHandler.addVersionConfigFile(defaultConfigFileName, hanaConfig.getSqlConfigFilePath()); - this.configManager = ConfigManager.newInstance(configVersionFileHandler, HanaConstants.CONNECTOR_NAME); - Map proMap = this.configManager.getConfigItemsMap(HanaConstants.CONNECTOR_NAME, versionName, ConfigConstants.CONFIG_UDF_MODULE_NAME); - return new HashMap<>(proMap); - } - @Test public void testDataAddFunctions() { diff --git a/hetu-hana/src/test/resources/Hana-ut-configurations.yml b/hetu-hana/src/test/resources/Hana-ut-configurations.yml deleted file mode 100644 index 7b11ebcf4..000000000 --- a/hetu-hana/src/test/resources/Hana-ut-configurations.yml +++ /dev/null @@ -1,65 +0,0 @@ -rewrite_functions: - #aggregate functions - CORR($1,$2) : CORR($1, $2) - #COUNT($1): COUNT($1) - #MIN($1) : MIN($1) - #MAX($1) : MAX($1) - #SUM($1) : SUM($1) - #AVG($1) : AVG($1) - STDDEV($1) : STDDEV($1) - VARIANCE($1): VAR($1) - #window rank functions - RANK(): RANK() - DENSE_RANK(): DENSE_RANK() - ROW_NUMBER(): ROW_NUMBER() - PERCENT_RANK(): PERCENT_RANK() - CUME_DIST(): CUME_DIST() - #math functions - ABS($1) : ABS($1) - ACOS($1) : ACOS($1) - ASIN($1) : ASIN($1) - ATAN($1) : ATAN($1) - ATAN2($1,$2) : ATAN2($1, $2) - CEIL($1) : CEIL($1) - CEILING($1) : CEIL($1) - COS($1) : COS($1) - EXP($1) : EXP($1) - FLOOR($1) : FLOOR($1) - LN($1) : LN($1) - LOG10($1) : LOG(10, $1) - LOG2($1) : LOG(2, $1) - LOG($1,$2) : LOG($1, $2) - MOD($1,$2) : MOD($1, $2) - POW($1,$2) : POW($1, $2) - POWER($1,$2) : POWER($1, $2) - RAND() : RAND() - RANDOM() : RAND() - ROUND($1) : ROUND($1) - ROUND($1,$2) : ROUND($1, $2) - SIGN($1) : SIGN($1) - SIN($1) : SIN($1) - SQRT($1) : SQRT($1) - TAN($1) : TAN($1) - #character functions - CONCAT($1,$2): CONCAT($1, $2) - LENGTH($1) : LENGTH($1) - LOWER($1) : LOWER($1) - LPAD($1,$2,$3) : LPAD($1, $2, $3) - LTRIM($1) : LTRIM($1) - REPLACE($1,$2) : REPLACE($1, $2, '') - REPLACE($1,$2,$3) : REPLACE($1, $2, $3) - RPAD($1,$2,$3) : RPAD($1, $2, $3) - RTRIM($1) : RTRIM($1) - STRPOS($1,$2) : LOCATE($1, $2) - SUBSTR($1,$2,$3) : SUBSTR($1, $2, $3) - POSITION($1,$2) : LOCATE($2, $1) - TRIM($1) : TRIM($1) - UPPER($1) : UPPER($1) - #date functions - YEAR($1) : EXTRACT(YEAR FROM $1) - MONTH($1) : EXTRACT(MONTH FROM $1) - DAY($1) : EXTRACT(DAY FROM $1) - HOUR($1) : EXTRACT(HOUR FROM $1) - MINUTE($1) : EXTRACT(MINUTE FROM $1) - SECOND($1) : EXTRACT(SECOND FROM $1) - DAY_OF_WEEK($1) : WEEKDAY($1) \ No newline at end of file diff --git a/pom.xml b/pom.xml index cff414c9e..4b5325db3 100644 --- a/pom.xml +++ b/pom.xml @@ -863,13 +863,6 @@ 5.1.47 - - - org.yaml - snakeyaml - 1.25 - - org.postgresql postgresql diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 91735d9b3..f288c8949 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -18,12 +18,6 @@ - - - org.yaml - snakeyaml - - io.hetu.core presto-plugin-toolkit diff --git a/presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigManager.java b/presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigManager.java deleted file mode 100644 index 30992413f..000000000 --- a/presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigManager.java +++ /dev/null @@ -1,256 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.prestosql.configmanager; - -import io.airlift.log.Logger; -import io.prestosql.configmanager.fileloader.ConfigFileParser; -import io.prestosql.configmanager.fileloader.ConfigFileParserGroup; - -import java.util.Collections; -import java.util.Map; -import java.util.Optional; - -import static java.util.Objects.requireNonNull; - -/** - * Config Dynamic Loader - * It use the ConfigFileParserGroup to manager the group of specific filenames - * - * @since 2019-12-26 - */ -public class ConfigManager -{ - private static volatile ConfigManager instance; - - private final Logger logger = Logger.get(ConfigManager.class); - - private ConfigVersionFileHandler configVersionFileHandler; - - private String connectorName; - - /** - * the constructor - */ - private ConfigManager(ConfigVersionFileHandler configVersionFileHandler, String connectorName) - { - requireNonNull(configVersionFileHandler); - requireNonNull(connectorName); - this.connectorName = connectorName; - this.configVersionFileHandler = configVersionFileHandler; - Map configFileMap = configVersionFileHandler.getFileMap(); - addConfigFiles(configFileMap); - threadStart(); - } - - private void threadStart() - { - RefreshThread refreshThread = new RefreshThread(); - refreshThread.setDaemon(true); - refreshThread.setName(connectorName + "'s ConnectorConfigFunctionRewriteThread"); - refreshThread.start(); - } - - /** - * add a config file to dynamic loader - * filename:filepath - * - * @param fileMap the config file paths's map - */ - private void addConfigFiles(Map fileMap) - { - requireNonNull(fileMap, " file path requires no null in addConfigFile"); - for (Map.Entry entry : fileMap.entrySet()) { - addConfigFile(entry.getKey(), entry.getValue()); - } - } - - /** - * add a config file to dynamic loader - * - * @param fileName fileName - * @param filePath the config file path - */ - public void addConfigFile(String fileName, String filePath) - { - requireNonNull(fileName); - requireNonNull(filePath); - ConfigFileParserGroup.addConfigFileParserInstance(fileName, filePath); - } - - /** - * get a config items from a specific config [connector name]-[version]-[module:...:property's name] - * - * @param connectorName connectorName - * @param versionName config version name - * @param configModuleNames Config Module Names include property item name - * @return Optional of the property's String values, it can be empty - */ - public Optional getConfigPropertyValue(String connectorName, String versionName, String... configModuleNames) - { - requireNonNull(connectorName); - requireNonNull(versionName); - requireNonNull(configModuleNames); - String configSearchPattern = ConfigUtil.buildConfigSearchPattern(configModuleNames); - String fileName = ConfigUtil.buildFileNameFromCoNameAndVerName(connectorName, versionName); - Optional optionalO = searchConfigValues(fileName, configSearchPattern); - if (optionalO.isPresent()) { - try { - return Optional.of((String) optionalO.get()); - } - catch (ClassCastException cce) { - logger.error("error config cast to string... "); - return Optional.empty(); - } - } - else { - return Optional.empty(); - } - } - - private Optional searchConfigValues(String configFileName, String pattern) - { - Optional configFileParserOp = ConfigFileParserGroup.getConfigFileParserInstance(configFileName); - if (configFileParserOp.isPresent()) { - ConfigFileParser configFileParser = configFileParserOp.get(); - Map yamlMap = configFileParser.loadConfigModuleMap(); - String[] patternNames = pattern.split(ConfigConstants.PATTERN_SPLIT); - for (int i = 0; i < patternNames.length - 1; i++) { - if (yamlMap.containsKey(patternNames[i])) { - try { - yamlMap = (Map) yamlMap.get(patternNames[i]); - } - catch (ClassCastException cce) { - logger.error("error config cast to map... "); - return Optional.empty(); - } - } - else { - logger.info("no such items in pattern... " + pattern); - return Optional.empty(); - } - } - try { - Object values = yamlMap.get(patternNames[patternNames.length - 1]); - if (values != null) { - return Optional.of(values); - } - else { - logger.info("no such items in pattern... " + pattern); - return Optional.empty(); - } - } - catch (ClassCastException cce) { - logger.error("error config cast to string... "); - return Optional.empty(); - } - } - else { - logger.info("no such file named: " + configFileName); - return Optional.empty(); - } - } - - /** - * get config map from a specific version and config module name - * once you get the config map of a specific version and config module, you are responsible for ensuring data consistency - * - * @param connectorName connector name - * @param versionName config version name - * @param configModuleNames Config Module Names do not include property item name - * @return config map, it can be empty - */ - @Deprecated - public Map getConfigItemsMap(String connectorName, String versionName, String... configModuleNames) - { - requireNonNull(versionName); - requireNonNull(configModuleNames); - String configSearchPattern = ConfigUtil.buildConfigSearchPattern(configModuleNames); - String fileName = ConfigUtil.buildFileNameFromCoNameAndVerName(connectorName, versionName); - Optional optionalO = searchConfigValues(fileName, configSearchPattern); - if (optionalO.isPresent()) { - try { - return (Map) optionalO.get(); - } - catch (ClassCastException cce) { - logger.error("error config cast to string... "); - return Collections.emptyMap(); - } - } - else { - return Collections.emptyMap(); - } - } - - /** - * remove a config file from the dynamic loader - * - * @param configFileName file name - */ - public void removeConfigFile(String configFileName) - { - requireNonNull(configFileName, " file path requires no null in removeConfigFile"); - ConfigFileParserGroup.removeConfigFileParser(configFileName); - } - - private class RefreshThread - extends Thread - { - private static final long SLEEP_MILLION_SECOND = 10000; - - /** - * When an object implementing interface Runnable is used to create a thread, starting the thread - * causes the object's run method to be called in that separately executing thread. - *

- * The general contract of the method run is that it may take any action whatsoever. - * - * @see Thread#run() - */ - @Override - public void run() - { - logger.info("Dynamic loader %s start at %d", Thread.currentThread().getName(), System.currentTimeMillis()); - while (true) { - ConfigFileParserGroup.refreshFactoryInstances(); - try { - Thread.sleep(SLEEP_MILLION_SECOND); - } - catch (InterruptedException e) { - logger.warn(e.getMessage()); - } - } - } - } - - /** - * ConfigManager is design as singleton, every connector/application can only construct one ConfigManager instance - * - * @param configVersionFileHandler config Version File Handler - * @param connectorName connector/application's Name - * @return an instance of ConfigManager - */ - public static ConfigManager newInstance(ConfigVersionFileHandler configVersionFileHandler, String connectorName) - { - requireNonNull(configVersionFileHandler); - requireNonNull(connectorName); - if (instance == null) { - synchronized (ConfigManager.class) { - if (instance == null) { - instance = new ConfigManager(configVersionFileHandler, connectorName); - } - } - } - return instance; - } -} diff --git a/presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigConstants.java b/presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigSupplier.java similarity index 55% rename from presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigConstants.java rename to presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigSupplier.java index d290f9620..7d9087598 100644 --- a/presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigConstants.java +++ b/presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigSupplier.java @@ -14,33 +14,22 @@ */ package io.prestosql.configmanager; -/** - * Config Constants - * @since 2019-12-28 - */ -public class ConfigConstants +import java.util.Map; +import java.util.Optional; + +public interface ConfigSupplier { - private ConfigConstants() - { - } + /** + * This method reture the configuration key-values stored in a map + * + * @return a map store configuration key-values + */ + Map getConfigKeyValueMap(); /** - * config pattern split string + * This method return the configurations value according to a key + * + * @return */ - public static final String PATTERN_SPLIT = ":"; - - /** - * udf yaml file module name - */ - public static final String CONFIG_UDF_MODULE_NAME = "rewrite_functions"; - - /** - * connector config file name mod - */ - public static final String CONNECTOR_INJECT_CONFIG_FILE_NAME = "#1-#2-configurations.yml"; - - /** - * the default version name - */ - public static final String DEFAULT_VERSION_NAME = "default"; + Optional getConfigValue(String key); } diff --git a/presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigUtil.java b/presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigUtil.java deleted file mode 100644 index 503adc8c5..000000000 --- a/presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigUtil.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.prestosql.configmanager; - -/** - * Config Utils - * - * @since 2019-12-31 - */ -public class ConfigUtil -{ - private ConfigUtil() - { - } - - /** - * Build Config Search Pattern - * - * @param propertyNames property module names - * @return Search Pattern - */ - public static String buildConfigSearchPattern(String[] propertyNames) - { - StringBuilder result = new StringBuilder(); - for (int i = 0; i < propertyNames.length - 1; i++) { - String name = propertyNames[i]; - result.append(name).append(ConfigConstants.PATTERN_SPLIT); - } - result.append(propertyNames[propertyNames.length - 1]); - return result.toString(); - } - - /** - * version file name builder, build File Name From Connector Name And Version Name - * Every config version of a connector has a config file named 'connector-configurations-version.yaml' - * - * @param connectorName connectorName - * @param versionName versionName - * @return file name connector-version-configurations.yaml - */ - public static String buildFileNameFromCoNameAndVerName(String connectorName, String versionName) - { - String mod = ConfigConstants.CONNECTOR_INJECT_CONFIG_FILE_NAME; - mod = mod.replace("#1", connectorName); - mod = mod.replace("#2", versionName); - return mod; - } -} diff --git a/presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigVersionFileHandler.java b/presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigVersionFileHandler.java deleted file mode 100644 index 1d55c14da..000000000 --- a/presto-base-jdbc/src/main/java/io/prestosql/configmanager/ConfigVersionFileHandler.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.prestosql.configmanager; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - -import static java.util.Objects.requireNonNull; - -/** - * Configuration Version Controller - * It contain the [file's name]:[file's path] - * You must use it to construct a ConfigManager instance - * - * @since 2019-12-26 - */ -public final class ConfigVersionFileHandler -{ - private Map moduleVersionFiles; - - public ConfigVersionFileHandler() - { - this.moduleVersionFiles = new ConcurrentHashMap<>(Collections.emptyMap()); - } - - public Optional getFilePathFromVersion(String versionFileName) - { - requireNonNull(versionFileName); - return Optional.ofNullable(this.moduleVersionFiles.get(versionFileName)); - } - - public Map getFileMap() - { - return new HashMap<>(moduleVersionFiles); - } - - public void addConfigVersionFiles(Map versionFiles) - { - requireNonNull(versionFiles, " moduleFiles require not null in ConfigVersionController"); - this.moduleVersionFiles.putAll(versionFiles); - } - - /** - * this method will refresh the config file for a specific version - * - * @param versionFileName versionName - * @param filePath filePath - */ - public void addVersionConfigFile(String versionFileName, String filePath) - { - moduleVersionFiles.put(versionFileName, filePath); - } -} diff --git a/presto-base-jdbc/src/main/java/io/prestosql/configmanager/DefaultUdfRewriteConfigSupplier.java b/presto-base-jdbc/src/main/java/io/prestosql/configmanager/DefaultUdfRewriteConfigSupplier.java new file mode 100644 index 000000000..67a765d69 --- /dev/null +++ b/presto-base-jdbc/src/main/java/io/prestosql/configmanager/DefaultUdfRewriteConfigSupplier.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.configmanager; + +import com.google.common.collect.ImmutableMap; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class DefaultUdfRewriteConfigSupplier + implements ConfigSupplier +{ + private final Map udfConfigMap; + + public DefaultUdfRewriteConfigSupplier(Map udfConfigMap) + { + requireNonNull(udfConfigMap); + this.udfConfigMap = new HashMap<>(udfConfigMap); + } + + @Override + public Map getConfigKeyValueMap() + { + return ImmutableMap.copyOf(this.udfConfigMap); + } + + @Override + public Optional getConfigValue(String key) + { + if (this.udfConfigMap.containsKey(key)) { + return Optional.of(this.udfConfigMap.get(key)); + } + else { + return Optional.empty(); + } + } +} diff --git a/presto-base-jdbc/src/main/java/io/prestosql/configmanager/fileloader/ConfigFileParser.java b/presto-base-jdbc/src/main/java/io/prestosql/configmanager/fileloader/ConfigFileParser.java deleted file mode 100644 index 4f0608b41..000000000 --- a/presto-base-jdbc/src/main/java/io/prestosql/configmanager/fileloader/ConfigFileParser.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.prestosql.configmanager.fileloader; - -import io.airlift.log.Logger; -import org.yaml.snakeyaml.Yaml; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import static java.util.Objects.requireNonNull; - -/** - * Config File Parser - * - * @since 2019-12-26 - */ -public final class ConfigFileParser -{ - private final Logger logger = Logger.get(ConfigFileParser.class); - - private String configFilePath; - - private String fileName; - - private String absFilePathWithFileName; - - private Yaml yaml; - - private Map yamlMaps; - - private boolean isYamlFileInJar; - - private long configFileLastModified = -1; - - /** - * the constructor - * - * @param fileName fileName - * @param yamlFilePath you can config the different file path - */ - public ConfigFileParser(String fileName, String yamlFilePath) - { - this.configFilePath = requireNonNull(yamlFilePath); - this.fileName = requireNonNull(fileName); - Path path = Paths.get(configFilePath, fileName); - this.absFilePathWithFileName = path.toString(); - reloadConfigRewriteMap(); - } - - public boolean getIsConfigFileInJar() - { - return isYamlFileInJar; - } - - public void reloadConfigRewriteMap() - { - InputStream inputStream; - // to find it in jar file properties - // UT and runtime pass way - inputStream = this.getClass().getClassLoader().getResourceAsStream(fileName); - isYamlFileInJar = true; - if (inputStream == null) { - logger.debug("yaml config path error..."); - } - else { - // parser the yaml file and save to matcherMap - this.yaml = new Yaml(); - try { - this.yamlMaps = new ConcurrentHashMap<>(yaml.load(inputStream)); - } - catch (ClassCastException cls) { - logger.info("Error format in the yaml config file!"); - this.yamlMaps = new ConcurrentHashMap<>(Collections.emptyMap()); - } - try { - inputStream.close(); - } - catch (IOException e) { - logger.debug("error closing stream in config load."); - } - } - } - - /** - * get a config module map by module name from yaml file - * - * @return the config key value map, it can be empty - */ - public Map loadConfigModuleMap() - { - if (yamlMaps != null) { - return new ConcurrentHashMap<>(this.yamlMaps); - } - else { - return Collections.emptyMap(); - } - } - - /** - * to get the result if the yaml file is modify or not - * - * @return true for having been modified and false for not - */ - public boolean isConfigFileModified() - { - if (isYamlFileInJar || absFilePathWithFileName == null) { - return false; - } - File file; - file = new File(absFilePathWithFileName); - if (!file.exists()) { - logger.info("Config file: %s path does not exists", absFilePathWithFileName); - return false; - } - boolean isChange = false; - if (file.lastModified() > configFileLastModified) { - logger.info("config file change in %s ", absFilePathWithFileName); - configFileLastModified = file.lastModified(); - isChange = true; - } - return isChange; - } -} diff --git a/presto-base-jdbc/src/main/java/io/prestosql/configmanager/fileloader/ConfigFileParserGroup.java b/presto-base-jdbc/src/main/java/io/prestosql/configmanager/fileloader/ConfigFileParserGroup.java deleted file mode 100644 index fcd93c29d..000000000 --- a/presto-base-jdbc/src/main/java/io/prestosql/configmanager/fileloader/ConfigFileParserGroup.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.prestosql.configmanager.fileloader; - -import java.util.Collections; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - -import static java.util.Objects.requireNonNull; - -/** - * Config File Parser Group - * use a file name as the specific identify of a file - * - * @since 2019-12-18 - */ -public final class ConfigFileParserGroup -{ - private static Map factoryInstances = new ConcurrentHashMap<>(Collections.emptyMap()); - - /** - * the constructor - */ - private ConfigFileParserGroup() - { - } - - /** - * refresh the ConfigFileParser instance - */ - public static void refreshFactoryInstances() - { - synchronized (ConfigFileParserGroup.class) { - for (Map.Entry entry : factoryInstances.entrySet()) { - ConfigFileParser configFileParser = entry.getValue(); - if (!configFileParser.getIsConfigFileInJar() && configFileParser.isConfigFileModified()) { - configFileParser.reloadConfigRewriteMap(); - } - } - } - } - - /** - * remove config file parser by specific file path - * - * @param configFileName config file name - */ - public static void removeConfigFileParser(String configFileName) - { - requireNonNull(configFileName, " file name requires no null..."); - synchronized (ConfigFileParserGroup.class) { - factoryInstances.remove(configFileName); - } - } - - /** - * is contain file - * - * @param configFileName config file name - * @return if config file is loaded - */ - public static boolean isContainFile(String configFileName) - { - requireNonNull(configFileName, " file name requires no null..."); - synchronized (ConfigFileParserGroup.class) { - return factoryInstances.containsKey(configFileName); - } - } - - /** - * the instance loader for singleton factory it will create a new instance using the config File path if there does - * not exist one - * - * @param fileName fileName - * @return the single instance for a special class - */ - public static Optional getConfigFileParserInstance(String fileName) - { - requireNonNull(fileName, " file name requires no null in getInstance"); - ConfigFileParser instance = factoryInstances.get(fileName); - if (instance == null) { - synchronized (ConfigFileParserGroup.class) { - if (!factoryInstances.containsKey(fileName)) { - return Optional.empty(); - } - else { - instance = factoryInstances.get(fileName); - } - } - } - return Optional.of(instance); - } - - /** - * the instance loader for singleton factory it will create a new instance using the config File path if there does - * not exist one - * - * @param fileName file name - * @param configFilePath config file path - */ - public static void addConfigFileParserInstance(String fileName, String configFilePath) - { - requireNonNull(fileName, " file name requires no null in getInstance"); - requireNonNull(configFilePath, " file path requires no null in getInstance"); - if (!factoryInstances.containsKey(fileName)) { - synchronized (ConfigFileParserGroup.class) { - if (!factoryInstances.containsKey(fileName)) { - ConfigFileParser instance = new ConfigFileParser(fileName, configFilePath); - factoryInstances.put(fileName, instance); - } - } - } - } -} diff --git a/presto-base-jdbc/src/main/java/io/prestosql/sql/builder/functioncall/FunctionWriterManagerGroup.java b/presto-base-jdbc/src/main/java/io/prestosql/sql/builder/functioncall/FunctionWriterManagerGroup.java index 0f4d341a4..3987f1d8b 100644 --- a/presto-base-jdbc/src/main/java/io/prestosql/sql/builder/functioncall/FunctionWriterManagerGroup.java +++ b/presto-base-jdbc/src/main/java/io/prestosql/sql/builder/functioncall/FunctionWriterManagerGroup.java @@ -18,8 +18,9 @@ import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter; import io.prestosql.sql.builder.functioncall.functions.config.DefaultConnectorConfigFunctionRewriter; import java.util.Collections; +import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -31,22 +32,40 @@ import static java.util.Objects.requireNonNull; */ public class FunctionWriterManagerGroup { - private static Map factoryInstances = new ConcurrentHashMap<>(Collections.emptyMap()); + private static final Map factoryInstances = new HashMap<>(Collections.emptyMap()); private FunctionWriterManagerGroup() { } /** - * Get a Function Writer Manager Instance, every version of a connector can have only one FunctionWriterManager instance + * get a Function Writer Manager Instance, every version of a connector can have only one FunctionWriterManager instance * * @param connectorName connector name - * @param version the specific identify of a FunctionWriterManager Instance's version + * @param version the data source's sql version + */ + public static Optional getFunctionWriterManagerInstance(String connectorName, String version) + { + synchronized (FunctionWriterManagerGroup.class) { + if (factoryInstances.containsKey(version)) { + return Optional.of(factoryInstances.get(version)); + } + else { + return Optional.empty(); + } + } + } + + /** + * New a Function Writer Manager Instance, every version of a connector can have only one FunctionWriterManager instance + * + * @param connectorName connector name + * @param version the data source's sql version * @param functionCallRewriterMap functionCallRewriterMap * @param connectorConfigFunctionRewriter Connector Config Function Re-writer * @return FunctionWriterManager Instance */ - public static FunctionWriterManager getFunctionWriterManagerInstance(String connectorName, String version, Map functionCallRewriterMap, DefaultConnectorConfigFunctionRewriter connectorConfigFunctionRewriter) + public static FunctionWriterManager newFunctionWriterManagerInstance(String connectorName, String version, Map functionCallRewriterMap, DefaultConnectorConfigFunctionRewriter connectorConfigFunctionRewriter) { requireNonNull(connectorName); requireNonNull(version); diff --git a/presto-base-jdbc/src/main/java/io/prestosql/sql/builder/functioncall/functions/config/DefaultConnectorConfigFunctionRewriter.java b/presto-base-jdbc/src/main/java/io/prestosql/sql/builder/functioncall/functions/config/DefaultConnectorConfigFunctionRewriter.java index 83c71d5ab..b5f994d1e 100644 --- a/presto-base-jdbc/src/main/java/io/prestosql/sql/builder/functioncall/functions/config/DefaultConnectorConfigFunctionRewriter.java +++ b/presto-base-jdbc/src/main/java/io/prestosql/sql/builder/functioncall/functions/config/DefaultConnectorConfigFunctionRewriter.java @@ -14,12 +14,10 @@ */ package io.prestosql.sql.builder.functioncall.functions.config; -import io.prestosql.configmanager.ConfigConstants; -import io.prestosql.configmanager.ConfigManager; -import io.prestosql.configmanager.ConfigUtil; -import io.prestosql.configmanager.ConfigVersionFileHandler; +import io.prestosql.configmanager.ConfigSupplier; import io.prestosql.sql.builder.functioncall.ConfigFunctionParser; import io.prestosql.sql.builder.functioncall.FunctionCallArgsPackage; +import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter; import java.util.Optional; import java.util.function.BiFunction; @@ -33,6 +31,7 @@ import static java.util.Objects.requireNonNull; * @since 2019-12-17 */ public class DefaultConnectorConfigFunctionRewriter + implements FunctionCallRewriter { protected static volatile DefaultConnectorConfigFunctionRewriter instance; @@ -42,21 +41,18 @@ public class DefaultConnectorConfigFunctionRewriter private String connectorName; - private String[] configModuleNames = {ConfigConstants.CONFIG_UDF_MODULE_NAME}; - - private ConfigVersionFileHandler configVersionFileHandler; - - private ConfigManager configManager; + private ConfigSupplier configSupplier; /** * the constructor, use an default defined function call args matcher * * @param connectorName connectorName - * @param defaultFilePath defaultFilePath + * @param configSupplier default configSupplier */ - public DefaultConnectorConfigFunctionRewriter(String connectorName, String defaultFilePath) + public DefaultConnectorConfigFunctionRewriter(String connectorName, ConfigSupplier configSupplier) { - this(connectorName, defaultFilePath, ConfigFunctionParser::baseFunctionArgsToConfigPropertyName, ConfigFunctionParser::baseConfigPropertyValueToFunctionPushDownString); + this(connectorName, configSupplier, ConfigFunctionParser::baseFunctionArgsToConfigPropertyName, + ConfigFunctionParser::baseConfigPropertyValueToFunctionPushDownString); } /** @@ -67,15 +63,12 @@ public class DefaultConnectorConfigFunctionRewriter * @param propertyNameBuilder propertyNameBuilder * @param resultFunctionStringBuilder argsFunctionStringBuilder */ - private DefaultConnectorConfigFunctionRewriter(String connectorName, String defaultFilePath, Function propertyNameBuilder, BiFunction resultFunctionStringBuilder) + private DefaultConnectorConfigFunctionRewriter(String connectorName, ConfigSupplier configSupplier, Function propertyNameBuilder, BiFunction resultFunctionStringBuilder) { this.connectorName = requireNonNull(connectorName, "versionName is null"); this.propertyNameBuilder = requireNonNull(propertyNameBuilder, "signatureBuilder is null"); this.resultFunctionStringBuilder = requireNonNull(resultFunctionStringBuilder, "argsFunction is null..."); - this.configVersionFileHandler = new ConfigVersionFileHandler(); - String defaultConfigFileName = ConfigUtil.buildFileNameFromCoNameAndVerName(connectorName, ConfigConstants.DEFAULT_VERSION_NAME); - this.configVersionFileHandler.addVersionConfigFile(defaultConfigFileName, defaultFilePath); - this.configManager = ConfigManager.newInstance(configVersionFileHandler, connectorName); + this.configSupplier = requireNonNull(configSupplier, "configSupplier is null"); } /** @@ -87,10 +80,7 @@ public class DefaultConnectorConfigFunctionRewriter public String rewriteFunctionCall(FunctionCallArgsPackage functionCallArgsPackage) { String functionPropertyName = propertyNameBuilder.apply(functionCallArgsPackage); - String[] modules = new String[configModuleNames.length + 1]; - System.arraycopy(configModuleNames, 0, modules, 0, configModuleNames.length); - modules[modules.length - 1] = functionPropertyName; - Optional propertyValue = this.configManager.getConfigPropertyValue(connectorName, ConfigConstants.DEFAULT_VERSION_NAME, modules); + Optional propertyValue = this.configSupplier.getConfigValue(functionPropertyName); return propertyValue.map(s -> resultFunctionStringBuilder.apply(functionCallArgsPackage, s)).orElse(null); } } diff --git a/presto-base-jdbc/src/test/java/io/prestosql/configmanager/TestDefaultUdfRewriteConfigSupplier.java b/presto-base-jdbc/src/test/java/io/prestosql/configmanager/TestDefaultUdfRewriteConfigSupplier.java new file mode 100644 index 000000000..1193168a1 --- /dev/null +++ b/presto-base-jdbc/src/test/java/io/prestosql/configmanager/TestDefaultUdfRewriteConfigSupplier.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.configmanager; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static org.testng.Assert.assertEquals; + +public class TestDefaultUdfRewriteConfigSupplier +{ + @Test + public void testDefaultUdfRewriteConfigSupplier() + { + Map propertiesMap = new ImmutableMap.Builder() + .put("k1", "v1") + .put("k2", "v2") + .build(); + DefaultUdfRewriteConfigSupplier defaultUdfRewriteConfigSupplier = new DefaultUdfRewriteConfigSupplier(propertiesMap); + Map map = defaultUdfRewriteConfigSupplier.getConfigKeyValueMap(); + for (Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + assertEquals(propertiesMap.get(key), value); + } + + assertEquals(defaultUdfRewriteConfigSupplier.getConfigValue("k1").get(), "v1"); + assertEquals(defaultUdfRewriteConfigSupplier.getConfigValue("k2").get(), "v2"); + } +} diff --git a/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/TestBaseFunctionUtil.java b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/TestBaseFunctionUtil.java new file mode 100644 index 000000000..3cb7adb4c --- /dev/null +++ b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/TestBaseFunctionUtil.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.sql.builder.functioncall; + +import io.prestosql.spi.sql.expression.QualifiedName; +import io.prestosql.spi.sql.expression.Selection; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.testng.Assert.assertEquals; + +public class TestBaseFunctionUtil +{ + @Test + public void testFormatIdentifier() + { + String col1String = "col1"; + Selection selection1 = new Selection("col11"); + String col2String = "col2"; + Selection selection2 = new Selection("col22"); + Map map = new HashMap<>(); + map.put(col1String, selection1); + map.put(col2String, selection2); + Optional> qualifiedNames = Optional.of(map); + + assertEquals(BaseFunctionUtil.formatIdentifier(qualifiedNames, "col1"), "col11"); + assertEquals(BaseFunctionUtil.formatIdentifier(qualifiedNames, "col2"), "col22"); + } + + @Test + public void testFormatQualifiedName() + { + List argsList = new ArrayList<>(); + argsList.add("var1"); + argsList.add("var2"); + QualifiedName qualifiedName = new QualifiedName(argsList); + assertEquals(BaseFunctionUtil.formatQualifiedName(qualifiedName), "var1.var2"); + } +} diff --git a/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/TestConfigFunctionParser.java b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/TestConfigFunctionParser.java new file mode 100644 index 000000000..e5df17ec5 --- /dev/null +++ b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/TestConfigFunctionParser.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.sql.builder.functioncall; + +import io.prestosql.spi.sql.expression.QualifiedName; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.testng.Assert.assertEquals; + +public class TestConfigFunctionParser +{ + @Test + public void testConfigFunctionParser() + { + String functionName = "LOG10"; + List functionList = new ArrayList<>(); + functionList.add(functionName); + List argsList = new ArrayList<>(); + argsList.add("var1"); + FunctionCallArgsPackage functionCallArgsPackage = + new FunctionCallArgsPackage(new QualifiedName(functionList), false, argsList, + Optional.empty(), Optional.empty(), Optional.empty()); + String configPropertyName = ConfigFunctionParser.baseFunctionArgsToConfigPropertyName(functionCallArgsPackage); + assertEquals(ConfigFunctionParser.baseConfigPropertyValueToFunctionPushDownString( + functionCallArgsPackage, UdfPropertiesConstants.Test_UDF_REWRITE_PATTERNS.get(configPropertyName)), "LOG(10, var1)"); + } +} diff --git a/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/TestFunctionWriterManagerGroup.java b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/TestFunctionWriterManagerGroup.java new file mode 100644 index 000000000..fb0704c4e --- /dev/null +++ b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/TestFunctionWriterManagerGroup.java @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.sql.builder.functioncall; + +import io.prestosql.configmanager.DefaultUdfRewriteConfigSupplier; +import io.prestosql.spi.sql.expression.QualifiedName; +import io.prestosql.sql.builder.functioncall.base.UnsupportedFunctionCallRewriterForUt; +import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter; +import io.prestosql.sql.builder.functioncall.functions.config.DefaultConnectorConfigFunctionRewriter; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static io.prestosql.sql.builder.functioncall.UdfPropertiesConstants.Test_UDF_REWRITE_PATTERNS; +import static org.testng.Assert.assertEquals; + +public class TestFunctionWriterManagerGroup +{ + private static FunctionWriterManager functionWriterManager1; + + @BeforeTest + public void setUp() + { + String connectorName = "jdbc_connector"; + String versionName = "default"; + Map functionCallRewriterMap = new HashMap<>(); + functionCallRewriterMap.put(VarbinaryLiteralFunctionCallRewriterForUt.INNER_FUNC_VARBINARY_LITERAL, new VarbinaryLiteralFunctionCallRewriterForUt()); + functionCallRewriterMap.put(UnsupportedFunctionCallRewriterForUt.UNSUPPORTED_FUNCTION_NAME_TEST, new UnsupportedFunctionCallRewriterForUt(connectorName)); + + DefaultUdfRewriteConfigSupplier defaultUdfRewriteConfigSupplier = new DefaultUdfRewriteConfigSupplier(Test_UDF_REWRITE_PATTERNS); + DefaultConnectorConfigFunctionRewriter defaultConnectorConfigFunctionRewriter = new DefaultConnectorConfigFunctionRewriter(connectorName, defaultUdfRewriteConfigSupplier); + + functionWriterManager1 = FunctionWriterManagerGroup.newFunctionWriterManagerInstance(connectorName, + versionName, functionCallRewriterMap, defaultConnectorConfigFunctionRewriter); + + FunctionWriterManager functionWriterManager2 = FunctionWriterManagerGroup.newFunctionWriterManagerInstance(connectorName, + versionName, functionCallRewriterMap, defaultConnectorConfigFunctionRewriter); + + assertEquals(functionWriterManager1.toString(), functionWriterManager2.toString()); + } + + @Test + public void testFunctionWriterManagerSupportFunctions() + { + String functionName = "LOG10"; + List functionList = new ArrayList<>(); + functionList.add(functionName); + List argsList = new ArrayList<>(); + argsList.add("var1"); + assertEquals(functionWriterManager1.getFunctionRewriteResult( + new QualifiedName(functionList), false, argsList, Optional.empty(), Optional.empty(), Optional.empty()), "LOG(10, var1)"); + + functionName = "$literal$varbinary"; + functionList.clear(); + functionList.add(functionName); + argsList.clear(); + argsList.add("1232"); + assertEquals(functionWriterManager1.getFunctionRewriteResult( + new QualifiedName(functionList), false, argsList, Optional.empty(), Optional.empty(), Optional.empty()), "X'1232'"); + + functionName = "CORR"; + functionList.clear(); + functionList.add(functionName); + argsList.clear(); + argsList.add("var1"); + argsList.add("var2"); + assertEquals(functionWriterManager1.getFunctionRewriteResult( + new QualifiedName(functionList), false, argsList, Optional.empty(), Optional.empty(), Optional.empty()), "CORR(var1, var2)"); + + functionName = "LOG10"; + functionList.clear(); + functionList.add(functionName); + argsList.clear(); + argsList.add("var1"); + assertEquals(functionWriterManager1.getFunctionRewriteResult( + new QualifiedName(functionList), false, argsList, Optional.empty(), Optional.empty(), Optional.empty()), "LOG(10, var1)"); + } + + @Test + public void testFunctionWriterManagerUnsupportedFunctions() + { + String functionName = "LO10"; + List functionList = new ArrayList<>(); + functionList.add(functionName); + List argsList = new ArrayList<>(); + argsList.add("var1"); + try { + functionWriterManager1.getFunctionRewriteResult( + new QualifiedName(functionList), false, argsList, Optional.empty(), Optional.empty(), Optional.empty()); + } + catch (UnsupportedOperationException exception) { + assertEquals(exception.getMessage(), "jdbc_connector Connector does not support function call of LO10"); + } + } +} diff --git a/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/UdfPropertiesConstants.java b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/UdfPropertiesConstants.java new file mode 100644 index 000000000..8e1f3fe86 --- /dev/null +++ b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/UdfPropertiesConstants.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.sql.builder.functioncall; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +public class UdfPropertiesConstants +{ + private UdfPropertiesConstants() + { + } + + /** + * udf rewrite pattern map + */ + public static final Map Test_UDF_REWRITE_PATTERNS = + new ImmutableMap.Builder() + //aggregate functions + .put("CORR($1,$2)", "CORR($1, $2)") + .put("LOG10($1)", "LOG(10, $1)") + .build(); +} diff --git a/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/VarbinaryLiteralFunctionCallRewriterForUt.java b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/VarbinaryLiteralFunctionCallRewriterForUt.java new file mode 100644 index 000000000..a3ae96478 --- /dev/null +++ b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/VarbinaryLiteralFunctionCallRewriterForUt.java @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.sql.builder.functioncall; + +import io.prestosql.spi.type.StandardTypes; +import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter; + +import static java.lang.String.format; + +/** + * VARBINARY literal expression is rewrite to $literal$varbinary function call. + * This is class is for rewrite inner function call to support hana expression pushdown + * + * @since 2019-09-29 + */ + +public class VarbinaryLiteralFunctionCallRewriterForUt + implements FunctionCallRewriter +{ + /** + * function call name of VARBINARY literal in HeTu inner + */ + public static final String INNER_FUNC_VARBINARY_LITERAL = "$literal$" + StandardTypes.VARBINARY; + + @Override + public String rewriteFunctionCall(FunctionCallArgsPackage functionCallArgsPackage) + { + return format("X'%s'", functionCallArgsPackage.getArgumentsList().get(0).replace("'", "")); + } +} diff --git a/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/base/TestFromBase64CallRewriter.java b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/base/TestFromBase64CallRewriter.java new file mode 100644 index 000000000..a5bb1c82e --- /dev/null +++ b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/base/TestFromBase64CallRewriter.java @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.sql.builder.functioncall.base; + +import io.prestosql.spi.sql.expression.QualifiedName; +import io.prestosql.sql.builder.functioncall.FunctionCallArgsPackage; +import io.prestosql.sql.builder.functioncall.functions.base.FromBase64CallRewriter; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static org.testng.Assert.assertEquals; + +public class TestFromBase64CallRewriter +{ + @Test + public void testFromBase64CallRewriter() + { + List list = new ArrayList<>(); + list.add("'12A69797965458999E'"); + FunctionCallArgsPackage functionCallArgsPackage = + new FunctionCallArgsPackage(new QualifiedName(Collections.emptyList()), false, list, + Optional.empty(), Optional.empty(), Optional.empty()); + FromBase64CallRewriter fromBase64CallRewriter = new FromBase64CallRewriter(); + assertEquals(fromBase64CallRewriter.rewriteFunctionCall(functionCallArgsPackage), "D7603AF7BF7BF7AE78E7CF7DF4"); + } +} diff --git a/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/base/UnsupportedFunctionCallRewriterForUt.java b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/base/UnsupportedFunctionCallRewriterForUt.java new file mode 100644 index 000000000..7ea442212 --- /dev/null +++ b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/base/UnsupportedFunctionCallRewriterForUt.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.sql.builder.functioncall.base; + +import io.prestosql.sql.builder.functioncall.functions.base.UnsupportedFunctionCallRewriter; + +public class UnsupportedFunctionCallRewriterForUt + extends UnsupportedFunctionCallRewriter +{ + public static final String UNSUPPORTED_FUNCTION_NAME_TEST = "unsupported_function"; + + /** + * the constructor of Unsupported Function Call Re-writer + * + * @param connectorName + */ + public UnsupportedFunctionCallRewriterForUt(String connectorName) + { + super(connectorName); + } +} diff --git a/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/config/TestDefaultConnectorConfigFunctionRewriter.java b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/config/TestDefaultConnectorConfigFunctionRewriter.java new file mode 100644 index 000000000..25f51795a --- /dev/null +++ b/presto-base-jdbc/src/test/java/io/prestosql/sql/builder/functioncall/config/TestDefaultConnectorConfigFunctionRewriter.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.sql.builder.functioncall.config; + +import io.prestosql.configmanager.DefaultUdfRewriteConfigSupplier; +import io.prestosql.spi.sql.expression.QualifiedName; +import io.prestosql.sql.builder.functioncall.FunctionCallArgsPackage; +import io.prestosql.sql.builder.functioncall.functions.config.DefaultConnectorConfigFunctionRewriter; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static io.prestosql.sql.builder.functioncall.UdfPropertiesConstants.Test_UDF_REWRITE_PATTERNS; +import static org.testng.Assert.assertEquals; + +public class TestDefaultConnectorConfigFunctionRewriter +{ + @Test + public void testDefaultConnectorConfigFunctionRewriter() + { + List functionName = new ArrayList<>(); + functionName.add("CORR"); + List argsList = new ArrayList<>(); + argsList.add("var1"); + argsList.add("var2"); + FunctionCallArgsPackage functionCallArgsPackage = + new FunctionCallArgsPackage(new QualifiedName(functionName), false, argsList, + Optional.empty(), Optional.empty(), Optional.empty()); + String connectorName = "jdbc_connector"; + DefaultUdfRewriteConfigSupplier defaultUdfRewriteConfigSupplier = new DefaultUdfRewriteConfigSupplier(Test_UDF_REWRITE_PATTERNS); + DefaultConnectorConfigFunctionRewriter defaultConnectorConfigFunctionRewriter = new DefaultConnectorConfigFunctionRewriter(connectorName, defaultUdfRewriteConfigSupplier); + assertEquals(defaultConnectorConfigFunctionRewriter.rewriteFunctionCall(functionCallArgsPackage), "CORR(var1, var2)"); + } +}