diff --git a/hetu-heuristic-index-cli/src/main/java/io/hetu/core/heuristicindex/IndexCommand.java b/hetu-heuristic-index-cli/src/main/java/io/hetu/core/heuristicindex/IndexCommand.java index e6549a953..f8a943417 100644 --- a/hetu-heuristic-index-cli/src/main/java/io/hetu/core/heuristicindex/IndexCommand.java +++ b/hetu-heuristic-index-cli/src/main/java/io/hetu/core/heuristicindex/IndexCommand.java @@ -218,12 +218,7 @@ public class IndexCommand } } catch (IOException e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Error occurred, please check the stacktrace for details: ", e); - } - else { - LOG.info("Error occurred. Enabled -v option for more details. {}", e.getMessage()); - } + LOG.error("Error occurred, please check the stacktrace for details: ", e); } return null; diff --git a/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/HeuristicIndexWriter.java b/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/HeuristicIndexWriter.java index e9c7a4bd6..064c85220 100644 --- a/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/HeuristicIndexWriter.java +++ b/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/HeuristicIndexWriter.java @@ -109,6 +109,7 @@ public class HeuristicIndexWriter requireNonNull(table, "no table specified"); requireNonNull(columns, "no columns specified"); requireNonNull(indexTypes, "no index types specified"); + checkIndexTypes(indexTypes); LOG.info("Creating index for: table={} columns={} partitions={}", table, Arrays.toString(columns), partitions == null ? "all" : Arrays.toString(partitions)); @@ -213,12 +214,6 @@ public class HeuristicIndexWriter // the instances in the map are the "base" instances bc they have their properties set // we need to create a new Index instance for each split and copy the properties the base has Index indexTypeBaseObj = indexTypesMap.get(indexType.toLowerCase(Locale.ENGLISH)); - if (indexTypeBaseObj == null) { - String msg = String.format(Locale.ENGLISH, "Index type %s not supported.", indexType); - LOG.error(msg); - throw new IllegalArgumentException(msg); - } - Index splitIndex; try { Constructor constructor = indexTypeBaseObj.getClass().getConstructor(); @@ -367,6 +362,18 @@ public class HeuristicIndexWriter } } + private void checkIndexTypes(String[] indexTypes) + { + for (String indexType : indexTypes) { + Index indexTypeBaseObj = indexTypesMap.get(indexType.toLowerCase(Locale.ENGLISH)); + if (indexTypeBaseObj == null) { + String msg = String.format(Locale.ENGLISH, "Index type %s not supported.", indexType); + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + } + } + private void cleanPartFiles(Collection partFiles) { if (!isCleanedUp) { diff --git a/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/util/IndexServiceUtils.java b/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/util/IndexServiceUtils.java index 8e0facd2b..f88b23057 100644 --- a/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/util/IndexServiceUtils.java +++ b/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/util/IndexServiceUtils.java @@ -52,11 +52,13 @@ public class IndexServiceUtils /** * there are minimum three parts in "catalog.schema.table"1 */ - private static final int MINIMUM_FULLY_QUALIFIED_TABLE_FORMAT_PARTS = 3; + private static final int FULLY_QUALIFIED_TABLE_FORMAT_PARTS = 3; - private static final int DATABASE_NAME_OFFSET = 2; + private static final int CATALOG_NAME_INDEX = 0; - private static final int TABLE_NAME_OFFSET = 1; + private static final int DATABASE_NAME_INDEX = 1; + + private static final int TABLE_NAME_INDEX = 2; private IndexServiceUtils() { @@ -76,6 +78,30 @@ public class IndexServiceUtils return path; } + /** + * check if a file with specific file path exist + * + * @param filePath filesPath that need to be checked + */ + public static void isFileExisting(String filePath) throws IOException + { + File file = Paths.get(filePath).toFile(); + isFileExisting(file); + } + + /** + * load properties from a filePath + * + * @param propertyFilePath property file path + * @return Property object which holds all properties + * @throws IOException when property file does NOT exist + */ + public static Properties loadProperties(String propertyFilePath) throws IOException + { + File propertyFile = Paths.get(propertyFilePath).toFile(); + return loadProperties(propertyFile); + } + /** * check if a file with specific file path exist * @@ -130,18 +156,17 @@ public class IndexServiceUtils { String[] parts = fullyQualifiedTableName.split("\\."); - checkArgument(parts.length >= MINIMUM_FULLY_QUALIFIED_TABLE_FORMAT_PARTS, + checkArgument(parts.length == FULLY_QUALIFIED_TABLE_FORMAT_PARTS, INVALID_TABLE_NAME_ERR_MSG); - String databaseName = parts[parts.length - DATABASE_NAME_OFFSET].trim(); + String catalogName = parts[CATALOG_NAME_INDEX].trim(); + checkArgument(!catalogName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG); + + String databaseName = parts[DATABASE_NAME_INDEX].trim(); checkArgument(!databaseName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG); - String catalogName = fullyQualifiedTableName - .substring(0, fullyQualifiedTableName.indexOf(databaseName) - 1).trim(); - checkArgument(!catalogName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG); - - String tableName = parts[parts.length - TABLE_NAME_OFFSET]; - checkArgument(!catalogName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG); + String tableName = parts[TABLE_NAME_INDEX]; + checkArgument(!tableName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG); return new String[]{catalogName, databaseName, tableName}; } diff --git a/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/datasource/hive/HadoopUtil.java b/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/datasource/hive/HadoopUtil.java index d798060e9..2096ba678 100644 --- a/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/datasource/hive/HadoopUtil.java +++ b/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/datasource/hive/HadoopUtil.java @@ -14,6 +14,7 @@ */ package io.hetu.core.plugin.heuristicindex.datasource.hive; +import io.hetu.core.heuristicindex.util.IndexServiceUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; diff --git a/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/datasource/hive/IndexServiceUtils.java b/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/datasource/hive/IndexServiceUtils.java deleted file mode 100644 index 9cc032d60..000000000 --- a/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/datasource/hive/IndexServiceUtils.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.hetu.core.plugin.heuristicindex.datasource.hive; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Paths; -import java.util.Properties; - -import static com.google.common.base.Preconditions.checkArgument; - -/** - * Util class for creating external index. - */ -public class IndexServiceUtils -{ - /** - * Error message for an invalid table name - */ - private static final String INVALID_TABLE_NAME_ERR_MSG = "fully qualified table name is invalid, expected 'catalog.schema.table'"; - - /** - * there are minimum three parts in "catalog.schema.table"1 - */ - private static final int MINIMUM_FULLY_QUALIFIED_TABLE_FORMAT_PARTS = 3; - - private static final int DATABASE_NAME_OFFSET = 2; - - private static final int TABLE_NAME_OFFSET = 1; - - private IndexServiceUtils() - { - } - - /** - * format a string into path format, add file separator if it's missing - * - * @param path path need to be formatted - * @return formatted path - */ - public static String formatPathAsFolder(String path) - { - if (!path.endsWith(File.separator)) { - return path + File.separator; - } - return path; - } - - /** - * check if a file with specific file path exist - * - * @param filePath filesPath that need to be checked - */ - public static void isFileExisting(String filePath) throws IOException - { - File file = Paths.get(filePath).toFile(); - isFileExisting(file); - } - - /** - * check if a file with specific file path exist - * - * @param file file need to be checked - */ - public static void isFileExisting(File file) throws IOException - { - checkArgument(file.exists(), file.getCanonicalPath() + " not found"); - } - - /** - * load properties from a filePath - * - * @param propertyFilePath property file path - * @return Property object which holds all properties - * @throws IOException when property file does NOT exist - */ - public static Properties loadProperties(String propertyFilePath) throws IOException - { - File propertyFile = Paths.get(propertyFilePath).toFile(); - return loadProperties(propertyFile); - } - - /** - * load properties from a file object - * - * @param propertyFile property file - * @return Property object which holds all properties - * @throws IOException when property file does NOT exist - */ - public static Properties loadProperties(File propertyFile) throws IOException - { - try (InputStream is = new FileInputStream(propertyFile)) { - Properties properties = new Properties(); - properties.load(is); - return properties; - } - } - - /** - * get files path with a specific suffix from a path array - * - * @param paths paths array - * @param suffix specific suffix - * @return first path with specific suffix from that array or null if nothing found - */ - public static String getPath(String[] paths, String suffix) - { - for (String path : paths) { - if (path.endsWith(suffix)) { - return path; - } - } - return null; - } - - /** - * split the fully qualified table name into three components - * [catalog, schema, table] - * - * @param fullyQualifiedTableName table name in the form "catalog.schema.table" - * @return a string array of size 3 containing the valid catalogName, databaseName, and tableName in sequence - */ - public static String[] getTableParts(String fullyQualifiedTableName) - { - String[] parts = fullyQualifiedTableName.split("\\."); - - checkArgument(parts.length >= MINIMUM_FULLY_QUALIFIED_TABLE_FORMAT_PARTS, - INVALID_TABLE_NAME_ERR_MSG); - - String databaseName = parts[parts.length - DATABASE_NAME_OFFSET].trim(); - checkArgument(!databaseName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG); - - String catalogName = fullyQualifiedTableName - .substring(0, fullyQualifiedTableName.indexOf(databaseName) - 1).trim(); - checkArgument(!catalogName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG); - - String tableName = parts[parts.length - TABLE_NAME_OFFSET]; - checkArgument(!catalogName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG); - - return new String[]{catalogName, databaseName, tableName}; - } -} diff --git a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/util/TestIndexServiceUtils.java b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/util/TestIndexServiceUtils.java index 542b0d8e7..e3194e8dd 100644 --- a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/util/TestIndexServiceUtils.java +++ b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/util/TestIndexServiceUtils.java @@ -47,11 +47,11 @@ public class TestIndexServiceUtils String expected1 = "efg_s"; checkStringEquals(IndexServiceUtils.getPath(input1, "_s"), expected1); - String[] input2 = new String[] {"random", "character"}; - String expected2 = "character"; - checkStringEquals(IndexServiceUtils.getPath(input2, "cter"), expected2); + String[] input2 = new String[] {"random", "字符"}; + String expected2 = "字符"; + checkStringEquals(IndexServiceUtils.getPath(input2, "符"), expected2); - assertNull(IndexServiceUtils.getPath(input2, "e_char")); + assertNull(IndexServiceUtils.getPath(input2, "字字符")); } @Test @@ -115,19 +115,11 @@ public class TestIndexServiceUtils @Test public void testValidGetTableParts() { - String[] parts; - - parts = IndexServiceUtils.getTableParts("catalog.schema.table"); + String[] parts = IndexServiceUtils.getTableParts("catalog.schema.table"); assertEquals(3, parts.length); assertEquals("catalog", parts[0]); assertEquals("schema", parts[1]); assertEquals("table", parts[2]); - - parts = IndexServiceUtils.getTableParts("dc.catalog.schema.table"); - assertEquals(3, parts.length); - assertEquals("dc.catalog", parts[0]); - assertEquals("schema", parts[1]); - assertEquals("table", parts[2]); } @Test @@ -156,8 +148,8 @@ public class TestIndexServiceUtils @DataProvider(name = "invalidTableNames") public static Object[][] invalidTableNames() { - return new Object[][] {{"schema.table", false}, {".schema.table", false}, {" .schema.table", false}, - {"table", false}}; + return new Object[][] {{"dc.catalog.schema.table", false}, {"schema.table", false}, {".schema.table", false}, {" .schema.table", false}, + {"table", false}, {"catalog..table", false}, {"catalog.schema.", false}}; } @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "invalidTableNames") diff --git a/hetu-heuristic-index/src/test/java/io/hetu/core/plugin/heuristicindex/datasource/hive/TestIndexServiceUtils.java b/hetu-heuristic-index/src/test/java/io/hetu/core/plugin/heuristicindex/datasource/hive/TestIndexServiceUtils.java deleted file mode 100644 index e19129806..000000000 --- a/hetu-heuristic-index/src/test/java/io/hetu/core/plugin/heuristicindex/datasource/hive/TestIndexServiceUtils.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.hetu.core.plugin.heuristicindex.datasource.hive; - -import io.hetu.core.common.filesystem.TempFolder; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.Properties; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertTrue; - -public class TestIndexServiceUtils -{ - @Test - public void testGetPath() - { - String[] input1 = new String[]{"abc", "efg_s", "f5%3132d", "dfs_s"}; - String expected1 = "efg_s"; - checkStringEquals(IndexServiceUtils.getPath(input1, "_s"), expected1); - - String[] input2 = new String[]{"random", "字符"}; - String expected2 = "字符"; - checkStringEquals(IndexServiceUtils.getPath(input2, "符"), expected2); - - assertNull(IndexServiceUtils.getPath(input2, "字字符")); - } - - @Test - public void testFormatPathAsFolder() - { - String testStr1 = "random"; - String expected1 = "random" + File.separator; - assertTrue(IndexServiceUtils.formatPathAsFolder(testStr1).equals(expected1)); - - String testStr2 = "random" + File.separator; - String expected2 = "random" + File.separator; - assertTrue(IndexServiceUtils.formatPathAsFolder(testStr2).equals(expected2)); - } - - @Test(expectedExceptions = IllegalArgumentException.class) - public void testIsFileExisting() throws IOException - { - IndexServiceUtils.isFileExisting("/root/hetu"); - } - - @Test - public void testLoadProperties() throws IOException - { - Properties props = new Properties(); - props.setProperty("connector.name", "hive-hadoop2"); - try (TempFolder folder = new TempFolder()) { - folder.create(); - File temp = folder.newFile(); - props.store(new FileOutputStream(temp), "test"); - - Properties properties = IndexServiceUtils.loadProperties(temp); - assertEquals("hive-hadoop2", properties.getProperty("connector.name")); - } - } - - @Test - public void testGetPathReturnNull() - { - String[] inputPath = {"/root/hetu"}; - String suffix = "user"; - String path = IndexServiceUtils.getPath(inputPath, suffix); - assertNull(path); - } - - @Test - public void testGetPathWithValidValue() - { - String[] inputPath = {"/root/hetu", "/root/user"}; - String suffix = "user"; - String path = IndexServiceUtils.getPath(inputPath, suffix); - assertEquals(inputPath[1], path); - } - - @Test - public void testValidGetTableParts() - { - String[] parts; - - parts = IndexServiceUtils.getTableParts("catalog.schema.table"); - assertEquals(3, parts.length); - assertEquals("catalog", parts[0]); - assertEquals("schema", parts[1]); - assertEquals("table", parts[2]); - - parts = IndexServiceUtils.getTableParts("dc.catalog.schema.table"); - assertEquals(3, parts.length); - assertEquals("dc.catalog", parts[0]); - assertEquals("schema", parts[1]); - assertEquals("table", parts[2]); - } - - @DataProvider(name = "invalidTableNames") - public static Object[][] invalidTableNames() - { - return new Object[][]{{"schema.table", false}, {".schema.table", false}, {" .schema.table", false}, - {"table", false}}; - } - - @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "invalidTableNames") - public void testInvalidGetTableParts(String tableName, Boolean expected) - { - IndexServiceUtils.getTableParts(tableName); - } - - private void checkStringEquals(String input, String expected) - { - if (!input.equals(expected)) { - throw new AssertionError("String not matched between input: " + input + " and expected: " + expected); - } - } -} diff --git a/hetu-hive-functions/src/main/java/io/hetu/core/hive/HiveFunctionsPlugin.java b/hetu-hive-functions/src/main/java/io/hetu/core/hive/HiveFunctionsPlugin.java index 41a9307d3..70dc19d25 100644 --- a/hetu-hive-functions/src/main/java/io/hetu/core/hive/HiveFunctionsPlugin.java +++ b/hetu-hive-functions/src/main/java/io/hetu/core/hive/HiveFunctionsPlugin.java @@ -123,12 +123,10 @@ public class HiveFunctionsPlugin return functions; } - for (String funcMetadataInfo : loadFunctionMetadataFromPropertiesFile()) { - RecognizedFunctions.addRecognizedFunction(FunctionMetadata.parseFunctionClassName(funcMetadataInfo)[1]); - } - for (String funcMetadataInfo : loadFunctionMetadataFromPropertiesFile()) { try { + RecognizedFunctions.addRecognizedFunction(FunctionMetadata.parseFunctionClassName(funcMetadataInfo)[1]); + FunctionMetadata functionMetadata = new FunctionMetadata(funcMetadataInfo, this.funcClassLoader); Method[] methods = functionMetadata.getClazz().getMethods(); for (Method method : methods) {