jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!216 bug fix for creating hindex 

Merge pull request !216 from rebecca-liu66/fix-hindex-bugs
This commit is contained in:
i-robot 2020-09-17 05:25:39 +08:00 committed by Gitee
parent 35051de6b5 b089192978
commit 0479353952
8 changed files with 60 additions and 332 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hetu-heuristic-index-cli/src/main/java/io/hetu/core/heuristicindex/IndexCommand.java
View File

@ -218,12 +218,7 @@ public class IndexCommand
}
}
catch (IOException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Error occurred, please check the stacktrace for details: ", e);
}
else {
LOG.info("Error occurred. Enabled -v option for more details. {}", e.getMessage());
}
LOG.error("Error occurred, please check the stacktrace for details: ", e);
}
return null;

19
hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/HeuristicIndexWriter.java
View File

@ -109,6 +109,7 @@ public class HeuristicIndexWriter
requireNonNull(table, "no table specified");
requireNonNull(columns, "no columns specified");
requireNonNull(indexTypes, "no index types specified");
checkIndexTypes(indexTypes);
LOG.info("Creating index for: table={} columns={} partitions={}", table, Arrays.toString(columns),
partitions == null ? "all" : Arrays.toString(partitions));
@ -213,12 +214,6 @@ public class HeuristicIndexWriter
// the instances in the map are the "base" instances bc they have their properties set
// we need to create a new Index instance for each split and copy the properties the base has
Index indexTypeBaseObj = indexTypesMap.get(indexType.toLowerCase(Locale.ENGLISH));
if (indexTypeBaseObj == null) {
String msg = String.format(Locale.ENGLISH, "Index type %s not supported.", indexType);
LOG.error(msg);
throw new IllegalArgumentException(msg);
}
Index splitIndex;
try {
Constructor<? extends Index> constructor = indexTypeBaseObj.getClass().getConstructor();
@ -367,6 +362,18 @@ public class HeuristicIndexWriter
}
}
private void checkIndexTypes(String[] indexTypes)
{
for (String indexType : indexTypes) {
Index indexTypeBaseObj = indexTypesMap.get(indexType.toLowerCase(Locale.ENGLISH));
if (indexTypeBaseObj == null) {
String msg = String.format(Locale.ENGLISH, "Index type %s not supported.", indexType);
LOG.error(msg);
throw new IllegalArgumentException(msg);
}
}
}
private void cleanPartFiles(Collection<String> partFiles)
{
if (!isCleanedUp) {

47
hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/util/IndexServiceUtils.java
View File

@ -52,11 +52,13 @@ public class IndexServiceUtils
/**
* there are minimum three parts in "catalog.schema.table"1
*/
private static final int MINIMUM_FULLY_QUALIFIED_TABLE_FORMAT_PARTS = 3;
private static final int FULLY_QUALIFIED_TABLE_FORMAT_PARTS = 3;
private static final int DATABASE_NAME_OFFSET = 2;
private static final int CATALOG_NAME_INDEX = 0;
private static final int TABLE_NAME_OFFSET = 1;
private static final int DATABASE_NAME_INDEX = 1;
private static final int TABLE_NAME_INDEX = 2;
private IndexServiceUtils()
{
@ -76,6 +78,30 @@ public class IndexServiceUtils
return path;
}
/**
* check if a file with specific file path exist
*
* @param filePath filesPath that need to be checked
*/
public static void isFileExisting(String filePath) throws IOException
{
File file = Paths.get(filePath).toFile();
isFileExisting(file);
}
/**
* load properties from a filePath
*
* @param propertyFilePath property file path
* @return Property object which holds all properties
* @throws IOException when property file does NOT exist
*/
public static Properties loadProperties(String propertyFilePath) throws IOException
{
File propertyFile = Paths.get(propertyFilePath).toFile();
return loadProperties(propertyFile);
}
/**
* check if a file with specific file path exist
*
@ -130,18 +156,17 @@ public class IndexServiceUtils
{
String[] parts = fullyQualifiedTableName.split("\\.");
checkArgument(parts.length >= MINIMUM_FULLY_QUALIFIED_TABLE_FORMAT_PARTS,
checkArgument(parts.length == FULLY_QUALIFIED_TABLE_FORMAT_PARTS,
INVALID_TABLE_NAME_ERR_MSG);
String databaseName = parts[parts.length - DATABASE_NAME_OFFSET].trim();
String catalogName = parts[CATALOG_NAME_INDEX].trim();
checkArgument(!catalogName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG);
String databaseName = parts[DATABASE_NAME_INDEX].trim();
checkArgument(!databaseName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG);
String catalogName = fullyQualifiedTableName
.substring(0, fullyQualifiedTableName.indexOf(databaseName) - 1).trim();
checkArgument(!catalogName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG);
String tableName = parts[parts.length - TABLE_NAME_OFFSET];
checkArgument(!catalogName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG);
String tableName = parts[TABLE_NAME_INDEX];
checkArgument(!tableName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG);
return new String[]{catalogName, databaseName, tableName};
}

1
hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/datasource/hive/HadoopUtil.java
View File

@ -14,6 +14,7 @@
*/
package io.hetu.core.plugin.heuristicindex.datasource.hive;
import io.hetu.core.heuristicindex.util.IndexServiceUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;

156
hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/datasource/hive/IndexServiceUtils.java
View File

@ -1,156 +0,0 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.heuristicindex.datasource.hive;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Paths;
import java.util.Properties;
import static com.google.common.base.Preconditions.checkArgument;
/**
* Util class for creating external index.
*/
public class IndexServiceUtils
{
/**
* Error message for an invalid table name
*/
private static final String INVALID_TABLE_NAME_ERR_MSG = "fully qualified table name is invalid, expected 'catalog.schema.table'";
/**
* there are minimum three parts in "catalog.schema.table"1
*/
private static final int MINIMUM_FULLY_QUALIFIED_TABLE_FORMAT_PARTS = 3;
private static final int DATABASE_NAME_OFFSET = 2;
private static final int TABLE_NAME_OFFSET = 1;
private IndexServiceUtils()
{
}
/**
* format a string into path format, add file separator if it's missing
*
* @param path path need to be formatted
* @return formatted path
*/
public static String formatPathAsFolder(String path)
{
if (!path.endsWith(File.separator)) {
return path + File.separator;
}
return path;
}
/**
* check if a file with specific file path exist
*
* @param filePath filesPath that need to be checked
*/
public static void isFileExisting(String filePath) throws IOException
{
File file = Paths.get(filePath).toFile();
isFileExisting(file);
}
/**
* check if a file with specific file path exist
*
* @param file file need to be checked
*/
public static void isFileExisting(File file) throws IOException
{
checkArgument(file.exists(), file.getCanonicalPath() + " not found");
}
/**
* load properties from a filePath
*
* @param propertyFilePath property file path
* @return Property object which holds all properties
* @throws IOException when property file does NOT exist
*/
public static Properties loadProperties(String propertyFilePath) throws IOException
{
File propertyFile = Paths.get(propertyFilePath).toFile();
return loadProperties(propertyFile);
}
/**
* load properties from a file object
*
* @param propertyFile property file
* @return Property object which holds all properties
* @throws IOException when property file does NOT exist
*/
public static Properties loadProperties(File propertyFile) throws IOException
{
try (InputStream is = new FileInputStream(propertyFile)) {
Properties properties = new Properties();
properties.load(is);
return properties;
}
}
/**
* get files path with a specific suffix from a path array
*
* @param paths paths array
* @param suffix specific suffix
* @return first path with specific suffix from that array or null if nothing found
*/
public static String getPath(String[] paths, String suffix)
{
for (String path : paths) {
if (path.endsWith(suffix)) {
return path;
}
}
return null;
}
/**
* split the fully qualified table name into three components
* [catalog, schema, table]
*
* @param fullyQualifiedTableName table name in the form "catalog.schema.table"
* @return a string array of size 3 containing the valid catalogName, databaseName, and tableName in sequence
*/
public static String[] getTableParts(String fullyQualifiedTableName)
{
String[] parts = fullyQualifiedTableName.split("\\.");
checkArgument(parts.length >= MINIMUM_FULLY_QUALIFIED_TABLE_FORMAT_PARTS,
INVALID_TABLE_NAME_ERR_MSG);
String databaseName = parts[parts.length - DATABASE_NAME_OFFSET].trim();
checkArgument(!databaseName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG);
String catalogName = fullyQualifiedTableName
.substring(0, fullyQualifiedTableName.indexOf(databaseName) - 1).trim();
checkArgument(!catalogName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG);
String tableName = parts[parts.length - TABLE_NAME_OFFSET];
checkArgument(!catalogName.isEmpty(), INVALID_TABLE_NAME_ERR_MSG);
return new String[]{catalogName, databaseName, tableName};
}
}

22
hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/util/TestIndexServiceUtils.java
View File

@ -47,11 +47,11 @@ public class TestIndexServiceUtils
String expected1 = "efg_s";
checkStringEquals(IndexServiceUtils.getPath(input1, "_s"), expected1);
String[] input2 = new String[] {"random", "character"};
String expected2 = "character";
checkStringEquals(IndexServiceUtils.getPath(input2, "cter"), expected2);
String[] input2 = new String[] {"random", "字符"};
String expected2 = "字符";
checkStringEquals(IndexServiceUtils.getPath(input2, "符"), expected2);
assertNull(IndexServiceUtils.getPath(input2, "e_char"));
assertNull(IndexServiceUtils.getPath(input2, "字字符"));
}
@Test
@ -115,19 +115,11 @@ public class TestIndexServiceUtils
@Test
public void testValidGetTableParts()
{
String[] parts;
parts = IndexServiceUtils.getTableParts("catalog.schema.table");
String[] parts = IndexServiceUtils.getTableParts("catalog.schema.table");
assertEquals(3, parts.length);
assertEquals("catalog", parts[0]);
assertEquals("schema", parts[1]);
assertEquals("table", parts[2]);
parts = IndexServiceUtils.getTableParts("dc.catalog.schema.table");
assertEquals(3, parts.length);
assertEquals("dc.catalog", parts[0]);
assertEquals("schema", parts[1]);
assertEquals("table", parts[2]);
}
@Test
@ -156,8 +148,8 @@ public class TestIndexServiceUtils
@DataProvider(name = "invalidTableNames")
public static Object[][] invalidTableNames()
{
return new Object[][] {{"schema.table", false}, {".schema.table", false}, {" .schema.table", false},
{"table", false}};
return new Object[][] {{"dc.catalog.schema.table", false}, {"schema.table", false}, {".schema.table", false}, {" .schema.table", false},
{"table", false}, {"catalog..table", false}, {"catalog.schema.", false}};
}
@Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "invalidTableNames")

134
hetu-heuristic-index/src/test/java/io/hetu/core/plugin/heuristicindex/datasource/hive/TestIndexServiceUtils.java
View File

@ -1,134 +0,0 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.heuristicindex.datasource.hive;
import io.hetu.core.common.filesystem.TempFolder;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Properties;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
public class TestIndexServiceUtils
{
@Test
public void testGetPath()
{
String[] input1 = new String[]{"abc", "efg_s", "f5%3132d", "dfs_s"};
String expected1 = "efg_s";
checkStringEquals(IndexServiceUtils.getPath(input1, "_s"), expected1);
String[] input2 = new String[]{"random", "字符"};
String expected2 = "字符";
checkStringEquals(IndexServiceUtils.getPath(input2, "符"), expected2);
assertNull(IndexServiceUtils.getPath(input2, "字字符"));
}
@Test
public void testFormatPathAsFolder()
{
String testStr1 = "random";
String expected1 = "random" + File.separator;
assertTrue(IndexServiceUtils.formatPathAsFolder(testStr1).equals(expected1));
String testStr2 = "random" + File.separator;
String expected2 = "random" + File.separator;
assertTrue(IndexServiceUtils.formatPathAsFolder(testStr2).equals(expected2));
}
@Test(expectedExceptions = IllegalArgumentException.class)
public void testIsFileExisting() throws IOException
{
IndexServiceUtils.isFileExisting("/root/hetu");
}
@Test
public void testLoadProperties() throws IOException
{
Properties props = new Properties();
props.setProperty("connector.name", "hive-hadoop2");
try (TempFolder folder = new TempFolder()) {
folder.create();
File temp = folder.newFile();
props.store(new FileOutputStream(temp), "test");
Properties properties = IndexServiceUtils.loadProperties(temp);
assertEquals("hive-hadoop2", properties.getProperty("connector.name"));
}
}
@Test
public void testGetPathReturnNull()
{
String[] inputPath = {"/root/hetu"};
String suffix = "user";
String path = IndexServiceUtils.getPath(inputPath, suffix);
assertNull(path);
}
@Test
public void testGetPathWithValidValue()
{
String[] inputPath = {"/root/hetu", "/root/user"};
String suffix = "user";
String path = IndexServiceUtils.getPath(inputPath, suffix);
assertEquals(inputPath[1], path);
}
@Test
public void testValidGetTableParts()
{
String[] parts;
parts = IndexServiceUtils.getTableParts("catalog.schema.table");
assertEquals(3, parts.length);
assertEquals("catalog", parts[0]);
assertEquals("schema", parts[1]);
assertEquals("table", parts[2]);
parts = IndexServiceUtils.getTableParts("dc.catalog.schema.table");
assertEquals(3, parts.length);
assertEquals("dc.catalog", parts[0]);
assertEquals("schema", parts[1]);
assertEquals("table", parts[2]);
}
@DataProvider(name = "invalidTableNames")
public static Object[][] invalidTableNames()
{
return new Object[][]{{"schema.table", false}, {".schema.table", false}, {" .schema.table", false},
{"table", false}};
}
@Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "invalidTableNames")
public void testInvalidGetTableParts(String tableName, Boolean expected)
{
IndexServiceUtils.getTableParts(tableName);
}
private void checkStringEquals(String input, String expected)
{
if (!input.equals(expected)) {
throw new AssertionError("String not matched between input: " + input + " and expected: " + expected);
}
}
}

6
hetu-hive-functions/src/main/java/io/hetu/core/hive/HiveFunctionsPlugin.java
View File

@ -123,12 +123,10 @@ public class HiveFunctionsPlugin
return functions;
}
for (String funcMetadataInfo : loadFunctionMetadataFromPropertiesFile()) {
RecognizedFunctions.addRecognizedFunction(FunctionMetadata.parseFunctionClassName(funcMetadataInfo)[1]);
}
for (String funcMetadataInfo : loadFunctionMetadataFromPropertiesFile()) {
try {
RecognizedFunctions.addRecognizedFunction(FunctionMetadata.parseFunctionClassName(funcMetadataInfo)[1]);
FunctionMetadata functionMetadata = new FunctionMetadata(funcMetadataInfo, this.funcClassLoader);
Method[] methods = functionMetadata.getClazz().getMethods();
for (Method method : methods) {