jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!466 Fix index QA bugs 

Merge pull request !466 from Han_Weng/index-fix-qa-1216
This commit is contained in:
i-robot 2020-12-18 05:55:48 +08:00 committed by Gitee
parent a20791bc0e 17df4298f9
commit 313ef45375
10 changed files with 17 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/FileIndexWriter.java
View File

@ -39,7 +39,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
@ -213,13 +212,13 @@ public class FileIndexWriter
}
// Create index and put values
try (Index index = HeuristicIndexFactory.createIndex(createIndexMetadata.getIndexType().toLowerCase(Locale.ENGLISH))) {
try (Index index = HeuristicIndexFactory.createIndex(createIndexMetadata.getIndexType())) {
index.setProperties(createIndexMetadata.getProperties());
index.setExpectedNumOfEntries(expectedNumEntries);
index.addValues(stripeData);
// Persist one index (e.g. 3.bloom)
String indexFileName = offset + "." + index.getId().toLowerCase(Locale.ENGLISH);
String indexFileName = offset + "." + index.getId();
try (OutputStream os = LOCAL_FS_CLIENT.newOutputStream(tmpPath.resolve(indexFileName))) {
index.serialize(os);
}

5
hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/HeuristicIndexClient.java
View File

@ -17,8 +17,6 @@ package io.hetu.core.heuristicindex;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import io.airlift.log.Logger;
import io.hetu.core.filesystem.HetuLocalFileSystemClient;
import io.hetu.core.filesystem.LocalConfig;
import io.hetu.core.heuristicindex.util.IndexConstants;
import io.hetu.core.plugin.heuristicindex.index.btree.BTreeIndex;
import io.prestosql.spi.connector.CreateIndexMetadata;
@ -42,7 +40,6 @@ import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.locks.Lock;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -58,8 +55,6 @@ import static java.util.Objects.requireNonNull;
public class HeuristicIndexClient
implements IndexClient
{
private static final HetuFileSystemClient LOCAL_FS_CLIENT = new HetuLocalFileSystemClient(
new LocalConfig(new Properties()), Paths.get("/"));
private static final Logger LOG = Logger.get(HeuristicIndexClient.class);
private HetuFileSystemClient fs;

10
hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestIndexRecordManager.java
View File

@ -216,12 +216,12 @@ public class TestIndexRecordManager
@Test
public void testAddAndLookUp()
throws IOException, IllegalAccessException
throws IOException
{
testIndexRecordAddLookUpHelper("testName", "testUser", "testTable", new String[] {"testColumn"}, "minmax", Collections.emptyList(), Collections.emptyList());
testIndexRecordAddLookUpHelper("testName", "testUser", "testTable", new String[] {"testColumn", "testColumn2"}, "minmax", Collections.emptyList(), Collections.emptyList());
testIndexRecordAddLookUpHelper("testName", "testUser", "testTable", new String[] {"testColumn"}, "minmax", Collections.emptyList(), ImmutableList.of("12"));
testIndexRecordAddLookUpHelper("testName", "testUser", "testTable", new String[] {"testColumn"}, "minmax", Collections.emptyList(), ImmutableList.of("12", "123"));
testIndexRecordAddLookUpHelper("testName", "testUser", "testTable", new String[] {"testColumn"}, "MINMAX", Collections.emptyList(), Collections.emptyList());
testIndexRecordAddLookUpHelper("testName", "testUser", "testTable", new String[] {"testColumn", "testColumn2"}, "MINMAX", Collections.emptyList(), Collections.emptyList());
testIndexRecordAddLookUpHelper("testName", "testUser", "testTable", new String[] {"testColumn"}, "MINMAX", Collections.emptyList(), ImmutableList.of("12"));
testIndexRecordAddLookUpHelper("testName", "testUser", "testTable", new String[] {"testColumn"}, "MINMAX", Collections.emptyList(), ImmutableList.of("12", "123"));
}
private void testIndexRecordAddLookUpHelper(String name, String user, String table, String[] columns, String indexType, List<String> indexProperties, List<String> partitions)

2
presto-hive/src/main/java/io/prestosql/plugin/hive/util/IndexCache.java
View File

@ -52,7 +52,7 @@ public class IndexCache
{
private static final Logger LOG = Logger.get(IndexCache.class);
private static final ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("Hive-IndexCache-pool-%d").setDaemon(true).build();
protected static final List<String> INDEX_TYPES = ImmutableList.of("minmax", "bloom", "bitmap");
protected static final List<String> INDEX_TYPES = ImmutableList.of("MINMAX", "BLOOM", "BITMAP");
private static ScheduledExecutorService executor;

2
presto-main/src/main/java/io/prestosql/heuristicindex/IndexCache.java
View File

@ -49,7 +49,7 @@ public class IndexCache
{
private static final Logger LOG = Logger.get(IndexCache.class);
private static final ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("Main-IndexCache-pool-%d").setDaemon(true).build();
protected static final List<String> INDEX_TYPES = ImmutableList.of("bloom", "minmax");
protected static final List<String> INDEX_TYPES = ImmutableList.of("BLOOM", "MINMAX");
private static ScheduledExecutorService executor;

3
presto-main/src/main/java/io/prestosql/query/CachedSqlQueryExecution.java
View File

@ -157,7 +157,8 @@ public class CachedSqlQueryExecution
isExecutionPlanCacheEnabled(session) &&
analysis.getParameters().isEmpty() &&
validateAndExtractTableAndColumns(analysis, metadata, session, tableNames, tableStatistics, columnTypes) &&
isCacheable(statement);
isCacheable(statement) &&
(!(analysis.getOriginalStatement() instanceof CreateIndex)); // create index should not be cached
cacheable = cacheable && !tableNames.isEmpty();
if (!cacheable) {

1
presto-main/src/main/java/io/prestosql/security/IndexAccessControlRule.java
View File

@ -1,4 +1,5 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at

1
presto-main/src/main/java/io/prestosql/testing/NoOpIndexWriter.java
View File

@ -1,4 +1,5 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at

3
presto-spi/src/main/java/io/prestosql/spi/connector/CreateIndexMetadata.java
View File

@ -20,6 +20,7 @@ import io.prestosql.spi.heuristicindex.Index;
import io.prestosql.spi.type.Type;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
@ -54,7 +55,7 @@ public class CreateIndexMetadata
{
this.indexName = checkNotEmpty(indexName, "indexName");
this.tableName = requireNonNull(tableName, "tableName is null");
this.indexType = requireNonNull(indexType, "indexType is null");
this.indexType = requireNonNull(indexType, "indexType is null").toUpperCase(Locale.ENGLISH);
this.indexColumns = indexColumns;
this.partitions = partitions;
this.properties = properties;

3
presto-spi/src/main/java/io/prestosql/spi/heuristicindex/IndexRecord.java
View File

@ -16,6 +16,7 @@ package io.prestosql.spi.heuristicindex;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.stream.Collectors;
@ -39,7 +40,7 @@ public class IndexRecord
this.user = user == null ? "" : user;
this.table = table;
this.columns = columns;
this.indexType = indexType;
this.indexType = indexType.toUpperCase(Locale.ENGLISH);
this.properties = properties;
this.partitions = partitions;
this.lastModifiedTime = System.currentTimeMillis();