diff --git a/hetu-docs/en/indexer/overview.md b/hetu-docs/en/indexer/overview.md index 0a6ab6ff7..d78c2fc17 100644 --- a/hetu-docs/en/indexer/overview.md +++ b/hetu-docs/en/indexer/overview.md @@ -122,6 +122,7 @@ Subsequent queries will utilize the index to reduce the amount of data read | hetu.heuristicindex.filter.cache.loading-delay | 10s | No | The delay to wait before async loading task starts to load index cache from indexstore| | hetu.heuristicindex.indexstore.uri | /opt/hetu/indices/ | No | Directory under which all index files are stored| | hetu.heuristicindex.indexstore.filesystem.profile | local-config-default| No | This property defines the filesystem profile used to read and write index| +| hetu.heuristicindex.filter.cache.preload.indices | | No | Preload the specified indices (comma-separated) when the server starts. Put `ALL` to load all indices| Heuristic indexer now uses Hetu Metastore to manage its metadata. Please check [Hetu Metastore](../admin/meta-store.md) for more information. diff --git a/hetu-docs/zh/indexer/overview.md b/hetu-docs/zh/indexer/overview.md index 0e40b5a3d..a1d49b00e 100644 --- a/hetu-docs/zh/indexer/overview.md +++ b/hetu-docs/zh/indexer/overview.md @@ -106,6 +106,7 @@ | hetu.heuristicindex.filter.cache.loading-delay | 10s | 否 | 在异步加载索引到缓存前等待的时长| | hetu.heuristicindex.indexstore.uri | /opt/hetu/indices/ | 否 | 所有索引文件存储的目录| | hetu.heuristicindex.indexstore.filesystem.profile | local-config-default| 否 | 用于存储索引文件的文件系统属性描述文件名称| +| hetu.heuristicindex.filter.cache.preload.indices | | 否 | 在服务器启动时预加载指定名称的索引(用逗号分隔), 当值为`ALL`时将预载入全部索引| 索引功能现使用Hetu Metastore管理元数据。请参阅 [Hetu Metastore](../admin/meta-store.md) 获取关于如何配置的更多信息。 diff --git a/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/HeuristicIndexClient.java b/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/HeuristicIndexClient.java index 45f4b7104..e47c280a6 100644 --- a/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/HeuristicIndexClient.java +++ b/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/HeuristicIndexClient.java @@ -347,7 +347,7 @@ public class HeuristicIndexClient String column = indexKeyPath.subpath(1, 2).toString(); List result = new ArrayList<>(); if (fs.exists(absolutePath)) { - List paths = fs.list(absolutePath).collect(Collectors.toList()); + List paths = fs.walk(absolutePath).filter(p -> !fs.isDirectory(p)).collect(Collectors.toList()); for (Path filePath : paths) { BTreeIndex index = new BTreeIndex(); InputStream inputStream = fs.newInputStream(filePath); diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/IndexCache.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/IndexCache.java index 690001e39..3879cb358 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/IndexCache.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/IndexCache.java @@ -22,10 +22,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Inject; import io.airlift.log.Logger; -import io.hetu.core.common.heuristicindex.IndexCacheKey; import io.prestosql.plugin.hive.HiveColumnHandle; import io.prestosql.plugin.hive.HiveSplit; import io.prestosql.spi.HetuConstant; +import io.prestosql.spi.heuristicindex.IndexCacheKey; import io.prestosql.spi.heuristicindex.IndexClient; import io.prestosql.spi.heuristicindex.IndexMetadata; import io.prestosql.spi.heuristicindex.IndexNotCreatedException; diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/IndexCacheLoader.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/IndexCacheLoader.java index 6d45c7531..9aa3a3578 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/IndexCacheLoader.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/IndexCacheLoader.java @@ -16,7 +16,7 @@ package io.prestosql.plugin.hive.util; import com.google.common.cache.CacheLoader; import com.google.inject.Inject; -import io.hetu.core.common.heuristicindex.IndexCacheKey; +import io.prestosql.spi.heuristicindex.IndexCacheKey; import io.prestosql.spi.heuristicindex.IndexClient; import io.prestosql.spi.heuristicindex.IndexMetadata; import io.prestosql.spi.heuristicindex.IndexNotCreatedException; diff --git a/presto-hive/src/test/java/io/prestosql/plugin/hive/util/TestIndexCacheLoader.java b/presto-hive/src/test/java/io/prestosql/plugin/hive/util/TestIndexCacheLoader.java index c5e2844c6..4f74d108c 100644 --- a/presto-hive/src/test/java/io/prestosql/plugin/hive/util/TestIndexCacheLoader.java +++ b/presto-hive/src/test/java/io/prestosql/plugin/hive/util/TestIndexCacheLoader.java @@ -15,8 +15,8 @@ package io.prestosql.plugin.hive.util; import io.airlift.units.Duration; -import io.hetu.core.common.heuristicindex.IndexCacheKey; import io.prestosql.spi.HetuConstant; +import io.prestosql.spi.heuristicindex.IndexCacheKey; import io.prestosql.spi.heuristicindex.IndexClient; import io.prestosql.spi.heuristicindex.IndexMetadata; import io.prestosql.spi.service.PropertyService; diff --git a/presto-main/src/main/java/io/prestosql/heuristicindex/HeuristicIndexerManager.java b/presto-main/src/main/java/io/prestosql/heuristicindex/HeuristicIndexerManager.java index f3b379458..6b1316d75 100644 --- a/presto-main/src/main/java/io/prestosql/heuristicindex/HeuristicIndexerManager.java +++ b/presto-main/src/main/java/io/prestosql/heuristicindex/HeuristicIndexerManager.java @@ -39,6 +39,7 @@ import java.nio.file.FileSystemException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -131,6 +132,25 @@ public class HeuristicIndexerManager } } + public void preloadIndex() + throws IOException + { + if (indexClient != null) { + if (PropertyService.containsProperty(HetuConstant.FILTER_CACHE_PRELOAD_INDICES)) { + String preloadNames = PropertyService.getStringProperty(HetuConstant.FILTER_CACHE_PRELOAD_INDICES); + List preloadNameList = Arrays.asList(preloadNames.split(",")); + if (!preloadNameList.isEmpty()) { + try { + SplitFiltering.preloadCache(indexClient, preloadNameList); + } + catch (Exception e) { + LOG.info("Error preloading index: " + e); + } + } + } + } + } + public void cleanUpIndexRecord(QueryInfo queryInfo) { try { diff --git a/presto-main/src/main/java/io/prestosql/heuristicindex/IndexCache.java b/presto-main/src/main/java/io/prestosql/heuristicindex/IndexCache.java index c58d59ff3..614043cab 100644 --- a/presto-main/src/main/java/io/prestosql/heuristicindex/IndexCache.java +++ b/presto-main/src/main/java/io/prestosql/heuristicindex/IndexCache.java @@ -21,10 +21,10 @@ import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.airlift.log.Logger; -import io.hetu.core.common.heuristicindex.IndexCacheKey; import io.prestosql.metadata.Split; import io.prestosql.spi.HetuConstant; import io.prestosql.spi.heuristicindex.Index; +import io.prestosql.spi.heuristicindex.IndexCacheKey; import io.prestosql.spi.heuristicindex.IndexClient; import io.prestosql.spi.heuristicindex.IndexMetadata; import io.prestosql.spi.heuristicindex.IndexNotCreatedException; @@ -33,6 +33,9 @@ import io.prestosql.spi.service.PropertyService; import java.io.IOException; import java.net.URI; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -44,6 +47,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import static io.prestosql.spi.HetuConstant.KILOBYTE; +import static io.prestosql.spi.heuristicindex.IndexCacheKey.LAST_MODIFIED_TIME_PLACE_HOLDER; public class IndexCache { @@ -67,14 +71,18 @@ public class IndexCache int numThreads = Math.min(Runtime.getRuntime().availableProcessors(), PropertyService.getLongProperty(HetuConstant.FILTER_CACHE_LOADING_THREADS).intValue()); executor = Executors.newScheduledThreadPool(numThreads, threadFactory); CacheBuilder> cacheBuilder = CacheBuilder.newBuilder() - .removalListener(e -> ((List) e.getValue()).stream().forEach(i -> { + .removalListener(e -> { try { - i.getIndex().close(); + if (!((IndexCacheKey) e.getKey()).skipCloseIndex()) { + for (IndexMetadata i : ((List) e.getValue())) { + i.getIndex().close(); + } + } } catch (IOException ioException) { - LOG.debug(ioException, "Failed to close index " + i); + LOG.debug(ioException, "Failed to close index:", e); } - })) + }) .expireAfterWrite(PropertyService.getDurationProperty(HetuConstant.FILTER_CACHE_TTL).toMillis(), TimeUnit.MILLISECONDS) .maximumWeight(PropertyService.getLongProperty(HetuConstant.FILTER_CACHE_MAX_MEMORY)) .weigher((indexCacheKey, indices) -> { @@ -127,6 +135,59 @@ public class IndexCache } } + public void preloadIndex(String table, String column, String type, Index.Level level) + { + String filterKeyPath = table + "/" + column + "/" + type; + IndexCacheKey filterKey = new IndexCacheKey(filterKeyPath, LAST_MODIFIED_TIME_PLACE_HOLDER, level); + filterKey.setNoCloseFlag(true); + executor.schedule(() -> { + List allLoaded; + try { + // Load index for the whole table with dummy last modified time first + allLoaded = cache.get(filterKey); + // Then 1. replace the filterKey with the actual last modified time read from index + // 2. for PARTITION and STRIPE index, the loaded whole table index should also be broken to stripe/partition indices + switch (level) { + case STRIPE: + // break index key from table/column/type to several table/column/type/split-path + for (IndexMetadata index : allLoaded) { + String indexUri = index.getUri(); + IndexCacheKey newKey = new IndexCacheKey(filterKeyPath + indexUri, index.getLastModifiedTime()); + cache.asMap().putIfAbsent(newKey, new ArrayList<>()); + cache.asMap().get(newKey).add(index); + } + cache.invalidate(filterKey); + break; + case PARTITION: + // break index key from table/column/type to several table/column/type/partition + for (IndexMetadata index : allLoaded) { + Path indexUri = Paths.get(index.getUri()); + String partition = null; + // get partition name from path if present + for (int i = indexUri.getNameCount() - 1; i >= 0; i--) { + if (indexUri.getName(i).toString().contains("=")) { + partition = indexUri.getName(i).toString(); + break; + } + } + if (partition != null) { + IndexCacheKey newKey = new IndexCacheKey(filterKeyPath + "/" + partition, index.getLastModifiedTime()); + cache.asMap().putIfAbsent(newKey, new ArrayList<>()); + cache.asMap().get(newKey).add(index); + } + } + cache.invalidate(filterKey); + break; + case TABLE: + // no need to break index, and lastModifiedTime is not used for TABLE level. no need to do anything + } + } + catch (ExecutionException e) { + LOG.debug("Failed to load into cache: " + filterKey, e); + } + }, 0, TimeUnit.MILLISECONDS); + } + public List getIndices(String table, String column, Split split) { if (cache == null) { @@ -192,7 +253,7 @@ public class IndexCache if (!partitions.isEmpty()) { for (String partition : partitions) { String filterKeyPath = table + "/" + column + "/" + indexType + "/" + partition; - IndexCacheKey filterKey = new IndexCacheKey(filterKeyPath, lastModifiedTime, Index.Level.PARTITION.name()); + IndexCacheKey filterKey = new IndexCacheKey(filterKeyPath, lastModifiedTime, Index.Level.PARTITION); List result = loadIndex(filterKey); if (result != null) { indices.addAll(result); @@ -206,7 +267,7 @@ public class IndexCache } String filterKeyPath = table + "/" + column + "/" + indexType; - IndexCacheKey filterKey = new IndexCacheKey(filterKeyPath, lastModifiedTime, Index.Level.PARTITION.name()); + IndexCacheKey filterKey = new IndexCacheKey(filterKeyPath, lastModifiedTime, Index.Level.TABLE); List result = loadIndex(filterKey); if (result != null) { indices.addAll(result); diff --git a/presto-main/src/main/java/io/prestosql/heuristicindex/IndexCacheLoader.java b/presto-main/src/main/java/io/prestosql/heuristicindex/IndexCacheLoader.java index ce941633a..e6dd659e5 100644 --- a/presto-main/src/main/java/io/prestosql/heuristicindex/IndexCacheLoader.java +++ b/presto-main/src/main/java/io/prestosql/heuristicindex/IndexCacheLoader.java @@ -15,8 +15,8 @@ package io.prestosql.heuristicindex; import com.google.common.cache.CacheLoader; -import io.hetu.core.common.heuristicindex.IndexCacheKey; import io.prestosql.spi.heuristicindex.Index; +import io.prestosql.spi.heuristicindex.IndexCacheKey; import io.prestosql.spi.heuristicindex.IndexClient; import io.prestosql.spi.heuristicindex.IndexMetadata; import io.prestosql.spi.heuristicindex.IndexNotCreatedException; @@ -24,6 +24,7 @@ import io.prestosql.spi.heuristicindex.IndexNotCreatedException; import java.util.List; import java.util.stream.Collectors; +import static io.prestosql.spi.heuristicindex.IndexCacheKey.LAST_MODIFIED_TIME_PLACE_HOLDER; import static java.util.Comparator.comparingLong; import static java.util.Objects.requireNonNull; @@ -43,7 +44,7 @@ public class IndexCacheLoader { requireNonNull(key); requireNonNull(indexClient); - if (Index.Level.PARTITION.name().equals(key.getIndexLevel())) { + if (key.getIndexLevel() == Index.Level.PARTITION || key.getIndexLevel() == Index.Level.TABLE) { return loadPartitionIndex(key); } else { @@ -57,19 +58,22 @@ public class IndexCacheLoader requireNonNull(key); requireNonNull(indexClient); - // only load index files if index lastModified matches key lastModified - long lastModified; + // only perform last modified time check if the key last modified time is not set to "skip" + if (key.getLastModifiedTime() != LAST_MODIFIED_TIME_PLACE_HOLDER) { + // only load index files if index lastModified matches key lastModified + long lastModified; - try { - lastModified = indexClient.getLastModified(key.getPath()); - } - catch (Exception e) { - // no lastModified file found, i.e. index doesn't exist - throw new IndexNotCreatedException(); - } + try { + lastModified = indexClient.getLastModified(key.getPath()); + } + catch (Exception e) { + // no lastModified file found, i.e. index doesn't exist + throw new IndexNotCreatedException(); + } - if (lastModified != key.getLastModifiedTime()) { - throw new Exception("Index file(s) are expired for key " + key); + if (lastModified != key.getLastModifiedTime()) { + throw new Exception("Index file(s) are expired for key " + key); + } } List indices; diff --git a/presto-main/src/main/java/io/prestosql/heuristicindex/SplitFiltering.java b/presto-main/src/main/java/io/prestosql/heuristicindex/SplitFiltering.java index c68ae20dc..29eb30ad3 100644 --- a/presto-main/src/main/java/io/prestosql/heuristicindex/SplitFiltering.java +++ b/presto-main/src/main/java/io/prestosql/heuristicindex/SplitFiltering.java @@ -18,11 +18,13 @@ import com.google.common.cache.CacheLoader; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import io.airlift.log.Logger; -import io.hetu.core.common.heuristicindex.IndexCacheKey; import io.prestosql.execution.SqlStageExecution; import io.prestosql.metadata.Split; import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.CreateIndexMetadata; import io.prestosql.spi.function.OperatorType; +import io.prestosql.spi.heuristicindex.Index; +import io.prestosql.spi.heuristicindex.IndexCacheKey; import io.prestosql.spi.heuristicindex.IndexClient; import io.prestosql.spi.heuristicindex.IndexFilter; import io.prestosql.spi.heuristicindex.IndexLookUpException; @@ -56,6 +58,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Queue; @@ -75,6 +78,8 @@ public class SplitFiltering private static final Set INVERTED_INDEX = Sets.newHashSet("BTREE"); private static final String MAX_MODIFIED_TIME = "__hetu__maxmodifiedtime"; private static final String TABLE_LEVEL_KEY = "__index__is__table__level__"; + private static final String PRELOAD_ALL_KEY = "ALL"; + private static IndexCache indexCache; private SplitFiltering() @@ -89,6 +94,38 @@ public class SplitFiltering } } + public static void preloadCache(IndexClient indexClient, List preloadIndexNames) + throws IOException + { + if (indexCache == null) { + initCache(indexClient); + } + + List indexToPreload = new ArrayList<>(preloadIndexNames.size()); + + if (preloadIndexNames.contains(PRELOAD_ALL_KEY)) { + indexToPreload = indexClient.getAllIndexRecords(); + LOG.info("Preloading all indices : " + indexToPreload.stream().map(r -> r.name).collect(Collectors.joining(","))); + } + else { + for (String indexName : preloadIndexNames) { + IndexRecord record = indexClient.lookUpIndexRecord(indexName); + if (record != null) { + indexToPreload.add(indexClient.lookUpIndexRecord(indexName)); + } + else { + LOG.info("Index " + indexName + " is not found. Preloading skipped."); + } + } + } + + for (IndexRecord record : indexToPreload) { + LOG.info("Preloading index for split filtering: " + record); + Index.Level indexLevel = Index.Level.valueOf(record.getProperty(CreateIndexMetadata.LEVEL_PROP_KEY).toUpperCase(Locale.ROOT)); + indexCache.preloadIndex(record.qualifiedTable, String.join(",", record.columns), record.indexType, indexLevel); + } + } + public static List getFilteredSplit(Optional expression, Optional tableName, Map assignments, SplitSource.SplitBatch nextSplits, HeuristicIndexerManager heuristicIndexerManager) { diff --git a/presto-main/src/main/java/io/prestosql/metadata/SessionPropertyManager.java b/presto-main/src/main/java/io/prestosql/metadata/SessionPropertyManager.java index 3bb53dc7c..779c018cb 100644 --- a/presto-main/src/main/java/io/prestosql/metadata/SessionPropertyManager.java +++ b/presto-main/src/main/java/io/prestosql/metadata/SessionPropertyManager.java @@ -365,6 +365,7 @@ public final class SessionPropertyManager PropertyService.setProperty(HetuConstant.FILTER_CACHE_SOFT_REFERENCE, hetuConfig.isIndexCacheSoftReferenceEnabled()); PropertyService.setProperty(HetuConstant.INDEXSTORE_URI, hetuConfig.getIndexStoreUri()); PropertyService.setProperty(HetuConstant.INDEXSTORE_FILESYSTEM_PROFILE, hetuConfig.getIndexStoreFileSystemProfile()); + PropertyService.setProperty(HetuConstant.FILTER_CACHE_PRELOAD_INDICES, hetuConfig.getIndexToPreload()); } } diff --git a/presto-main/src/main/java/io/prestosql/server/PrestoServer.java b/presto-main/src/main/java/io/prestosql/server/PrestoServer.java index 5e630847c..e10b62054 100755 --- a/presto-main/src/main/java/io/prestosql/server/PrestoServer.java +++ b/presto-main/src/main/java/io/prestosql/server/PrestoServer.java @@ -153,6 +153,10 @@ public class PrestoServer // State Store launchEmbeddedStateStore(injector.getInstance(HetuConfig.class), injector.getInstance(StateStoreLauncher.class)); injector.getInstance(StateStoreProvider.class).loadStateStore(); + // preload index (on coordinator only) + if (injector.getInstance(ServerConfig.class).isCoordinator()) { + injector.getInstance(HeuristicIndexerManager.class).preloadIndex(); + } // register dynamic filter listener registerStateStoreListeners( injector.getInstance(StateStoreListenerManager.class), diff --git a/presto-main/src/main/java/io/prestosql/utils/HetuConfig.java b/presto-main/src/main/java/io/prestosql/utils/HetuConfig.java index 91bdff99f..93d41c4b5 100644 --- a/presto-main/src/main/java/io/prestosql/utils/HetuConfig.java +++ b/presto-main/src/main/java/io/prestosql/utils/HetuConfig.java @@ -42,6 +42,7 @@ public class HetuConfig private Boolean indexCacheSoftReference = Boolean.TRUE; private String indexStoreUri = "/opt/hetu/indices/"; private String indexStoreFileSystemProfile = "local-config-default"; + private String indexToPreload; private Boolean enableEmbeddedStateStore = Boolean.FALSE; private Boolean enableMultipleCoordinator = Boolean.FALSE; private Duration stateUpdateInterval = new Duration(100, TimeUnit.MILLISECONDS); @@ -116,6 +117,19 @@ public class HetuConfig return this; } + public String getIndexToPreload() + { + return indexToPreload; + } + + @Config(HetuConstant.FILTER_CACHE_PRELOAD_INDICES) + @ConfigDescription("Comma separated list of index names to preload when server starts, or ALL to load all indices") + public HetuConfig setIndexToPreload(String indices) + { + this.indexToPreload = indices; + return this; + } + public Duration getIndexCacheLoadingDelay() { return this.indexCacheLoadingDelay; diff --git a/presto-main/src/test/java/io/prestosql/heuristicindex/TestIndexCacheLoader.java b/presto-main/src/test/java/io/prestosql/heuristicindex/TestIndexCacheLoader.java index c3bf389bd..2d9742112 100644 --- a/presto-main/src/test/java/io/prestosql/heuristicindex/TestIndexCacheLoader.java +++ b/presto-main/src/test/java/io/prestosql/heuristicindex/TestIndexCacheLoader.java @@ -14,8 +14,8 @@ */ package io.prestosql.heuristicindex; -import io.hetu.core.common.heuristicindex.IndexCacheKey; import io.prestosql.spi.HetuConstant; +import io.prestosql.spi.heuristicindex.IndexCacheKey; import io.prestosql.spi.heuristicindex.IndexClient; import io.prestosql.spi.heuristicindex.IndexMetadata; import io.prestosql.spi.service.PropertyService; diff --git a/presto-main/src/test/java/io/prestosql/utils/TestHetuConfig.java b/presto-main/src/test/java/io/prestosql/utils/TestHetuConfig.java index b6fcc77a2..20af0e1fb 100644 --- a/presto-main/src/test/java/io/prestosql/utils/TestHetuConfig.java +++ b/presto-main/src/test/java/io/prestosql/utils/TestHetuConfig.java @@ -52,7 +52,8 @@ public class TestHetuConfig .setDataCenterConsumerTimeout(new Duration(10, TimeUnit.MINUTES)) .setSplitCacheMapEnabled(false) .setSplitCacheStateUpdateInterval(new Duration(2, TimeUnit.SECONDS)) - .setTraceStackVisible(false)); + .setTraceStackVisible(false) + .setIndexToPreload(null)); } @Test @@ -81,6 +82,7 @@ public class TestHetuConfig .put("hetu.split-cache-map.enabled", "true") .put("hetu.split-cache-map.state-update-interval", "5s") .put("stack-trace-visible", "true") + .put("hetu.heuristicindex.filter.cache.preload-indices", "idx1,idx2") .build(); HetuConfig expected = new HetuConfig() @@ -105,7 +107,8 @@ public class TestHetuConfig .setDataCenterConsumerTimeout(new Duration(5, TimeUnit.MINUTES)) .setSplitCacheMapEnabled(true) .setSplitCacheStateUpdateInterval(new Duration(5, TimeUnit.SECONDS)) - .setTraceStackVisible(true); + .setTraceStackVisible(true) + .setIndexToPreload("idx1,idx2"); ConfigAssertions.assertFullMapping(properties, expected); } diff --git a/presto-spi/src/main/java/io/prestosql/spi/HetuConstant.java b/presto-spi/src/main/java/io/prestosql/spi/HetuConstant.java index 9459b16d7..61ed613af 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/HetuConstant.java +++ b/presto-spi/src/main/java/io/prestosql/spi/HetuConstant.java @@ -29,6 +29,7 @@ public class HetuConstant public static final String FILTER_CACHE_LOADING_DELAY = "hetu.heuristicindex.filter.cache.loading-delay"; public static final String FILTER_CACHE_TTL = "hetu.heuristicindex.filter.cache.ttl"; public static final String FILTER_CACHE_SOFT_REFERENCE = "hetu.heuristicindex.filter.cache.soft-reference"; + public static final String FILTER_CACHE_PRELOAD_INDICES = "hetu.heuristicindex.filter.cache.preload-indices"; public static final String INDEXSTORE_URI = "hetu.heuristicindex.indexstore.uri"; public static final String INDEXSTORE_FILESYSTEM_PROFILE = "hetu.heuristicindex.indexstore.filesystem.profile"; public static final String DATA_CENTER_CONNECTOR_NAME = "dc"; diff --git a/presto-spi/src/main/java/io/prestosql/spi/heuristicindex/Index.java b/presto-spi/src/main/java/io/prestosql/spi/heuristicindex/Index.java index 4d57e9617..56d7720ba 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/heuristicindex/Index.java +++ b/presto-spi/src/main/java/io/prestosql/spi/heuristicindex/Index.java @@ -227,7 +227,6 @@ public interface Index default void close() throws IOException { - return; } enum Level diff --git a/hetu-common/src/main/java/io/hetu/core/common/heuristicindex/IndexCacheKey.java b/presto-spi/src/main/java/io/prestosql/spi/heuristicindex/IndexCacheKey.java similarity index 73% rename from hetu-common/src/main/java/io/hetu/core/common/heuristicindex/IndexCacheKey.java rename to presto-spi/src/main/java/io/prestosql/spi/heuristicindex/IndexCacheKey.java index 2320fe395..8ece54e4f 100644 --- a/hetu-common/src/main/java/io/hetu/core/common/heuristicindex/IndexCacheKey.java +++ b/presto-spi/src/main/java/io/prestosql/spi/heuristicindex/IndexCacheKey.java @@ -12,22 +12,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.common.heuristicindex; +package io.prestosql.spi.heuristicindex; import java.util.Objects; public class IndexCacheKey { - private String path; - private long lastModifiedTime; - private String indexLevel = "STRIPE"; + public static final long LAST_MODIFIED_TIME_PLACE_HOLDER = 0; + + private final String path; + private final long lastModifiedTime; + private final Index.Level indexLevel; + private boolean noCloseFlag; /** - * @param path path to the file the index files should be read for + * @param path path to the file the index files should be read for * @param lastModifiedTime lastModifiedTime of the file, used to validate the indexes - * @param indexLevel see Index.Level in presto-spi + * @param indexLevel see Index.Level in presto-spi */ - public IndexCacheKey(String path, long lastModifiedTime, String indexLevel) + public IndexCacheKey(String path, long lastModifiedTime, Index.Level indexLevel) { this.path = path; this.lastModifiedTime = lastModifiedTime; @@ -42,7 +45,7 @@ public class IndexCacheKey */ public IndexCacheKey(String path, long lastModifiedTime) { - this(path, lastModifiedTime, "STRIPE"); + this(path, lastModifiedTime, Index.Level.STRIPE); } public String getPath() @@ -55,11 +58,21 @@ public class IndexCacheKey return lastModifiedTime; } - public String getIndexLevel() + public Index.Level getIndexLevel() { return this.indexLevel; } + public void setNoCloseFlag(boolean flag) + { + this.noCloseFlag = true; + } + + public boolean skipCloseIndex() + { + return noCloseFlag; + } + // only the path should be used as the key // the lastModifiedTime time is only used to check if index is valid @Override diff --git a/presto-spi/src/main/java/io/prestosql/spi/heuristicindex/IndexRecord.java b/presto-spi/src/main/java/io/prestosql/spi/heuristicindex/IndexRecord.java index 312292c20..03d9b2864 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/heuristicindex/IndexRecord.java +++ b/presto-spi/src/main/java/io/prestosql/spi/heuristicindex/IndexRecord.java @@ -144,6 +144,17 @@ public class IndexRecord return this.properties.stream().anyMatch(property -> property.startsWith(INPROGRESS_PROPERTY_KEY)); } + public String getProperty(String key) + { + for (String property : properties) { + if (property.toLowerCase(Locale.ROOT).startsWith(key.toLowerCase(Locale.ROOT))) { + String[] entry = property.split("="); + return entry[1]; + } + } + return null; + } + @Override public boolean equals(Object o) {