From 543a7e6de804f291f6ffca913784d922a585aa90 Mon Sep 17 00:00:00 2001 From: Han Weng Date: Mon, 22 Mar 2021 15:10:03 -0400 Subject: [PATCH] Improve index lookup performance and optimize temp folder usage --- .../core/common/filesystem/TempFolder.java | 37 +++++++++++++------ .../hbase/metadata/TestingHetuMetastore.java | 1 + .../heuristicindex/IndexRecordManager.java | 35 +++++++++--------- .../index/btree/BTreeIndex.java | 3 +- .../metastore/HetuMetastoreCacheConfig.java | 4 +- .../TestHetuMetastoreCacheConfig.java | 3 +- .../metastore/TestHetuMetastoreModule.java | 1 + .../hetu/core/plugin/vdm/VdmQueryRunner.java | 1 + .../plugin/hive/HiveQueryRunner.java | 1 + 9 files changed, 51 insertions(+), 35 deletions(-) diff --git a/hetu-common/src/main/java/io/hetu/core/common/filesystem/TempFolder.java b/hetu-common/src/main/java/io/hetu/core/common/filesystem/TempFolder.java index e267bb724..976e8b873 100644 --- a/hetu-common/src/main/java/io/hetu/core/common/filesystem/TempFolder.java +++ b/hetu-common/src/main/java/io/hetu/core/common/filesystem/TempFolder.java @@ -24,14 +24,23 @@ import java.util.UUID; public class TempFolder implements AutoCloseable { + private final String prefix; private File root; - public TempFolder() {} + public TempFolder() + { + this(""); + } + + public TempFolder(String user) + { + this.prefix = "hetu-tmp-folder-" + user; + } public TempFolder create() throws IOException { - root = Files.createTempDirectory("hetu-tmp-folder-").toFile(); + root = Files.createTempDirectory(prefix).toFile(); return this; } @@ -72,6 +81,20 @@ public class TempFolder throw new IOException("Not able to create folder " + relativePath); } + @Override + public void close() + { + if (root != null && root.exists()) { + // retry deletion for 3 times + for (int i = 0; i < 3; i++) { + if (deleteRecursively(root)) { + return; + } + } + throw new RuntimeException("Temporary folder not deleted. Manual deletion required."); + } + } + private boolean deleteRecursively(File fileToDelete) { if (fileToDelete.delete()) { @@ -87,14 +110,4 @@ public class TempFolder } return fileToDelete.delete(); } - - @Override - public void close() - { - if (root != null && root.exists()) { - if (!deleteRecursively(root)) { - throw new RuntimeException("Temporary folder not deleted. Manual deletion required."); - } - } - } } diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHetuMetastore.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHetuMetastore.java index d1f70d331..350a102f6 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHetuMetastore.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHetuMetastore.java @@ -75,6 +75,7 @@ public class TestingHetuMetastore .put("hetu.metastore.db.url", mySqlServer.getJdbcUrl(TEST_DATABASES)) .put("hetu.metastore.db.user", TEST_MYSQL_USER) .put("hetu.metastore.db.password", TEST_MYSQL_PASSWORD) + .put("hetu.metastore.cache.ttl", "0s") .build(); Bootstrap app = new Bootstrap( diff --git a/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/IndexRecordManager.java b/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/IndexRecordManager.java index f0deeac43..6f8e47cfc 100644 --- a/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/IndexRecordManager.java +++ b/hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/IndexRecordManager.java @@ -14,7 +14,6 @@ */ package io.hetu.core.heuristicindex; -import com.google.common.collect.ImmutableList; import io.airlift.log.Logger; import io.prestosql.spi.heuristicindex.IndexRecord; import io.prestosql.spi.metastore.HetuMetastore; @@ -24,11 +23,11 @@ import io.prestosql.spi.metastore.model.TableEntity; import io.prestosql.spi.metastore.model.TableEntityType; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.stream.Stream; public class IndexRecordManager { @@ -45,17 +44,27 @@ public class IndexRecordManager { long startTime = System.currentTimeMillis(); - ImmutableList.Builder records = ImmutableList.builder(); - getNewIndexStream().forEach(records::add); - LOG.debug("%dms spent on index record scan from hetu metastore", System.currentTimeMillis() - startTime); + List records = new ArrayList<>(); + for (CatalogEntity catalogEntity : metastore.getCatalogs()) { + for (DatabaseEntity databaseEntity : metastore.getAllDatabases(catalogEntity.getName())) { + for (TableEntity tableEntity : metastore.getAllTables(catalogEntity.getName(), databaseEntity.getName())) { + for (Map.Entry param : tableEntity.getParameters().entrySet()) { + if (param.getKey().startsWith(IndexRecord.INDEX_METASTORE_PREFIX)) { + records.add(new IndexRecord(tableEntity, param)); + } + } + } + } + } - return records.build(); + LOG.debug("%dms spent on index record scan from hetu metastore", System.currentTimeMillis() - startTime); + return records; } public IndexRecord lookUpIndexRecord(String name) throws IOException { - return getNewIndexStream().filter(indexRecord -> indexRecord.name.equals(name)).findFirst().orElse(null); + return getIndexRecords().stream().filter(indexRecord -> indexRecord.name.equals(name)).findFirst().orElse(null); } public IndexRecord lookUpIndexRecord(String table, String[] columns, String indexType) @@ -129,7 +138,7 @@ public class IndexRecordManager public synchronized void deleteIndexRecord(String name, List partitionsToRemove) throws IOException { - getNewIndexStream().filter(record -> record.name.equals(name)) + getIndexRecords().stream().filter(record -> record.name.equals(name)) .forEach(record -> { if (partitionsToRemove.isEmpty()) { metastore.alterTableParameter( @@ -152,14 +161,4 @@ public class IndexRecordManager } }); } - - private Stream getNewIndexStream() - { - return metastore.getCatalogs().stream() - .flatMap(catalogEntity -> metastore.getAllDatabases(catalogEntity.getName()).stream()) - .flatMap(databaseEntity -> metastore.getAllTables(databaseEntity.getCatalogName(), databaseEntity.getName()).stream()) - .flatMap(tableEntity -> tableEntity.getParameters().entrySet().stream() - .filter(e -> e.getKey().startsWith(IndexRecord.INDEX_METASTORE_PREFIX)) - .map(e -> new IndexRecord(tableEntity, e))); - } } diff --git a/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/btree/BTreeIndex.java b/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/btree/BTreeIndex.java index df58e167c..a33171131 100644 --- a/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/btree/BTreeIndex.java +++ b/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/btree/BTreeIndex.java @@ -89,10 +89,11 @@ public class BTreeIndex public BTreeIndex() { - dataDir = new TempFolder(); + dataDir = new TempFolder("btree"); try { dataDir.create(); dataFile = dataDir.getRoot().toPath().resolve("btree-" + UUID.randomUUID().toString()).toFile(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> dataDir.close())); } catch (IOException e) { dataDir.close(); diff --git a/hetu-metastore/src/main/java/io/hetu/core/metastore/HetuMetastoreCacheConfig.java b/hetu-metastore/src/main/java/io/hetu/core/metastore/HetuMetastoreCacheConfig.java index 49017f8f3..258bf8639 100644 --- a/hetu-metastore/src/main/java/io/hetu/core/metastore/HetuMetastoreCacheConfig.java +++ b/hetu-metastore/src/main/java/io/hetu/core/metastore/HetuMetastoreCacheConfig.java @@ -19,12 +19,12 @@ import io.airlift.configuration.ConfigDescription; import io.airlift.units.Duration; import io.airlift.units.MinDuration; -import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.concurrent.TimeUnit.HOURS; public class HetuMetastoreCacheConfig { private long metaStoreCacheMaxSize = 10000; - private Duration metaStoreCacheTtl = new Duration(0, SECONDS); + private Duration metaStoreCacheTtl = new Duration(4, HOURS); @Config("hetu.metastore.cache.size") @ConfigDescription("Set the max metastore cache size, default value 50000.") diff --git a/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreCacheConfig.java b/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreCacheConfig.java index a72866322..44e926575 100644 --- a/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreCacheConfig.java +++ b/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreCacheConfig.java @@ -24,7 +24,6 @@ import static io.airlift.configuration.testing.ConfigAssertions.assertFullMappin import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.concurrent.TimeUnit.SECONDS; public class TestHetuMetastoreCacheConfig { @@ -33,7 +32,7 @@ public class TestHetuMetastoreCacheConfig { assertRecordedDefaults(recordDefaults(HetuMetastoreCacheConfig.class) .setMetaStoreCacheMaxSize(10000) - .setMetaStoreCacheTtl(new Duration(0, SECONDS))); + .setMetaStoreCacheTtl(new Duration(4, HOURS))); } @Test diff --git a/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreModule.java b/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreModule.java index 0d7787657..307c493d4 100644 --- a/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreModule.java +++ b/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreModule.java @@ -79,6 +79,7 @@ public class TestHetuMetastoreModule .put("hetu.metastore.db.url", database.getUrl()) .put("hetu.metastore.db.user", user) .put("hetu.metastore.db.password", password) + .put("hetu.metastore.cache.ttl", "0s") .build(); try { Bootstrap app = new Bootstrap( diff --git a/hetu-vdm/src/test/java/io/hetu/core/plugin/vdm/VdmQueryRunner.java b/hetu-vdm/src/test/java/io/hetu/core/plugin/vdm/VdmQueryRunner.java index 0d2d2647e..bb67dd084 100644 --- a/hetu-vdm/src/test/java/io/hetu/core/plugin/vdm/VdmQueryRunner.java +++ b/hetu-vdm/src/test/java/io/hetu/core/plugin/vdm/VdmQueryRunner.java @@ -90,6 +90,7 @@ public final class VdmQueryRunner bufferedWriter.write("hetu.metastore.type = jdbc\n"); bufferedWriter.write("hetu.metastore.db.user = user\n"); bufferedWriter.write("hetu.metastore.db.password = testpass\n"); + bufferedWriter.write("hetu.metastore.cache.ttl = 0s"); } queryRunner = DistributedQueryRunner.builder(createSession()) .setNodeCount(1) diff --git a/presto-hive/src/test/java/io/prestosql/plugin/hive/HiveQueryRunner.java b/presto-hive/src/test/java/io/prestosql/plugin/hive/HiveQueryRunner.java index 92a66693f..a13cabd09 100644 --- a/presto-hive/src/test/java/io/prestosql/plugin/hive/HiveQueryRunner.java +++ b/presto-hive/src/test/java/io/prestosql/plugin/hive/HiveQueryRunner.java @@ -163,6 +163,7 @@ public final class HiveQueryRunner bufferedWriter.write("hetu.metastore.type = jdbc\n"); bufferedWriter.write("hetu.metastore.db.user = user\n"); bufferedWriter.write("hetu.metastore.db.password = testpass\n"); + bufferedWriter.write("hetu.metastore.cache.ttl = 0s"); } queryRunner.installPlugin(new HetuMetastorePlugin()); queryRunner.getCoordinator().loadMetastore();