jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Improve index lookup performance and optimize temp folder usage

This commit is contained in:
Han Weng 2021-03-22 15:10:03 -04:00
parent b0293ac885
commit 543a7e6de8
9 changed files with 51 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
hetu-common/src/main/java/io/hetu/core/common/filesystem/TempFolder.java
View File

@ -24,14 +24,23 @@ import java.util.UUID;
public class TempFolder
implements AutoCloseable
{
private final String prefix;
private File root;
public TempFolder() {}
public TempFolder()
{
this("");
}
public TempFolder(String user)
{
this.prefix = "hetu-tmp-folder-" + user;
}
public TempFolder create()
throws IOException
{
root = Files.createTempDirectory("hetu-tmp-folder-").toFile();
root = Files.createTempDirectory(prefix).toFile();
return this;
}
@ -72,6 +81,20 @@ public class TempFolder
throw new IOException("Not able to create folder " + relativePath);
}
@Override
public void close()
{
if (root != null && root.exists()) {
// retry deletion for 3 times
for (int i = 0; i < 3; i++) {
if (deleteRecursively(root)) {
return;
}
}
throw new RuntimeException("Temporary folder not deleted. Manual deletion required.");
}
}
private boolean deleteRecursively(File fileToDelete)
{
if (fileToDelete.delete()) {
@ -87,14 +110,4 @@ public class TempFolder
}
return fileToDelete.delete();
}
@Override
public void close()
{
if (root != null && root.exists()) {
if (!deleteRecursively(root)) {
throw new RuntimeException("Temporary folder not deleted. Manual deletion required.");
}
}
}
}

1
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHetuMetastore.java
View File

@ -75,6 +75,7 @@ public class TestingHetuMetastore
.put("hetu.metastore.db.url", mySqlServer.getJdbcUrl(TEST_DATABASES))
.put("hetu.metastore.db.user", TEST_MYSQL_USER)
.put("hetu.metastore.db.password", TEST_MYSQL_PASSWORD)
.put("hetu.metastore.cache.ttl", "0s")
.build();
Bootstrap app = new Bootstrap(

35
hetu-heuristic-index/src/main/java/io/hetu/core/heuristicindex/IndexRecordManager.java
View File

@ -14,7 +14,6 @@
*/
package io.hetu.core.heuristicindex;
import com.google.common.collect.ImmutableList;
import io.airlift.log.Logger;
import io.prestosql.spi.heuristicindex.IndexRecord;
import io.prestosql.spi.metastore.HetuMetastore;
@ -24,11 +23,11 @@ import io.prestosql.spi.metastore.model.TableEntity;
import io.prestosql.spi.metastore.model.TableEntityType;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;
public class IndexRecordManager
{
@ -45,17 +44,27 @@ public class IndexRecordManager
{
long startTime = System.currentTimeMillis();
ImmutableList.Builder<IndexRecord> records = ImmutableList.builder();
getNewIndexStream().forEach(records::add);
LOG.debug("%dms spent on index record scan from hetu metastore", System.currentTimeMillis() - startTime);
List<IndexRecord> records = new ArrayList<>();
for (CatalogEntity catalogEntity : metastore.getCatalogs()) {
for (DatabaseEntity databaseEntity : metastore.getAllDatabases(catalogEntity.getName())) {
for (TableEntity tableEntity : metastore.getAllTables(catalogEntity.getName(), databaseEntity.getName())) {
for (Map.Entry<String, String> param : tableEntity.getParameters().entrySet()) {
if (param.getKey().startsWith(IndexRecord.INDEX_METASTORE_PREFIX)) {
records.add(new IndexRecord(tableEntity, param));
}
}
}
}
}
return records.build();
LOG.debug("%dms spent on index record scan from hetu metastore", System.currentTimeMillis() - startTime);
return records;
}
public IndexRecord lookUpIndexRecord(String name)
throws IOException
{
return getNewIndexStream().filter(indexRecord -> indexRecord.name.equals(name)).findFirst().orElse(null);
return getIndexRecords().stream().filter(indexRecord -> indexRecord.name.equals(name)).findFirst().orElse(null);
}
public IndexRecord lookUpIndexRecord(String table, String[] columns, String indexType)
@ -129,7 +138,7 @@ public class IndexRecordManager
public synchronized void deleteIndexRecord(String name, List<String> partitionsToRemove)
throws IOException
{
getNewIndexStream().filter(record -> record.name.equals(name))
getIndexRecords().stream().filter(record -> record.name.equals(name))
.forEach(record -> {
if (partitionsToRemove.isEmpty()) {
metastore.alterTableParameter(
@ -152,14 +161,4 @@ public class IndexRecordManager
}
});
}
private Stream<IndexRecord> getNewIndexStream()
{
return metastore.getCatalogs().stream()
.flatMap(catalogEntity -> metastore.getAllDatabases(catalogEntity.getName()).stream())
.flatMap(databaseEntity -> metastore.getAllTables(databaseEntity.getCatalogName(), databaseEntity.getName()).stream())
.flatMap(tableEntity -> tableEntity.getParameters().entrySet().stream()
.filter(e -> e.getKey().startsWith(IndexRecord.INDEX_METASTORE_PREFIX))
.map(e -> new IndexRecord(tableEntity, e)));
}
}

3
hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/btree/BTreeIndex.java
View File

@ -89,10 +89,11 @@ public class BTreeIndex
public BTreeIndex()
{
dataDir = new TempFolder();
dataDir = new TempFolder("btree");
try {
dataDir.create();
dataFile = dataDir.getRoot().toPath().resolve("btree-" + UUID.randomUUID().toString()).toFile();
Runtime.getRuntime().addShutdownHook(new Thread(() -> dataDir.close()));
}
catch (IOException e) {
dataDir.close();

4
hetu-metastore/src/main/java/io/hetu/core/metastore/HetuMetastoreCacheConfig.java
View File

@ -19,12 +19,12 @@ import io.airlift.configuration.ConfigDescription;
import io.airlift.units.Duration;
import io.airlift.units.MinDuration;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.concurrent.TimeUnit.HOURS;
public class HetuMetastoreCacheConfig
{
private long metaStoreCacheMaxSize = 10000;
private Duration metaStoreCacheTtl = new Duration(0, SECONDS);
private Duration metaStoreCacheTtl = new Duration(4, HOURS);
@Config("hetu.metastore.cache.size")
@ConfigDescription("Set the max metastore cache size, default value 50000.")

3
hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreCacheConfig.java
View File

@ -24,7 +24,6 @@ import static io.airlift.configuration.testing.ConfigAssertions.assertFullMappin
import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults;
import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults;
import static java.util.concurrent.TimeUnit.HOURS;
import static java.util.concurrent.TimeUnit.SECONDS;
public class TestHetuMetastoreCacheConfig
{
@ -33,7 +32,7 @@ public class TestHetuMetastoreCacheConfig
{
assertRecordedDefaults(recordDefaults(HetuMetastoreCacheConfig.class)
.setMetaStoreCacheMaxSize(10000)
.setMetaStoreCacheTtl(new Duration(0, SECONDS)));
.setMetaStoreCacheTtl(new Duration(4, HOURS)));
}
@Test

1
hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreModule.java
View File

@ -79,6 +79,7 @@ public class TestHetuMetastoreModule
.put("hetu.metastore.db.url", database.getUrl())
.put("hetu.metastore.db.user", user)
.put("hetu.metastore.db.password", password)
.put("hetu.metastore.cache.ttl", "0s")
.build();
try {
Bootstrap app = new Bootstrap(

1
hetu-vdm/src/test/java/io/hetu/core/plugin/vdm/VdmQueryRunner.java
View File

@ -90,6 +90,7 @@ public final class VdmQueryRunner
bufferedWriter.write("hetu.metastore.type = jdbc\n");
bufferedWriter.write("hetu.metastore.db.user = user\n");
bufferedWriter.write("hetu.metastore.db.password = testpass\n");
bufferedWriter.write("hetu.metastore.cache.ttl = 0s");
}
queryRunner = DistributedQueryRunner.builder(createSession())
.setNodeCount(1)

1
presto-hive/src/test/java/io/prestosql/plugin/hive/HiveQueryRunner.java
View File

@ -163,6 +163,7 @@ public final class HiveQueryRunner
bufferedWriter.write("hetu.metastore.type = jdbc\n");
bufferedWriter.write("hetu.metastore.db.user = user\n");
bufferedWriter.write("hetu.metastore.db.password = testpass\n");
bufferedWriter.write("hetu.metastore.cache.ttl = 0s");
}
queryRunner.installPlugin(new HetuMetastorePlugin());
queryRunner.getCoordinator().loadMetastore();