From 2be43a3402543f74bdcb90d527d35b03c9ffed72 Mon Sep 17 00:00:00 2001 From: shuaiwang999 Date: Tue, 19 Jan 2021 16:52:42 +0800 Subject: [PATCH] add cache for metastore --- hetu-docs/en/admin/meta-store.md | 59 ++ hetu-docs/en/index.md | 1 + hetu-docs/zh/admin/meta-store.md | 57 ++ hetu-docs/zh/index.md | 1 + .../core/metastore/ForHetuMetastoreCache.java | 32 + .../core/metastore/HetuMetastoreCache.java | 269 +++++++++ .../metastore/HetuMetastoreCacheConfig.java | 55 ++ .../hetufilesystem/HetuFsMetastoreModule.java | 11 +- .../metastore/jdbc/JdbcMetastoreModule.java | 11 +- .../metastore/TestHetuMetastoreCache.java | 570 ++++++++++++++++++ .../TestHetuMetastoreCacheConfig.java | 51 ++ .../metastore/TestHetuMetastoreModule.java | 3 +- 12 files changed, 1114 insertions(+), 6 deletions(-) create mode 100644 hetu-docs/en/admin/meta-store.md create mode 100644 hetu-docs/zh/admin/meta-store.md create mode 100644 hetu-metastore/src/main/java/io/hetu/core/metastore/ForHetuMetastoreCache.java create mode 100644 hetu-metastore/src/main/java/io/hetu/core/metastore/HetuMetastoreCache.java create mode 100644 hetu-metastore/src/main/java/io/hetu/core/metastore/HetuMetastoreCacheConfig.java create mode 100644 hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreCache.java create mode 100644 hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreCacheConfig.java diff --git a/hetu-docs/en/admin/meta-store.md b/hetu-docs/en/admin/meta-store.md new file mode 100644 index 000000000..0aa9b7e89 --- /dev/null +++ b/hetu-docs/en/admin/meta-store.md @@ -0,0 +1,59 @@ +# Meta Store +This section describes the openLooKeng meta store. Meta store is used to store metadata. Meta store supports to +store metadata in the RDBMS or HDFS. + +## Configuring Meta Store + +Create the meta store property file `etc/hetu-metastore.properties` first. + + +### RDBMS Storage + +Add the following contents in `hetu-metastore.properties`: + +``` properties +hetu.metastore.type=jdbc +hetu.metastore.db.url=jdbc:mysql://.... +hetu.metastore.db.user=root +hetu.metastore.db.password=123456 +``` + +The above properties are described below: + +- `hetu.metastore.type`：The type of meta store, set `jdbc` to use RDBMS storage. +- `hetu.metastore.db.url`：URL of RDBMS to connect to. +- `hetu.metastore.db.user` :User name of RDBMS to connect to. +- `hetu.metastore.db.password` :Password of RDBMS to connect to. + +### HDFS Storage + +Add the following contents in `hetu-metastore.properties`: + +``` +hetu.metastore.type=hetufilesystem +hetu.metastore.hetufilesystem.profile-name=hdfs-config-metastore +hetu.metastore.hetufilesystem.path=/etc/openlookeng/metastore +``` + +The above properties are described below: + +- `hetu.metastore.type`：The type of meta store, set `hetufilesystem` to use HDFS storage. +- `hetu.metastore.hetufilesystem.profile-name`：The profile name of file system. +- `hetu.metastore.hetufilesystem.path`：The path of metastore storage in the file system. + +### Meta Store Cache + +openLooKeng can be configured to enable metadata caching. After the metadata caching is enabled, metadata will be cached +in the memory to improve access efficiency for the next access. + +Add the following contents in `hetu-metastore.properties`: + +``` properties +hetu.metastore.cache.size=10000 +hetu.metastore.cache.ttl=4h +``` + +The above properties are described below: + +- `hetu.metastore.cache.size`：Set the max metastore cache size, default value 10000. +- `hetu.metastore.cache.ttl`：Set ttl for metastore cache, default value 0 (metadata caching is disabled). diff --git a/hetu-docs/en/index.md b/hetu-docs/en/index.md index abd82b73f..a019f985d 100644 --- a/hetu-docs/en/index.md +++ b/hetu-docs/en/index.md @@ -40,6 +40,7 @@ headless: true - [Horizontal Scaling]({{< relref "./docs/admin/horizontal-scaling.md" >}}) - [Dynamic Filtering]({{< relref "./docs/admin/dynamic-filters.md" >}}) - [State store]({{< relref "./docs/admin/state-store.md" >}}) + - [Meta Store]({{< relref "./docs/admin/meta-store.md" >}}) - [Audit Log]({{< relref "./docs/admin/audit-log.md" >}}) - [Query Optimizer]("#") diff --git a/hetu-docs/zh/admin/meta-store.md b/hetu-docs/zh/admin/meta-store.md new file mode 100644 index 000000000..95c7e75df --- /dev/null +++ b/hetu-docs/zh/admin/meta-store.md @@ -0,0 +1,57 @@ +# 元数据存储 +本节介绍openLooKeng元数据存储。元数据存储用于存储元数据信息, 当前元数据存储支持将元数据保存在RDBMS或HDFS中。 + +## 配置元数据存储 + +首先在etc目录下创建配置文件`hetu-metastore.properties`。 + +### 使用RDBMS存储 + +在配置文件`hetu-metastore.properties`增加如下配置： + +``` properties +hetu.metastore.type=jdbc +hetu.metastore.db.url=jdbc:mysql://.... +hetu.metastore.db.user=root +hetu.metastore.db.password=123456 +``` + +上述属性说明如下： + +- `hetu.metastore.type`：元数据存储类型，使用RDBMS存储时配置为`jdbc`。 +- `hetu.metastore.db.url`：连接RDBMS的URL。 +- `hetu.metastore.db.user` :连接RDBMS的用户名。 +- `hetu.metastore.db.password` :连接RDBMS的密码。 + +### 使用HDFS存储 + +在配置文件`hetu-metastore.properties`增加如下配置： + +``` +hetu.metastore.type=hetufilesystem +hetu.metastore.hetufilesystem.profile-name=hdfs-config-metastore +hetu.metastore.hetufilesystem.path=/etc/openlookeng/metastore +``` + +上述属性说明如下： + +- `hetu.metastore.type`：元数据存储类型，使用HDFS存储时配置为`hetufilesystem`。 +- `hetu.metastore.hetufilesystem.profile-name`：使用HDFS存储配置文件的名称。 +- `hetu.metastore.hetufilesystem.path`：配置文件的路径。 + +可以从[文件系统](../develop/filesystem.md )中获取更多的文件系统相关的信息 + +### 元数据存储缓存 + +openLooKeng可以配置开启元数据缓存，开启后访问元数据时会将其缓存到内存中，再次访问时可以提高元数据访问效率。 +在配置文件`hetu-metastore.properties`增加如下配置： + +``` properties +hetu.metastore.cache.size=10000 +hetu.metastore.cache.ttl=4h +``` + +上述属性说明如下： + +- `hetu.metastore.cache.size`：元数据缓存大小，默认10000。 +- `hetu.metastore.cache.ttl`：缓存元数据的过期时间，默认值0 (元数据缓存关闭)。 \ No newline at end of file diff --git a/hetu-docs/zh/index.md b/hetu-docs/zh/index.md index e25e05cb6..3349d046d 100644 --- a/hetu-docs/zh/index.md +++ b/hetu-docs/zh/index.md @@ -40,6 +40,7 @@ headless: true - [水平伸缩]({{< relref "./docs/admin/horizontal-scaling.md" >}}) - [动态过滤]({{< relref "./docs/admin/dynamic-filters.md" >}}) - [State Store]({{< relref "./docs/admin/state-store.md" >}}) + - [元数据存储]({{< relref "./docs/admin/meta-store.md" >}}) - [审计日志]({{< relref "./docs/admin/audit-log.md" >}}) - [查询优化器]("#") diff --git a/hetu-metastore/src/main/java/io/hetu/core/metastore/ForHetuMetastoreCache.java b/hetu-metastore/src/main/java/io/hetu/core/metastore/ForHetuMetastoreCache.java new file mode 100644 index 000000000..e0f86daff --- /dev/null +++ b/hetu-metastore/src/main/java/io/hetu/core/metastore/ForHetuMetastoreCache.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.metastore; + +import javax.inject.Qualifier; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({FIELD, PARAMETER, METHOD}) +@Qualifier +public @interface ForHetuMetastoreCache +{ +} diff --git a/hetu-metastore/src/main/java/io/hetu/core/metastore/HetuMetastoreCache.java b/hetu-metastore/src/main/java/io/hetu/core/metastore/HetuMetastoreCache.java new file mode 100644 index 000000000..c28a643a9 --- /dev/null +++ b/hetu-metastore/src/main/java/io/hetu/core/metastore/HetuMetastoreCache.java @@ -0,0 +1,269 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.metastore; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import io.airlift.log.Logger; +import io.prestosql.spi.metastore.HetuMetastore; +import io.prestosql.spi.metastore.model.CatalogEntity; +import io.prestosql.spi.metastore.model.DatabaseEntity; +import io.prestosql.spi.metastore.model.TableEntity; + +import javax.inject.Inject; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; + +public class HetuMetastoreCache + implements HetuMetastore +{ + private static final Logger log = Logger.get(HetuMetastoreCache.class); + + private HetuMetastore delegate; + + private Cache> catalogCache; + private Cache> catalogsCache; + private Cache> databaseCache; + private Cache> databasesCache; + private Cache> tableCache; + private Cache> tablesCache; + + @Inject + public HetuMetastoreCache(@ForHetuMetastoreCache HetuMetastore delegate, HetuMetastoreCacheConfig hetuMetastoreCacheConfig) + { + this.delegate = delegate; + + buildMetaStoreCache(hetuMetastoreCacheConfig.getMetaStoreCacheMaxSize(), + Duration.ofMillis(hetuMetastoreCacheConfig.getMetaStoreCacheTtl().toMillis())); + } + + @Override + public void createCatalog(CatalogEntity catalog) + { + try { + delegate.createCatalog(catalog); + } + finally { + catalogsCache.invalidateAll(); + } + } + + @Override + public void alterCatalog(String catalogName, CatalogEntity newCatalog) + { + try { + delegate.alterCatalog(catalogName, newCatalog); + } + finally { + catalogCache.invalidate(catalogName); + catalogsCache.invalidateAll(); + } + } + + @Override + public void dropCatalog(String catalogName) + { + try { + delegate.dropCatalog(catalogName); + } + finally { + catalogCache.invalidate(catalogName); + catalogsCache.invalidateAll(); + } + } + + @Override + public Optional getCatalog(String catalogName) + { + try { + return catalogCache.get(catalogName, () -> { + return delegate.getCatalog(catalogName); + }); + } + catch (ExecutionException executionException) { + log.debug(executionException.getCause(), + String.format("Error while caching catalog[%s] metadata. Falling back to default flow", catalogName)); + return delegate.getCatalog(catalogName); + } + } + + @Override + public List getCatalogs() + { + try { + return catalogsCache.get("", () -> { + return delegate.getCatalogs(); + }); + } + catch (ExecutionException executionException) { + log.debug(executionException.getCause(), + "Error while caching all catalogs metadata. Falling back to default flow"); + return delegate.getCatalogs(); + } + } + + @Override + public void createDatabase(DatabaseEntity database) + { + try { + delegate.createDatabase(database); + } + finally { + databasesCache.invalidate(database.getCatalogName()); + } + } + + @Override + public void alterDatabase(String catalogName, String databaseName, DatabaseEntity newDatabase) + { + try { + delegate.alterDatabase(catalogName, databaseName, newDatabase); + } + finally { + String key = catalogName + "." + databaseName; + databaseCache.invalidate(key); + databasesCache.invalidate(catalogName); + } + } + + @Override + public void dropDatabase(String catalogName, String databaseName) + { + try { + delegate.dropDatabase(catalogName, databaseName); + } + finally { + String key = catalogName + "." + databaseName; + databaseCache.invalidate(key); + databasesCache.invalidate(catalogName); + } + } + + @Override + public Optional getDatabase(String catalogName, String databaseName) + { + try { + String key = catalogName + "." + databaseName; + return databaseCache.get(key, () -> { + return delegate.getDatabase(catalogName, databaseName); + }); + } + catch (ExecutionException executionException) { + log.debug(executionException.getCause(), + String.format("Error while caching database[%s.%s] metadata. Falling back to default flow"), catalogName, databaseName); + return delegate.getDatabase(catalogName, databaseName); + } + } + + @Override + public List getAllDatabases(String catalogName) + { + try { + return databasesCache.get(catalogName, () -> { + return delegate.getAllDatabases(catalogName); + }); + } + catch (ExecutionException executionException) { + log.debug(executionException.getCause(), + String.format("Error while caching all databases metadata in catalog[%s]. Falling back to default flow", catalogName)); + return delegate.getAllDatabases(catalogName); + } + } + + @Override + public void createTable(TableEntity table) + { + try { + delegate.createTable(table); + } + finally { + String key = table.getCatalogName() + "." + table.getDatabaseName(); + tablesCache.invalidate(key); + } + } + + @Override + public void dropTable(String catalogName, String databaseName, String tableName) + { + try { + delegate.dropTable(catalogName, databaseName, tableName); + } + finally { + String databaseKey = catalogName + '.' + databaseName; + String tableKey = catalogName + '.' + databaseName + '.' + tableName; + tableCache.invalidate(tableKey); + tablesCache.invalidate(databaseKey); + } + } + + @Override + public void alterTable(String catalogName, String databaseName, String oldTableName, TableEntity newTable) + { + try { + delegate.alterTable(catalogName, databaseName, oldTableName, newTable); + } + finally { + String databaseKey = catalogName + '.' + databaseName; + String tableKey = catalogName + "." + databaseName + "." + oldTableName; + tableCache.invalidate(tableKey); + tablesCache.invalidate(databaseKey); + } + } + + @Override + public Optional getTable(String catalogName, String databaseName, String tableName) + { + try { + String key = catalogName + '.' + databaseName + '.' + tableName; + return tableCache.get(key, () -> { + return delegate.getTable(catalogName, databaseName, tableName); + }); + } + catch (ExecutionException executionException) { + log.debug(executionException.getCause(), + String.format("Error while caching table[%s.%s.%s] metadata. Falling back to default flow", catalogName, databaseName, tableName)); + return delegate.getTable(catalogName, databaseName, tableName); + } + } + + @Override + public List getAllTables(String catalogName, String databaseName) + { + try { + String key = catalogName + "." + databaseName; + return tablesCache.get(key, () -> { + return delegate.getAllTables(catalogName, databaseName); + }); + } + catch (ExecutionException executionException) { + log.debug(executionException.getCause(), + String.format("Error while caching all tables metadata in %s.%s. Falling back to default flow"), catalogName, databaseName); + return delegate.getAllTables(catalogName, databaseName); + } + } + + private void buildMetaStoreCache(long maximumSize, Duration ttl) + { + catalogCache = CacheBuilder.newBuilder().maximumSize(maximumSize).expireAfterAccess(ttl).build(); + catalogsCache = CacheBuilder.newBuilder().maximumSize(maximumSize).expireAfterAccess(ttl).build(); + databaseCache = CacheBuilder.newBuilder().maximumSize(maximumSize).expireAfterAccess(ttl).build(); + databasesCache = CacheBuilder.newBuilder().maximumSize(maximumSize).expireAfterAccess(ttl).build(); + tableCache = CacheBuilder.newBuilder().maximumSize(maximumSize).expireAfterAccess(ttl).build(); + tablesCache = CacheBuilder.newBuilder().maximumSize(maximumSize).expireAfterAccess(ttl).build(); + } +} diff --git a/hetu-metastore/src/main/java/io/hetu/core/metastore/HetuMetastoreCacheConfig.java b/hetu-metastore/src/main/java/io/hetu/core/metastore/HetuMetastoreCacheConfig.java new file mode 100644 index 000000000..49017f8f3 --- /dev/null +++ b/hetu-metastore/src/main/java/io/hetu/core/metastore/HetuMetastoreCacheConfig.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.metastore; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; + +import static java.util.concurrent.TimeUnit.SECONDS; + +public class HetuMetastoreCacheConfig +{ + private long metaStoreCacheMaxSize = 10000; + private Duration metaStoreCacheTtl = new Duration(0, SECONDS); + + @Config("hetu.metastore.cache.size") + @ConfigDescription("Set the max metastore cache size, default value 50000.") + public HetuMetastoreCacheConfig setMetaStoreCacheMaxSize(long metaStoreCacheMaxSize) + { + this.metaStoreCacheMaxSize = metaStoreCacheMaxSize; + return this; + } + + public long getMetaStoreCacheMaxSize() + { + return metaStoreCacheMaxSize; + } + + @Config("hetu.metastore.cache.ttl") + @ConfigDescription("Set ttl for metastore cache, default value 4h.") + public HetuMetastoreCacheConfig setMetaStoreCacheTtl(Duration metaStoreCacheTtl) + { + this.metaStoreCacheTtl = metaStoreCacheTtl; + return this; + } + + @MinDuration("0ms") + public Duration getMetaStoreCacheTtl() + { + return metaStoreCacheTtl; + } +} diff --git a/hetu-metastore/src/main/java/io/hetu/core/metastore/hetufilesystem/HetuFsMetastoreModule.java b/hetu-metastore/src/main/java/io/hetu/core/metastore/hetufilesystem/HetuFsMetastoreModule.java index 132abe6c9..dba0fd0f0 100644 --- a/hetu-metastore/src/main/java/io/hetu/core/metastore/hetufilesystem/HetuFsMetastoreModule.java +++ b/hetu-metastore/src/main/java/io/hetu/core/metastore/hetufilesystem/HetuFsMetastoreModule.java @@ -18,6 +18,9 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.Scopes; +import io.hetu.core.metastore.ForHetuMetastoreCache; +import io.hetu.core.metastore.HetuMetastoreCache; +import io.hetu.core.metastore.HetuMetastoreCacheConfig; import io.prestosql.spi.filesystem.HetuFileSystemClient; import io.prestosql.spi.metastore.HetuMetastore; @@ -38,8 +41,12 @@ public class HetuFsMetastoreModule public void configure(Binder binder) { configBinder(binder).bindConfig(HetuFsMetastoreConfig.class); - binder.bind(HetuMetastore.class).to(HetuFsMetastore.class).in(Scopes.SINGLETON); - newExporter(binder).export(HetuMetastore.class).as(generator -> generator.generatedNameOf(HetuMetastore.class)); + configBinder(binder).bindConfig(HetuMetastoreCacheConfig.class); + binder.bind(HetuMetastore.class).annotatedWith(ForHetuMetastoreCache.class) + .to(HetuFsMetastore.class).in(Scopes.SINGLETON); + binder.bind(HetuMetastore.class).to(HetuMetastoreCache.class).in(Scopes.SINGLETON); + newExporter(binder).export(HetuMetastore.class) + .as(generator -> generator.generatedNameOf(HetuMetastoreCache.class)); } @Provides diff --git a/hetu-metastore/src/main/java/io/hetu/core/metastore/jdbc/JdbcMetastoreModule.java b/hetu-metastore/src/main/java/io/hetu/core/metastore/jdbc/JdbcMetastoreModule.java index 1dafc685b..dcaf31ddf 100644 --- a/hetu-metastore/src/main/java/io/hetu/core/metastore/jdbc/JdbcMetastoreModule.java +++ b/hetu-metastore/src/main/java/io/hetu/core/metastore/jdbc/JdbcMetastoreModule.java @@ -17,6 +17,9 @@ package io.hetu.core.metastore.jdbc; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Scopes; +import io.hetu.core.metastore.ForHetuMetastoreCache; +import io.hetu.core.metastore.HetuMetastoreCache; +import io.hetu.core.metastore.HetuMetastoreCacheConfig; import io.prestosql.spi.metastore.HetuMetastore; import static io.airlift.configuration.ConfigBinder.configBinder; @@ -34,7 +37,11 @@ public class JdbcMetastoreModule public void configure(Binder binder) { configBinder(binder).bindConfig(JdbcMetastoreConfig.class); - binder.bind(HetuMetastore.class).to(JdbcHetuMetastore.class).in(Scopes.SINGLETON); - newExporter(binder).export(HetuMetastore.class).as(generator -> generator.generatedNameOf(HetuMetastore.class)); + configBinder(binder).bindConfig(HetuMetastoreCacheConfig.class); + binder.bind(HetuMetastore.class).annotatedWith(ForHetuMetastoreCache.class) + .to(JdbcHetuMetastore.class).in(Scopes.SINGLETON); + binder.bind(HetuMetastore.class).to(HetuMetastoreCache.class).in(Scopes.SINGLETON); + newExporter(binder).export(HetuMetastore.class) + .as(generator -> generator.generatedNameOf(HetuMetastoreCache.class)); } } diff --git a/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreCache.java b/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreCache.java new file mode 100644 index 000000000..0ce53c464 --- /dev/null +++ b/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreCache.java @@ -0,0 +1,570 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.metastore; + +import com.google.common.cache.Cache; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.Resources; +import com.google.inject.Injector; +import io.airlift.bootstrap.Bootstrap; +import io.hetu.core.filesystem.HetuLocalFileSystemClient; +import io.hetu.core.filesystem.LocalConfig; +import io.hetu.core.metastore.hetufilesystem.HetuFsMetastoreModule; +import io.prestosql.plugin.base.jmx.MBeanServerModule; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.connector.SchemaTableName; +import io.prestosql.spi.filesystem.HetuFileSystemClient; +import io.prestosql.spi.metastore.HetuMetastore; +import io.prestosql.spi.metastore.model.CatalogEntity; +import io.prestosql.spi.metastore.model.DatabaseEntity; +import io.prestosql.spi.metastore.model.TableEntity; +import io.prestosql.spi.metastore.model.TableEntityType; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; +import org.weakref.jmx.guice.MBeanModule; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.base.Throwables.throwIfUnchecked; +import static io.prestosql.spi.metastore.HetuErrorCode.HETU_METASTORE_CODE; +import static java.util.Collections.emptyMap; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TestHetuMetastoreCache +{ + private HetuMetastore metastore; + private HetuFileSystemClient client; + private CatalogEntity defaultCatalog; + private DatabaseEntity defaultDatabase; + private String path = Resources.getResource("").getPath() + File.separator + "metastoreCache"; + + private Cache> catalogCache; + private Cache> catalogsCache; + private Cache> databaseCache; + private Cache> databasesCache; + private Cache> tableCache; + private Cache> tablesCache; + + /** + * setUp + * + * @throws Exception Exception + */ + @BeforeClass + public void setUp() + throws Throwable + { + try { + Map config = new ImmutableMap.Builder() + .put("hetu.metastore.hetufilesystem.path", path) + .put("hetu.metastore.hetufilesystem.profile-name", "local-config-catalog") + .put("hetu.metastore.cache.size", "10000") + .put("hetu.metastore.cache.ttl", "10h") + .build(); + + LocalConfig localConfig = new LocalConfig(null); + client = new HetuLocalFileSystemClient(localConfig, Paths.get(path)); + + if (!client.exists(Paths.get(path))) { + client.createDirectories(Paths.get(path)); + } + client.deleteRecursively(Paths.get(path)); + Bootstrap app = new Bootstrap( + new MBeanModule(), + new MBeanServerModule(), + new HetuFsMetastoreModule(client)); + + Injector injector = app + .strictConfig() + .doNotInitializeLogging() + .setRequiredConfigurationProperties(config) + .initialize(); + metastore = injector.getInstance(HetuMetastore.class); + } + catch (Exception ex) { + throwIfUnchecked(ex); + throw new PrestoException(HETU_METASTORE_CODE, + "init hetu metastore module failed."); + } + + // create default catalog and database + defaultCatalog = CatalogEntity.builder() + .setCatalogName("hetu1") + .setOwner("hetu1").build(); + metastore.createCatalog(defaultCatalog); + + defaultDatabase = DatabaseEntity.builder() + .setCatalogName(defaultCatalog.getName()) + .setDatabaseName("db1").build(); + metastore.createDatabase(defaultDatabase); + + // get metastore catalog cache + Field catalogCacheField = metastore.getClass().getDeclaredField("catalogCache"); + catalogCacheField.setAccessible(true); + catalogCache = (Cache>) catalogCacheField.get(metastore); + + Field catalogsCacheField = metastore.getClass().getDeclaredField("catalogsCache"); + catalogsCacheField.setAccessible(true); + catalogsCache = (Cache>) catalogsCacheField.get(metastore); + + // get metastore database cache + Field databaseCacheField = metastore.getClass().getDeclaredField("databaseCache"); + databaseCacheField.setAccessible(true); + databaseCache = (Cache>) databaseCacheField.get(metastore); + + Field databasesCacheField = metastore.getClass().getDeclaredField("databasesCache"); + databasesCacheField.setAccessible(true); + databasesCache = (Cache>) databasesCacheField.get(metastore); + + // get metastore table cache + Field tableCacheField = metastore.getClass().getDeclaredField("tableCache"); + tableCacheField.setAccessible(true); + tableCache = (Cache>) tableCacheField.get(metastore); + + Field tablesCacheField = metastore.getClass().getDeclaredField("tablesCache"); + tablesCacheField.setAccessible(true); + tablesCache = (Cache>) tablesCacheField.get(metastore); + } + + /** + * tearDown + * + * @throws Exception Exception + */ + @AfterClass(alwaysRun = true) + public void tearDown() + { + try { + client.deleteRecursively(Paths.get(path)); + } + catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * test create catalog with metastore cache + */ + @Test + public void testCreateCatalog() + { + Map properties = ImmutableMap.builder().build(); + CatalogEntity catalogEntity = CatalogEntity.builder() + .setCatalogName("catalog1") + .setOwner("root1") + .setComment(Optional.of("Hetu create catalog")) + .setParameters(properties) + .setCreateTime(System.currentTimeMillis()) + .build(); + metastore.createCatalog(catalogEntity); + + assertEquals(catalogsCache.getIfPresent(""), null); + + metastore.dropCatalog(catalogEntity.getName()); + } + + /** + * test drop dropCatalog with metastore cache + */ + @Test + public void testDropCatalog() + { + Map properties = ImmutableMap.builder().build(); + CatalogEntity catalogEntity = CatalogEntity.builder() + .setCatalogName("catalog2") + .setOwner("root2") + .setComment(Optional.of("Hetu create catalog")) + .setParameters(properties) + .setCreateTime(System.currentTimeMillis()) + .build(); + metastore.createCatalog(catalogEntity); + metastore.dropCatalog(catalogEntity.getName()); + + assertEquals(catalogCache.getIfPresent("catalog2"), null); + assertEquals(catalogsCache.getIfPresent(""), null); + } + + /** + * test get catalog with metastore cache + */ + @Test + public void testGetCatalog() + { + Map properties = ImmutableMap.builder().build(); + CatalogEntity catalogEntity = CatalogEntity.builder() + .setCatalogName("catalog3") + .setOwner("root3") + .setComment(Optional.of("Hetu create catalog")) + .setParameters(properties) + .setCreateTime(System.currentTimeMillis()) + .build(); + metastore.createCatalog(catalogEntity); + + Optional catalogInfo = metastore.getCatalog("catalog3"); + assertTrue(catalogInfo.isPresent()); + assertEquals(catalogInfo.get(), catalogEntity); + + metastore.dropCatalog(catalogEntity .getName()); + } + + /** + * test get All catalogs with metastore cache + */ + @Test + public void testGetAllCatalogs() + { + Map properties = ImmutableMap.builder().build(); + CatalogEntity catalog1 = CatalogEntity.builder() + .setCatalogName("catalog4") + .setOwner("root4") + .setComment(Optional.of("Hetu create catalog")) + .setParameters(properties) + .setCreateTime(System.currentTimeMillis()) + .build(); + metastore.createCatalog(catalog1); + + CatalogEntity catalog2 = CatalogEntity.builder() + .setCatalogName("catalog5") + .setOwner("root5") + .setComment(Optional.of("Hetu create catalog")) + .setParameters(emptyMap()) + .setCreateTime(System.currentTimeMillis()) + .build(); + metastore.createCatalog(catalog2); + + List catalogEntities = metastore.getCatalogs(); + assertEquals(catalogsCache.getIfPresent(""), catalogEntities); + + metastore.dropCatalog(catalog1.getName()); + metastore.dropCatalog(catalog2.getName()); + } + + /** + * test alter catalog with metastore cache + */ + @Test + public void testAlterCatalog() + { + Map properties = ImmutableMap.builder().build(); + CatalogEntity catalog1 = CatalogEntity.builder() + .setCatalogName("catalog6") + .setOwner("root6") + .setComment(Optional.of("Hetu create catalog")) + .setParameters(properties) + .setCreateTime(System.currentTimeMillis()) + .build(); + metastore.createCatalog(catalog1); + + CatalogEntity catalog2 = CatalogEntity.builder() + .setCatalogName("catalog6") + .setOwner("hive") + .setComment(Optional.of("Hetu alter catalog")) + .setParameters(emptyMap()) + .setCreateTime(System.currentTimeMillis()) + .build(); + metastore.alterCatalog("catalog6", catalog2); + + assertEquals(catalogsCache.getIfPresent(""), null); + + metastore.dropCatalog(catalog2.getName()); + } + + /** + * test create database with metastore cache + */ + @Test + public void testCreateDatabase() + { + Map properties = ImmutableMap.builder() + .put("desc", "vschema") + .build(); + DatabaseEntity databaseEntity = DatabaseEntity.builder() + .setCatalogName(defaultCatalog.getName()) + .setDatabaseName("db2") + .setOwner("root7") + .setComment(Optional.of("Hetu create database")) + .setCreateTime(System.currentTimeMillis()) + .setParameters(properties) + .build(); + metastore.createDatabase(databaseEntity); + + assertEquals(databasesCache.getIfPresent(defaultCatalog.getName()), null); + + metastore.dropDatabase(defaultCatalog.getName(), "db2"); + } + + /** + * test drop database with metastore cache + */ + @Test + public void testDropDatabase() + { + Map properties = ImmutableMap.builder() + .put("desc", "vschema") + .build(); + DatabaseEntity databaseEntity = DatabaseEntity.builder() + .setCatalogName(defaultCatalog.getName()) + .setDatabaseName("db3") + .setOwner("root8") + .setComment(Optional.of("Hetu create database")) + .setCreateTime(System.currentTimeMillis()) + .setParameters(properties) + .build(); + metastore.createDatabase(databaseEntity); + metastore.dropDatabase(defaultCatalog.getName(), "db3"); + + String key = defaultCatalog.getName() + "." + "db3"; + assertEquals(databaseCache.getIfPresent(key), null); + assertEquals(databasesCache.getIfPresent(defaultCatalog.getName()), null); + } + + /** + * test get database with metastore cache + */ + @Test + public void testGetDatabase() + { + Map properties = ImmutableMap.builder() + .put("desc", "vschema") + .build(); + DatabaseEntity databaseEntity = DatabaseEntity.builder() + .setCatalogName(defaultCatalog.getName()) + .setDatabaseName("db4") + .setOwner("root9") + .setComment(Optional.of("Hetu create database")) + .setCreateTime(System.currentTimeMillis()) + .setParameters(properties) + .build(); + metastore.createDatabase(databaseEntity); + + Optional databaseEntity2 = metastore.getDatabase(defaultCatalog.getName(), "db4"); + + assertTrue(databaseEntity2.isPresent()); + assertEquals(databaseEntity2.get(), databaseEntity); + + metastore.dropDatabase(defaultCatalog.getName(), "db4"); + } + + /** + * test get all databases with metastore cache + */ + @Test + public void testAllDatabases() + { + Map properties = ImmutableMap.builder() + .put("desc", "vschema") + .build(); + DatabaseEntity databaseEntity1 = DatabaseEntity.builder() + .setCatalogName(defaultCatalog.getName()) + .setDatabaseName("db5") + .setOwner("root10") + .setComment(Optional.of("Hetu create database")) + .setCreateTime(System.currentTimeMillis()) + .setParameters(properties) + .build(); + metastore.createDatabase(databaseEntity1); + + DatabaseEntity databaseEntity2 = DatabaseEntity.builder() + .setCatalogName(defaultCatalog.getName()) + .setDatabaseName("db6") + .setOwner("root11") + .setComment(Optional.of("Hetu create database")) + .setCreateTime(System.currentTimeMillis()) + .setParameters(properties) + .build(); + metastore.createDatabase(databaseEntity2); + + List databaseEntities = metastore.getAllDatabases(defaultCatalog.getName()); + assertEquals(databasesCache.getIfPresent(defaultCatalog.getName()), databaseEntities); + + metastore.dropDatabase(defaultCatalog.getName(), "db5"); + metastore.dropDatabase(defaultCatalog.getName(), "db6"); + } + + /** + * test alter database with metastore cache + */ + @Test + public void testAlterDatabase() + { + String oldDatabaseName = "db7"; + String newDatabaseName = "db8"; + Map properties = ImmutableMap.builder() + .put("desc", "vschema") + .build(); + DatabaseEntity databaseEntity1 = DatabaseEntity.builder() + .setCatalogName(defaultCatalog.getName()) + .setDatabaseName(oldDatabaseName) + .setOwner("root12") + .setComment(Optional.of("Hetu create database")) + .setCreateTime(System.currentTimeMillis()) + .setParameters(properties) + .build(); + metastore.createDatabase(databaseEntity1); + + DatabaseEntity databaseEntity2 = DatabaseEntity.builder() + .setCatalogName(defaultCatalog.getName()) + .setDatabaseName(newDatabaseName) + .setOwner("root13") + .setComment(Optional.of("Hetu create database")) + .setCreateTime(System.currentTimeMillis()) + .setParameters(properties) + .build(); + + metastore.alterDatabase(defaultCatalog.getName(), oldDatabaseName, databaseEntity2); + + String key = defaultCatalog.getName() + "." + oldDatabaseName; + assertEquals(databaseCache.getIfPresent(key), null); + assertEquals(databasesCache.getIfPresent(defaultCatalog.getName()), null); + + metastore.dropDatabase(defaultCatalog.getName(), newDatabaseName); + } + + /** + * testCreateView with metastore cache + */ + @Test + public void testCreateTable() + { + String tableName = "table1"; + SchemaTableName schemaTableName = new SchemaTableName(defaultDatabase.getName(), tableName); + TableEntity tableEntity = TableEntity.builder() + .setCatalogName(defaultDatabase.getCatalogName()) + .setDatabaseName(defaultDatabase.getName()) + .setTableName(schemaTableName.getTableName()) + .setTableType(TableEntityType.TABLE.toString()) + .build(); + metastore.createTable(tableEntity); + + String tablesKey = defaultDatabase.getCatalogName() + "." + defaultDatabase.getName(); + assertEquals(tablesCache.getIfPresent(tablesKey), null); + + metastore.dropTable(defaultDatabase.getCatalogName(), defaultDatabase.getName(), tableName); + } + + /** + * test drop table with metastore cache + */ + @Test + public void testDropTable() + { + String tableName = "table2"; + SchemaTableName schemaTableName = new SchemaTableName(defaultDatabase.getName(), tableName); + TableEntity tableEntity = TableEntity.builder() + .setCatalogName(defaultDatabase.getCatalogName()) + .setDatabaseName(defaultDatabase.getName()) + .setTableName(schemaTableName.getTableName()) + .setTableType(TableEntityType.TABLE.toString()) + .build(); + metastore.createTable(tableEntity); + metastore.dropTable(defaultDatabase.getCatalogName(), defaultDatabase.getName(), tableName); + + String tablesKey = defaultDatabase.getCatalogName() + "." + defaultDatabase.getName(); + String tableKey = defaultDatabase.getCatalogName() + "." + defaultDatabase.getName() + "." + tableName; + assertEquals(tableCache.getIfPresent(tableKey), null); + assertEquals(databasesCache.getIfPresent(tablesKey), null); + } + + /** + * test get table with metastore cache + */ + @Test + public void testGetTable() + { + String tableName = "table3"; + SchemaTableName schemaTableName = new SchemaTableName(defaultDatabase.getName(), tableName); + TableEntity tableEntity = TableEntity.builder() + .setCatalogName(defaultDatabase.getCatalogName()) + .setDatabaseName(defaultDatabase.getName()) + .setTableName(schemaTableName.getTableName()) + .setTableType(TableEntityType.TABLE.toString()) + .build(); + metastore.createTable(tableEntity); + + Optional tableEntity2 = metastore.getTable(defaultDatabase.getCatalogName(), defaultDatabase.getName(), tableName); + + String tableKey = defaultDatabase.getCatalogName() + "." + defaultDatabase.getName() + "." + tableName; + assertEquals(tableCache.getIfPresent(tableKey).get(), tableEntity2.get()); + + metastore.dropTable(defaultDatabase.getCatalogName(), defaultDatabase.getName(), tableName); + } + + /** + * test get all table with metastore cache + */ + @Test + public void testGetAllTables() + { + String tableName = "table4"; + SchemaTableName schemaTableName = new SchemaTableName(defaultDatabase.getName(), tableName); + TableEntity tableEntity = TableEntity.builder() + .setCatalogName(defaultDatabase.getCatalogName()) + .setDatabaseName(defaultDatabase.getName()) + .setTableName(schemaTableName.getTableName()) + .setTableType(TableEntityType.TABLE.toString()) + .build(); + metastore.createTable(tableEntity); + + List tableEntities = metastore.getAllTables(defaultDatabase.getCatalogName(), defaultDatabase.getName()); + + String tablesKey = defaultDatabase.getCatalogName() + "." + defaultDatabase.getName(); + assertEquals(tablesCache.getIfPresent(tablesKey), tableEntities); + + metastore.dropTable(defaultDatabase.getCatalogName(), defaultDatabase.getName(), tableName); + } + + /** + * test alter table with metastore cache + */ + @Test + public void testAlterTable() + { + String tableName1 = "table5"; + SchemaTableName schemaTableName1 = new SchemaTableName(defaultDatabase.getName(), tableName1); + TableEntity tableEntity1 = TableEntity.builder() + .setCatalogName(defaultDatabase.getCatalogName()) + .setDatabaseName(defaultDatabase.getName()) + .setTableName(schemaTableName1.getTableName()) + .setTableType(TableEntityType.TABLE.toString()) + .build(); + metastore.createTable(tableEntity1); + + String tableName2 = "table6"; + SchemaTableName schemaTableName2 = new SchemaTableName(defaultDatabase.getName(), tableName2); + TableEntity tableEntity2 = TableEntity.builder() + .setCatalogName(defaultDatabase.getCatalogName()) + .setDatabaseName(defaultDatabase.getName()) + .setTableName(schemaTableName2.getTableName()) + .setTableType(TableEntityType.TABLE.toString()) + .build(); + + metastore.alterTable(defaultDatabase.getCatalogName(), defaultDatabase.getName(), tableName1, tableEntity2); + + String tablesKey = defaultDatabase.getCatalogName() + "." + defaultDatabase.getName(); + String tableKey = defaultDatabase.getCatalogName() + "." + defaultDatabase.getName() + "." + tableName1; + assertEquals(tablesCache.getIfPresent(tablesKey), null); + assertEquals(tableCache.getIfPresent(tableKey), null); + + metastore.dropTable(defaultDatabase.getCatalogName(), defaultDatabase.getName(), tableName2); + } +} diff --git a/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreCacheConfig.java b/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreCacheConfig.java new file mode 100644 index 000000000..a72866322 --- /dev/null +++ b/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreCacheConfig.java @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.metastore; + +import com.google.common.collect.ImmutableMap; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.concurrent.TimeUnit.SECONDS; + +public class TestHetuMetastoreCacheConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(HetuMetastoreCacheConfig.class) + .setMetaStoreCacheMaxSize(10000) + .setMetaStoreCacheTtl(new Duration(0, SECONDS))); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("hetu.metastore.cache.size", "100000") + .put("hetu.metastore.cache.ttl", "10h") + .build(); + HetuMetastoreCacheConfig config = new HetuMetastoreCacheConfig() + .setMetaStoreCacheMaxSize(100000) + .setMetaStoreCacheTtl(new Duration(10, HOURS)); + assertFullMapping(properties, config); + } +} diff --git a/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreModule.java b/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreModule.java index 059fb54e4..0d7787657 100644 --- a/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreModule.java +++ b/hetu-metastore/src/test/java/io/hetu/core/metastore/TestHetuMetastoreModule.java @@ -18,7 +18,6 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; -import io.hetu.core.metastore.jdbc.JdbcHetuMetastore; import io.hetu.core.metastore.jdbc.JdbcMetastoreModule; import io.prestosql.plugin.base.jmx.MBeanServerModule; import io.prestosql.spi.PrestoException; @@ -92,7 +91,7 @@ public class TestHetuMetastoreModule .doNotInitializeLogging() .setRequiredConfigurationProperties(properties) .initialize(); - assertTrue(injector.getInstance(HetuMetastore.class) instanceof JdbcHetuMetastore); + assertTrue(injector.getInstance(HetuMetastore.class) instanceof HetuMetastoreCache); } catch (Exception ex) { throwIfUnchecked(ex);