diff --git a/hetu-docs/en/connector/hive.md b/hetu-docs/en/connector/hive.md index 552c7820c..abaa7b213 100644 --- a/hetu-docs/en/connector/hive.md +++ b/hetu-docs/en/connector/hive.md @@ -128,6 +128,10 @@ Please see the [Hive Security Configuration](./hive-security.md) section for a m | `hive.metastore-client-service-threads` | Number of threads for metastore clients to operate in parallel to communicate with hive metastore. | 4 | | `hive.worker-metastore-cache-enabled` | Enable the caching of the hive metastore on the worker nodes also. | `false` | | `hive.metastore-write-batch-size` | Number of partitions sent to meta store in per request. | `8` | +| `hive.metastore-cache-ttl` | Metastore Cache eviction time for table & partition metadata. | `0s` | +| `hive.metastore-refresh-interval` | Time after which metastore cache entries for table and partition metadata are refreshed from Hive metastore. | `1s` | +| `hive.metastore-db-cache-ttl` | Metastore Cache eviction time for DB, Roles, Configs, Table & Views list objects. | `0s` | +| `hive.metastore-db-refresh-interval` | Time after which metastore cache entry is refreshed from Hive metastore for DB, Table List, View List, Roles objects. | `1s` | @@ -712,6 +716,28 @@ Drop a schema: DROP SCHEMA hive.web ``` +## Metastore Cache: + +Hive connector maintains a metastore cache to service the metastore request faster to various operations. Loading, reloading and retention times of the cache entries can be configured in `hive.properties`. + + ```properties + # Table & Partition Cache specific configurations + hive.metastore-cache-ttl=24h + hive.metastore-refresh-interval=23h + + # DB, Table & View list, Roles, configurations related cache configuration + hive.metastore-db-cache-ttl=4m + hive.metastore-db-refresh-interval=3m + ``` + +**Note:** In cases where user operates on the data directly and if hive metastore is modified externally (eg. directly by Hive, Spark), there is a possibility of cache having older data. For the same user should configure the cache refresh and eviction times accordingly. + +In order reduce the inconsistency, hive connector also validates the partition & its statistics cache entries `on read` against table and partition-names cache (which refreshes at higher frequency) in case the table refresh time is higher than `5mins`. + +```sql +REFRESH META CACHE +``` +Additionally, metadata cache refresh command can be used to reload the metastore cache by user. ## Performance tuning notes: diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java index 6c0fb2fdd..7d0dda07a 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java @@ -86,8 +86,8 @@ public class HiveConfig private Duration metastoreCacheTtl = new Duration(0, TimeUnit.SECONDS); private Duration metastoreRefreshInterval = new Duration(1, TimeUnit.SECONDS); - private Duration metastoreTableCacheTtl = new Duration(0, TimeUnit.SECONDS); - private Duration metastoreTableRefreshInterval = new Duration(1, TimeUnit.SECONDS); + private Duration metastoreDBCacheTtl = new Duration(0, TimeUnit.SECONDS); + private Duration metastoreDBRefreshInterval = new Duration(1, TimeUnit.SECONDS); private long metastoreCacheMaximumSize = 10000; private long perTransactionMetastoreCacheMaximumSize = 1000; @@ -451,28 +451,28 @@ public class HiveConfig } @NotNull - public @MinDuration("0ms") Duration getMetastoreTableCacheTtl() + public @MinDuration("0ms") Duration getMetastoreDBCacheTtl() { - return metastoreTableCacheTtl; + return metastoreDBCacheTtl; } - @Config("hive.metastore-table-cache-ttl") - public HiveConfig setMetastoreTableCacheTtl(Duration metastoreCacheTtl) + @Config("hive.metastore-db-cache-ttl") + public HiveConfig setMetastoreDBCacheTtl(Duration metastoreCacheTtl) { - this.metastoreTableCacheTtl = metastoreCacheTtl; + this.metastoreDBCacheTtl = metastoreCacheTtl; return this; } @NotNull - public @MinDuration("1ms") Duration getMetastoreTableRefreshInterval() + public @MinDuration("1ms") Duration getMetastoreDBRefreshInterval() { - return metastoreTableRefreshInterval; + return metastoreDBRefreshInterval; } - @Config("hive.metastore-table-refresh-interval") - public HiveConfig setMetastoreTableRefreshInterval(Duration metastoreTableRefreshInterval) + @Config("hive.metastore-db-refresh-interval") + public HiveConfig setMetastoreDBRefreshInterval(Duration metastoreDBRefreshInterval) { - this.metastoreTableRefreshInterval = metastoreTableRefreshInterval; + this.metastoreDBRefreshInterval = metastoreDBRefreshInterval; return this; } diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/metastore/CachingHiveMetastore.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/metastore/CachingHiveMetastore.java index 9eb0ed87a..01a6d6bdd 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/metastore/CachingHiveMetastore.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/metastore/CachingHiveMetastore.java @@ -120,8 +120,8 @@ public class CachingHiveMetastore executor, hiveConfig.getMetastoreCacheTtl(), hiveConfig.getMetastoreRefreshInterval(), - hiveConfig.getMetastoreTableCacheTtl(), - hiveConfig.getMetastoreTableRefreshInterval(), + hiveConfig.getMetastoreDBCacheTtl(), + hiveConfig.getMetastoreDBRefreshInterval(), hiveConfig.getMetastoreCacheMaximumSize(), !(nodeManager.getCurrentNode().isCoordinator() || hiveConfig.getWorkerMetaStoreCacheEnabled())); } @@ -156,16 +156,16 @@ public class CachingHiveMetastore } private CachingHiveMetastore(HiveMetastore delegate, Executor executor, - OptionalLong expiresAfterWriteMillis, OptionalLong refreshMills, OptionalLong expiresAfterWriteMillisTable, OptionalLong refreshMillsTable, + OptionalLong expiresAfterWriteMillisDB, OptionalLong refreshMillsDB, long maximumSize, boolean skipCache) { this.delegate = requireNonNull(delegate, "delegate is null"); requireNonNull(executor, "executor is null"); - // if refreshMills is present and is 0 , keeps cache unrefreshed. + // if refreshMillsDB is present and is 0 , keeps cache unrefreshed. this.skipCache = skipCache - || (refreshMills.isPresent() && refreshMills.getAsLong() == 0); + || (refreshMillsDB.isPresent() && refreshMillsDB.getAsLong() == 0); this.skipTableCache = skipCache || (refreshMillsTable.isPresent() && refreshMillsTable.getAsLong() == 0); @@ -184,16 +184,16 @@ public class CachingHiveMetastore tableRefreshTtl = refreshMillsTable; } - databaseNamesCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize) + databaseNamesCache = newCacheBuilder(expiresAfterWriteMillisDB, refreshMillsDB, maximumSize) .build(asyncReloading(CacheLoader.from(this::loadAllDatabases), executor)); - databaseCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize) + databaseCache = newCacheBuilder(expiresAfterWriteMillisDB, refreshMillsDB, maximumSize) .build(asyncReloading(CacheLoader.from(this::loadDatabase), executor)); - tableNamesCache = newCacheBuilder(expiresAfterWriteMillisTable, refreshMills, maximumSize) + tableNamesCache = newCacheBuilder(expiresAfterWriteMillisTable, refreshMillsDB, maximumSize) .build(asyncReloading(CacheLoader.from(this::loadAllTables), executor)); - viewNamesCache = newCacheBuilder(expiresAfterWriteMillisTable, refreshMills, maximumSize) + viewNamesCache = newCacheBuilder(expiresAfterWriteMillisTable, refreshMillsDB, maximumSize) .build(asyncReloading(CacheLoader.from(this::loadAllViews), executor)); tableCache = newCacheBuilder(tableCacheTtl, tableRefreshTtl, maximumSize) @@ -250,13 +250,13 @@ public class CachingHiveMetastore } }, executor)); - rolesCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize) + rolesCache = newCacheBuilder(expiresAfterWriteMillisDB, refreshMillsDB, maximumSize) .build(asyncReloading(CacheLoader.from(() -> loadRoles()), executor)); - roleGrantsCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize) + roleGrantsCache = newCacheBuilder(expiresAfterWriteMillisDB, refreshMillsDB, maximumSize) .build(asyncReloading(CacheLoader.from(this::loadRoleGrants), executor)); - configValuesCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize) + configValuesCache = newCacheBuilder(expiresAfterWriteMillisDB, refreshMillsDB, maximumSize) .build(asyncReloading(CacheLoader.from(this::loadConfigValue), executor)); } diff --git a/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveConfig.java b/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveConfig.java index 036987f02..d8c0e62bf 100644 --- a/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveConfig.java +++ b/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveConfig.java @@ -46,8 +46,8 @@ public class TestHiveConfig .setAllowCorruptWritesForTesting(false) .setMetastoreCacheTtl(new Duration(0, TimeUnit.SECONDS)) .setMetastoreRefreshInterval(new Duration(1, TimeUnit.SECONDS)) - .setMetastoreTableCacheTtl(new Duration(0, TimeUnit.SECONDS)) - .setMetastoreTableRefreshInterval(new Duration(1, TimeUnit.SECONDS)) + .setMetastoreDBCacheTtl(new Duration(0, TimeUnit.SECONDS)) + .setMetastoreDBRefreshInterval(new Duration(1, TimeUnit.SECONDS)) .setMetastoreCacheMaximumSize(10000) .setPerTransactionMetastoreCacheMaximumSize(1000) .setMaxMetastoreRefreshThreads(100) @@ -165,8 +165,8 @@ public class TestHiveConfig .put("hive.allow-corrupt-writes-for-testing", "true") .put("hive.metastore-cache-ttl", "2h") .put("hive.metastore-refresh-interval", "30m") - .put("hive.metastore-table-cache-ttl", "2h") - .put("hive.metastore-table-refresh-interval", "30m") + .put("hive.metastore-db-cache-ttl", "2h") + .put("hive.metastore-db-refresh-interval", "30m") .put("hive.metastore-cache-maximum-size", "5000") .put("hive.per-transaction-metastore-cache-maximum-size", "500") .put("hive.metastore-refresh-max-threads", "2500") @@ -291,8 +291,8 @@ public class TestHiveConfig .setAllowCorruptWritesForTesting(true) .setMetastoreCacheTtl(new Duration(2, TimeUnit.HOURS)) .setMetastoreRefreshInterval(new Duration(30, TimeUnit.MINUTES)) - .setMetastoreTableCacheTtl(new Duration(2, TimeUnit.HOURS)) - .setMetastoreTableRefreshInterval(new Duration(30, TimeUnit.MINUTES)) + .setMetastoreDBCacheTtl(new Duration(2, TimeUnit.HOURS)) + .setMetastoreDBRefreshInterval(new Duration(30, TimeUnit.MINUTES)) .setMetastoreCacheMaximumSize(5000) .setPerTransactionMetastoreCacheMaximumSize(500) .setMaxMetastoreRefreshThreads(2500)