jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!935 [I3V5FH] config names and documentation update for hive metastore cache related configurations 

Merge pull request !935 from Nitin-Kashyap/metacache.VerificationFixes
This commit is contained in:
i-robot 2021-06-17 22:57:07 +08:00 committed by Gitee
parent 8cc975e0e8 d3c27578d5
commit 2ab3d5d8c9
4 changed files with 56 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hetu-docs/en/connector/hive.md
View File

@ -128,6 +128,10 @@ Please see the [Hive Security Configuration](./hive-security.md) section for a m
| `hive.metastore-client-service-threads` | Number of threads for metastore clients to operate in parallel to communicate with hive metastore. | 4 |
| `hive.worker-metastore-cache-enabled` | Enable the caching of the hive metastore on the worker nodes also. | `false` |
| `hive.metastore-write-batch-size` | Number of partitions sent to meta store in per request. | `8` |
| `hive.metastore-cache-ttl` | Metastore Cache eviction time for table & partition metadata. | `0s` |
| `hive.metastore-refresh-interval` | Time after which metastore cache entries for table and partition metadata are refreshed from Hive metastore. | `1s` |
| `hive.metastore-db-cache-ttl` | Metastore Cache eviction time for DB, Roles, Configs, Table & Views list objects. | `0s` |
| `hive.metastore-db-refresh-interval` | Time after which metastore cache entry is refreshed from Hive metastore for DB, Table List, View List, Roles objects. | `1s` |
@ -712,6 +716,28 @@ Drop a schema:
DROP SCHEMA hive.web
```
## Metastore Cache:
Hive connector maintains a metastore cache to service the metastore request faster to various operations. Loading, reloading and retention times of the cache entries can be configured in `hive.properties`.
```properties
# Table & Partition Cache specific configurations
hive.metastore-cache-ttl=24h
hive.metastore-refresh-interval=23h
# DB, Table & View list, Roles, configurations related cache configuration
hive.metastore-db-cache-ttl=4m
hive.metastore-db-refresh-interval=3m
```
**Note:** In cases where user operates on the data directly and if hive metastore is modified externally (eg. directly by Hive, Spark), there is a possibility of cache having older data. For the same user should configure the cache refresh and eviction times accordingly.
In order reduce the inconsistency, hive connector also validates the partition & its statistics cache entries `on read` against table and partition-names cache (which refreshes at higher frequency) in case the table refresh time is higher than `5mins`.
```sql
REFRESH META CACHE
```
Additionally, metadata cache refresh command can be used to reload the metastore cache by user.
## Performance tuning notes:

24
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java
View File

@ -86,8 +86,8 @@ public class HiveConfig
private Duration metastoreCacheTtl = new Duration(0, TimeUnit.SECONDS);
private Duration metastoreRefreshInterval = new Duration(1, TimeUnit.SECONDS);
private Duration metastoreTableCacheTtl = new Duration(0, TimeUnit.SECONDS);
private Duration metastoreTableRefreshInterval = new Duration(1, TimeUnit.SECONDS);
private Duration metastoreDBCacheTtl = new Duration(0, TimeUnit.SECONDS);
private Duration metastoreDBRefreshInterval = new Duration(1, TimeUnit.SECONDS);
private long metastoreCacheMaximumSize = 10000;
private long perTransactionMetastoreCacheMaximumSize = 1000;
@ -451,28 +451,28 @@ public class HiveConfig
}
@NotNull
public @MinDuration("0ms") Duration getMetastoreTableCacheTtl()
public @MinDuration("0ms") Duration getMetastoreDBCacheTtl()
{
return metastoreTableCacheTtl;
return metastoreDBCacheTtl;
}
@Config("hive.metastore-table-cache-ttl")
public HiveConfig setMetastoreTableCacheTtl(Duration metastoreCacheTtl)
@Config("hive.metastore-db-cache-ttl")
public HiveConfig setMetastoreDBCacheTtl(Duration metastoreCacheTtl)
{
this.metastoreTableCacheTtl = metastoreCacheTtl;
this.metastoreDBCacheTtl = metastoreCacheTtl;
return this;
}
@NotNull
public @MinDuration("1ms") Duration getMetastoreTableRefreshInterval()
public @MinDuration("1ms") Duration getMetastoreDBRefreshInterval()
{
return metastoreTableRefreshInterval;
return metastoreDBRefreshInterval;
}
@Config("hive.metastore-table-refresh-interval")
public HiveConfig setMetastoreTableRefreshInterval(Duration metastoreTableRefreshInterval)
@Config("hive.metastore-db-refresh-interval")
public HiveConfig setMetastoreDBRefreshInterval(Duration metastoreDBRefreshInterval)
{
this.metastoreTableRefreshInterval = metastoreTableRefreshInterval;
this.metastoreDBRefreshInterval = metastoreDBRefreshInterval;
return this;
}

24
presto-hive/src/main/java/io/prestosql/plugin/hive/metastore/CachingHiveMetastore.java
View File

@ -120,8 +120,8 @@ public class CachingHiveMetastore
executor,
hiveConfig.getMetastoreCacheTtl(),
hiveConfig.getMetastoreRefreshInterval(),
hiveConfig.getMetastoreTableCacheTtl(),
hiveConfig.getMetastoreTableRefreshInterval(),
hiveConfig.getMetastoreDBCacheTtl(),
hiveConfig.getMetastoreDBRefreshInterval(),
hiveConfig.getMetastoreCacheMaximumSize(),
!(nodeManager.getCurrentNode().isCoordinator() || hiveConfig.getWorkerMetaStoreCacheEnabled()));
}
@ -156,16 +156,16 @@ public class CachingHiveMetastore
}
private CachingHiveMetastore(HiveMetastore delegate, Executor executor,
OptionalLong expiresAfterWriteMillis, OptionalLong refreshMills,
OptionalLong expiresAfterWriteMillisTable, OptionalLong refreshMillsTable,
OptionalLong expiresAfterWriteMillisDB, OptionalLong refreshMillsDB,
long maximumSize, boolean skipCache)
{
this.delegate = requireNonNull(delegate, "delegate is null");
requireNonNull(executor, "executor is null");
// if refreshMills is present and is 0 , keeps cache unrefreshed.
// if refreshMillsDB is present and is 0 , keeps cache unrefreshed.
this.skipCache = skipCache
|| (refreshMills.isPresent() && refreshMills.getAsLong() == 0);
|| (refreshMillsDB.isPresent() && refreshMillsDB.getAsLong() == 0);
this.skipTableCache = skipCache
|| (refreshMillsTable.isPresent() && refreshMillsTable.getAsLong() == 0);
@ -184,16 +184,16 @@ public class CachingHiveMetastore
tableRefreshTtl = refreshMillsTable;
}
databaseNamesCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize)
databaseNamesCache = newCacheBuilder(expiresAfterWriteMillisDB, refreshMillsDB, maximumSize)
.build(asyncReloading(CacheLoader.from(this::loadAllDatabases), executor));
databaseCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize)
databaseCache = newCacheBuilder(expiresAfterWriteMillisDB, refreshMillsDB, maximumSize)
.build(asyncReloading(CacheLoader.from(this::loadDatabase), executor));
tableNamesCache = newCacheBuilder(expiresAfterWriteMillisTable, refreshMills, maximumSize)
tableNamesCache = newCacheBuilder(expiresAfterWriteMillisTable, refreshMillsDB, maximumSize)
.build(asyncReloading(CacheLoader.from(this::loadAllTables), executor));
viewNamesCache = newCacheBuilder(expiresAfterWriteMillisTable, refreshMills, maximumSize)
viewNamesCache = newCacheBuilder(expiresAfterWriteMillisTable, refreshMillsDB, maximumSize)
.build(asyncReloading(CacheLoader.from(this::loadAllViews), executor));
tableCache = newCacheBuilder(tableCacheTtl, tableRefreshTtl, maximumSize)
@ -250,13 +250,13 @@ public class CachingHiveMetastore
}
}, executor));
rolesCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize)
rolesCache = newCacheBuilder(expiresAfterWriteMillisDB, refreshMillsDB, maximumSize)
.build(asyncReloading(CacheLoader.from(() -> loadRoles()), executor));
roleGrantsCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize)
roleGrantsCache = newCacheBuilder(expiresAfterWriteMillisDB, refreshMillsDB, maximumSize)
.build(asyncReloading(CacheLoader.from(this::loadRoleGrants), executor));
configValuesCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize)
configValuesCache = newCacheBuilder(expiresAfterWriteMillisDB, refreshMillsDB, maximumSize)
.build(asyncReloading(CacheLoader.from(this::loadConfigValue), executor));
}

12
presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveConfig.java
View File

@ -46,8 +46,8 @@ public class TestHiveConfig
.setAllowCorruptWritesForTesting(false)
.setMetastoreCacheTtl(new Duration(0, TimeUnit.SECONDS))
.setMetastoreRefreshInterval(new Duration(1, TimeUnit.SECONDS))
.setMetastoreTableCacheTtl(new Duration(0, TimeUnit.SECONDS))
.setMetastoreTableRefreshInterval(new Duration(1, TimeUnit.SECONDS))
.setMetastoreDBCacheTtl(new Duration(0, TimeUnit.SECONDS))
.setMetastoreDBRefreshInterval(new Duration(1, TimeUnit.SECONDS))
.setMetastoreCacheMaximumSize(10000)
.setPerTransactionMetastoreCacheMaximumSize(1000)
.setMaxMetastoreRefreshThreads(100)
@ -165,8 +165,8 @@ public class TestHiveConfig
.put("hive.allow-corrupt-writes-for-testing", "true")
.put("hive.metastore-cache-ttl", "2h")
.put("hive.metastore-refresh-interval", "30m")
.put("hive.metastore-table-cache-ttl", "2h")
.put("hive.metastore-table-refresh-interval", "30m")
.put("hive.metastore-db-cache-ttl", "2h")
.put("hive.metastore-db-refresh-interval", "30m")
.put("hive.metastore-cache-maximum-size", "5000")
.put("hive.per-transaction-metastore-cache-maximum-size", "500")
.put("hive.metastore-refresh-max-threads", "2500")
@ -291,8 +291,8 @@ public class TestHiveConfig
.setAllowCorruptWritesForTesting(true)
.setMetastoreCacheTtl(new Duration(2, TimeUnit.HOURS))
.setMetastoreRefreshInterval(new Duration(30, TimeUnit.MINUTES))
.setMetastoreTableCacheTtl(new Duration(2, TimeUnit.HOURS))
.setMetastoreTableRefreshInterval(new Duration(30, TimeUnit.MINUTES))
.setMetastoreDBCacheTtl(new Duration(2, TimeUnit.HOURS))
.setMetastoreDBRefreshInterval(new Duration(30, TimeUnit.MINUTES))
.setMetastoreCacheMaximumSize(5000)
.setPerTransactionMetastoreCacheMaximumSize(500)
.setMaxMetastoreRefreshThreads(2500)