From abc4b45af111f1f2fea050fa03dde6d224d02da2 Mon Sep 17 00:00:00 2001 From: Ankita Kejriwal Date: Thu, 10 Nov 2022 18:56:43 -0800 Subject: [PATCH] Set the storage quota on tenant groups instead of tenants Update all the relevant data structures and monitors accordingly. --- fdbclient/ManagementAPI.actor.cpp | 8 +- fdbclient/SystemData.cpp | 4 +- .../include/fdbclient/ManagementAPI.actor.h | 6 +- fdbclient/include/fdbclient/SystemData.h | 4 +- fdbserver/TenantCache.actor.cpp | 78 +++++++++++-------- fdbserver/include/fdbserver/TenantCache.h | 6 +- 6 files changed, 61 insertions(+), 45 deletions(-) diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index 2ec6c419ef..d6043a2091 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -2559,15 +2559,15 @@ bool schemaMatch(json_spirit::mValue const& schemaValue, } } -void setStorageQuota(Transaction& tr, StringRef tenantName, int64_t quota) { +void setStorageQuota(Transaction& tr, StringRef tenantGroupName, int64_t quota) { tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - auto key = storageQuotaKey(tenantName); + auto key = storageQuotaKey(tenantGroupName); tr.set(key, BinaryWriter::toValue(quota, Unversioned())); } -ACTOR Future> getStorageQuota(Transaction* tr, StringRef tenantName) { +ACTOR Future> getStorageQuota(Transaction* tr, StringRef tenantGroupName) { tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); - state Optional v = wait(tr->get(storageQuotaKey(tenantName))); + state Optional v = wait(tr->get(storageQuotaKey(tenantGroupName))); if (!v.present()) { return Optional(); } diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 39aca67c2e..153804fb1e 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -1663,8 +1663,8 @@ BlobWorkerInterface decodeBlobWorkerListValue(ValueRef const& value) { const KeyRangeRef storageQuotaKeys("\xff/storageQuota/"_sr, "\xff/storageQuota0"_sr); const KeyRef storageQuotaPrefix = storageQuotaKeys.begin; -Key storageQuotaKey(StringRef tenantName) { - return tenantName.withPrefix(storageQuotaPrefix); +Key storageQuotaKey(StringRef tenantGroupName) { + return tenantGroupName.withPrefix(storageQuotaPrefix); } const KeyRangeRef idempotencyIdKeys("\xff\x02/idmp/"_sr, "\xff\x02/idmp0"_sr); diff --git a/fdbclient/include/fdbclient/ManagementAPI.actor.h b/fdbclient/include/fdbclient/ManagementAPI.actor.h index e220f0b156..3ff7097eaa 100644 --- a/fdbclient/include/fdbclient/ManagementAPI.actor.h +++ b/fdbclient/include/fdbclient/ManagementAPI.actor.h @@ -163,9 +163,9 @@ bool schemaMatch(json_spirit::mValue const& schema, // storage nodes ACTOR Future mgmtSnapCreate(Database cx, Standalone snapCmd, UID snapUID); -// Set and get the storage quota per tenant -void setStorageQuota(Transaction& tr, StringRef tenantName, int64_t quota); -ACTOR Future> getStorageQuota(Transaction* tr, StringRef tenantName); +// Set and get the storage quota per tenant group +void setStorageQuota(Transaction& tr, StringRef tenantGroupName, int64_t quota); +ACTOR Future> getStorageQuota(Transaction* tr, StringRef tenantGroupName); #include "flow/unactorcompiler.h" #endif diff --git a/fdbclient/include/fdbclient/SystemData.h b/fdbclient/include/fdbclient/SystemData.h index 1393dc63b1..d06b91f64b 100644 --- a/fdbclient/include/fdbclient/SystemData.h +++ b/fdbclient/include/fdbclient/SystemData.h @@ -711,10 +711,10 @@ const Value blobWorkerListValue(BlobWorkerInterface const& interface); BlobWorkerInterface decodeBlobWorkerListValue(ValueRef const& value); // Storage quota per tenant -// "\xff/storageQuota/[[tenantName]]" := "[[quota]]" +// "\xff/storageQuota/[[tenantGroupName]]" := "[[quota]]" extern const KeyRangeRef storageQuotaKeys; extern const KeyRef storageQuotaPrefix; -Key storageQuotaKey(StringRef tenantName); +Key storageQuotaKey(StringRef tenantGroupName); extern const KeyRangeRef idempotencyIdKeys; extern const KeyRef idempotencyIdsExpiredVersion; diff --git a/fdbserver/TenantCache.actor.cpp b/fdbserver/TenantCache.actor.cpp index 928a66a15e..c980a8c65c 100644 --- a/fdbserver/TenantCache.actor.cpp +++ b/fdbserver/TenantCache.actor.cpp @@ -127,25 +127,38 @@ public: loop { state double fetchStartTime = now(); - state std::vector tenants = tenantCache->getTenantList(); + state std::vector groups; + for (const auto& [group, storage] : tenantCache->tenantStorageMap) { + groups.push_back(group); + } state int i; - for (i = 0; i < tenants.size(); i++) { - state ReadYourWritesTransaction tr(tenantCache->dbcx(), tenants[i]); - loop { - try { - state int64_t size = wait(tr.getEstimatedRangeSizeBytes(normalKeys)); - tenantCache->tenantStorageMap[tenants[i]].usage = size; - break; - } catch (Error& e) { - if (e.code() == error_code_tenant_not_found) { - tenantCache->tenantStorageMap.erase(tenants[i]); + for (i = 0; i < groups.size(); i++) { + state TenantGroupName group = groups[i]; + state int64_t usage = 0; + // `tenants` needs to be a copy so that the erase (below) or inserts/erases from other + // functions (when this actor yields) do not interfere with the iteration + state std::unordered_set tenants = tenantCache->tenantStorageMap[group].tenants; + state std::unordered_set::iterator iter = tenants.begin(); + for (; iter != tenants.end(); iter++) { + state TenantName tenant = *iter; + state ReadYourWritesTransaction tr(tenantCache->dbcx(), tenant); + loop { + try { + state int64_t size = wait(tr.getEstimatedRangeSizeBytes(normalKeys)); + usage += size; break; - } else { - TraceEvent("TenantCacheGetStorageUsageError", tenantCache->id()).error(e); - wait(tr.onError(e)); + } catch (Error& e) { + if (e.code() == error_code_tenant_not_found) { + tenantCache->tenantStorageMap[group].tenants.erase(tenant); + break; + } else { + TraceEvent("TenantCacheGetStorageUsageError", tenantCache->id()).error(e); + wait(tr.onError(e)); + } } } } + tenantCache->tenantStorageMap[group].usage = usage; } lastTenantListFetchTime = now(); @@ -162,22 +175,19 @@ public: state Transaction tr(tenantCache->dbcx()); loop { - loop { - try { - state RangeResult currentQuotas = wait(tr.getRange(storageQuotaKeys, CLIENT_KNOBS->TOO_MANY)); - for (auto const kv : currentQuotas) { - TenantName const tenant = kv.key.removePrefix(storageQuotaPrefix); - int64_t const quota = BinaryReader::fromStringRef(kv.value, Unversioned()); - tenantCache->tenantStorageMap[tenant].quota = quota; - } - tr.reset(); - break; - } catch (Error& e) { - TraceEvent("TenantCacheGetStorageQuotaError", tenantCache->id()).error(e); - wait(tr.onError(e)); + try { + state RangeResult currentQuotas = wait(tr.getRange(storageQuotaKeys, CLIENT_KNOBS->TOO_MANY)); + for (const auto kv : currentQuotas) { + const TenantGroupName group = kv.key.removePrefix(storageQuotaPrefix); + const int64_t quota = BinaryReader::fromStringRef(kv.value, Unversioned()); + tenantCache->tenantStorageMap[group].quota = quota; } + tr.reset(); + wait(delay(SERVER_KNOBS->TENANT_CACHE_STORAGE_QUOTA_REFRESH_INTERVAL)); + } catch (Error& e) { + TraceEvent("TenantCacheGetStorageQuotaError", tenantCache->id()).error(e); + wait(tr.onError(e)); } - wait(delay(SERVER_KNOBS->TENANT_CACHE_STORAGE_QUOTA_REFRESH_INTERVAL)); } } }; @@ -189,6 +199,10 @@ void TenantCache::insert(TenantName& tenantName, TenantMapEntry& tenant) { TenantInfo tenantInfo(tenantName, Optional>(), tenant.id); tenantCache[tenantPrefix] = makeReference(tenantInfo, tenant.prefix); tenantCache[tenantPrefix]->updateCacheGeneration(generation); + + if (tenant.tenantGroup.present()) { + tenantStorageMap[tenant.tenantGroup.get()].tenants.insert(tenantName); + } } void TenantCache::startRefresh() { @@ -289,13 +303,13 @@ Optional> TenantCache::tenantOwning(KeyRef key) const { } std::unordered_set TenantCache::getTenantsOverQuota() const { - std::unordered_set tenants; - for (const auto& [tenant, storage] : tenantStorageMap) { + std::unordered_set tenantsOverQuota; + for (const auto& [tenantGroup, storage] : tenantStorageMap) { if (storage.usage > storage.quota) { - tenants.insert(tenant); + tenantsOverQuota.insert(storage.tenants.begin(), storage.tenants.end()); } } - return tenants; + return tenantsOverQuota; } Future TenantCache::monitorTenantMap() { diff --git a/fdbserver/include/fdbserver/TenantCache.h b/fdbserver/include/fdbserver/TenantCache.h index 335a785b6c..ef7a3d8a01 100644 --- a/fdbserver/include/fdbserver/TenantCache.h +++ b/fdbserver/include/fdbserver/TenantCache.h @@ -35,8 +35,9 @@ typedef Map> TenantMapByPrefix; struct Storage { int64_t quota = std::numeric_limits::max(); int64_t usage = 0; + std::unordered_set tenants; }; -typedef std::unordered_map TenantStorageMap; +typedef std::unordered_map TenantStorageMap; struct TenantCacheTenantCreated { KeyRange keys; @@ -56,7 +57,8 @@ private: uint64_t generation; TenantMapByPrefix tenantCache; - // Map from tenant names to storage quota and usage + // Map from tenant group names to the list of tenants, cumumlative storage used by + // all the tenants in the group, and its storage quota. TenantStorageMap tenantStorageMap; // mark the start of a new sweep of the tenant cache