wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Set the storage quota on tenant groups instead of tenants 

Update all the relevant data structures and monitors accordingly.
This commit is contained in:
Ankita Kejriwal 2022-11-10 18:56:43 -08:00
parent 012465ec26
commit abc4b45af1
6 changed files with 61 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
fdbclient/ManagementAPI.actor.cpp
View File

@ -2559,15 +2559,15 @@ bool schemaMatch(json_spirit::mValue const& schemaValue,
}
}
void setStorageQuota(Transaction& tr, StringRef tenantName, int64_t quota) {
void setStorageQuota(Transaction& tr, StringRef tenantGroupName, int64_t quota) {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
auto key = storageQuotaKey(tenantName);
auto key = storageQuotaKey(tenantGroupName);
tr.set(key, BinaryWriter::toValue<int64_t>(quota, Unversioned()));
}
ACTOR Future<Optional<int64_t>> getStorageQuota(Transaction* tr, StringRef tenantName) {
ACTOR Future<Optional<int64_t>> getStorageQuota(Transaction* tr, StringRef tenantGroupName) {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
state Optional<Value> v = wait(tr->get(storageQuotaKey(tenantName)));
state Optional<Value> v = wait(tr->get(storageQuotaKey(tenantGroupName)));
if (!v.present()) {
return Optional<int64_t>();
}

4
fdbclient/SystemData.cpp
View File

@ -1663,8 +1663,8 @@ BlobWorkerInterface decodeBlobWorkerListValue(ValueRef const& value) {
const KeyRangeRef storageQuotaKeys("\xff/storageQuota/"_sr, "\xff/storageQuota0"_sr);
const KeyRef storageQuotaPrefix = storageQuotaKeys.begin;
Key storageQuotaKey(StringRef tenantName) {
return tenantName.withPrefix(storageQuotaPrefix);
Key storageQuotaKey(StringRef tenantGroupName) {
return tenantGroupName.withPrefix(storageQuotaPrefix);
}
const KeyRangeRef idempotencyIdKeys("\xff\x02/idmp/"_sr, "\xff\x02/idmp0"_sr);

6
fdbclient/include/fdbclient/ManagementAPI.actor.h
View File

@ -163,9 +163,9 @@ bool schemaMatch(json_spirit::mValue const& schema,
// storage nodes
ACTOR Future<Void> mgmtSnapCreate(Database cx, Standalone<StringRef> snapCmd, UID snapUID);
// Set and get the storage quota per tenant
void setStorageQuota(Transaction& tr, StringRef tenantName, int64_t quota);
ACTOR Future<Optional<int64_t>> getStorageQuota(Transaction* tr, StringRef tenantName);
// Set and get the storage quota per tenant group
void setStorageQuota(Transaction& tr, StringRef tenantGroupName, int64_t quota);
ACTOR Future<Optional<int64_t>> getStorageQuota(Transaction* tr, StringRef tenantGroupName);
#include "flow/unactorcompiler.h"
#endif

4
fdbclient/include/fdbclient/SystemData.h
View File

@ -711,10 +711,10 @@ const Value blobWorkerListValue(BlobWorkerInterface const& interface);
BlobWorkerInterface decodeBlobWorkerListValue(ValueRef const& value);
// Storage quota per tenant
// "\xff/storageQuota/[[tenantName]]" := "[[quota]]"
// "\xff/storageQuota/[[tenantGroupName]]" := "[[quota]]"
extern const KeyRangeRef storageQuotaKeys;
extern const KeyRef storageQuotaPrefix;
Key storageQuotaKey(StringRef tenantName);
Key storageQuotaKey(StringRef tenantGroupName);
extern const KeyRangeRef idempotencyIdKeys;
extern const KeyRef idempotencyIdsExpiredVersion;

78
fdbserver/TenantCache.actor.cpp
View File

@ -127,25 +127,38 @@ public:
loop {
state double fetchStartTime = now();
state std::vector<TenantName> tenants = tenantCache->getTenantList();
state std::vector<TenantGroupName> groups;
for (const auto& [group, storage] : tenantCache->tenantStorageMap) {
groups.push_back(group);
}
state int i;
for (i = 0; i < tenants.size(); i++) {
state ReadYourWritesTransaction tr(tenantCache->dbcx(), tenants[i]);
loop {
try {
state int64_t size = wait(tr.getEstimatedRangeSizeBytes(normalKeys));
tenantCache->tenantStorageMap[tenants[i]].usage = size;
break;
} catch (Error& e) {
if (e.code() == error_code_tenant_not_found) {
tenantCache->tenantStorageMap.erase(tenants[i]);
for (i = 0; i < groups.size(); i++) {
state TenantGroupName group = groups[i];
state int64_t usage = 0;
// `tenants` needs to be a copy so that the erase (below) or inserts/erases from other
// functions (when this actor yields) do not interfere with the iteration
state std::unordered_set<TenantName> tenants = tenantCache->tenantStorageMap[group].tenants;
state std::unordered_set<TenantName>::iterator iter = tenants.begin();
for (; iter != tenants.end(); iter++) {
state TenantName tenant = *iter;
state ReadYourWritesTransaction tr(tenantCache->dbcx(), tenant);
loop {
try {
state int64_t size = wait(tr.getEstimatedRangeSizeBytes(normalKeys));
usage += size;
break;
} else {
TraceEvent("TenantCacheGetStorageUsageError", tenantCache->id()).error(e);
wait(tr.onError(e));
} catch (Error& e) {
if (e.code() == error_code_tenant_not_found) {
tenantCache->tenantStorageMap[group].tenants.erase(tenant);
break;
} else {
TraceEvent("TenantCacheGetStorageUsageError", tenantCache->id()).error(e);
wait(tr.onError(e));
}
}
}
}
tenantCache->tenantStorageMap[group].usage = usage;
}
lastTenantListFetchTime = now();
@ -162,22 +175,19 @@ public:
state Transaction tr(tenantCache->dbcx());
loop {
loop {
try {
state RangeResult currentQuotas = wait(tr.getRange(storageQuotaKeys, CLIENT_KNOBS->TOO_MANY));
for (auto const kv : currentQuotas) {
TenantName const tenant = kv.key.removePrefix(storageQuotaPrefix);
int64_t const quota = BinaryReader::fromStringRef<int64_t>(kv.value, Unversioned());
tenantCache->tenantStorageMap[tenant].quota = quota;
}
tr.reset();
break;
} catch (Error& e) {
TraceEvent("TenantCacheGetStorageQuotaError", tenantCache->id()).error(e);
wait(tr.onError(e));
try {
state RangeResult currentQuotas = wait(tr.getRange(storageQuotaKeys, CLIENT_KNOBS->TOO_MANY));
for (const auto kv : currentQuotas) {
const TenantGroupName group = kv.key.removePrefix(storageQuotaPrefix);
const int64_t quota = BinaryReader::fromStringRef<int64_t>(kv.value, Unversioned());
tenantCache->tenantStorageMap[group].quota = quota;
}
tr.reset();
wait(delay(SERVER_KNOBS->TENANT_CACHE_STORAGE_QUOTA_REFRESH_INTERVAL));
} catch (Error& e) {
TraceEvent("TenantCacheGetStorageQuotaError", tenantCache->id()).error(e);
wait(tr.onError(e));
}
wait(delay(SERVER_KNOBS->TENANT_CACHE_STORAGE_QUOTA_REFRESH_INTERVAL));
}
}
};
@ -189,6 +199,10 @@ void TenantCache::insert(TenantName& tenantName, TenantMapEntry& tenant) {
TenantInfo tenantInfo(tenantName, Optional<Standalone<StringRef>>(), tenant.id);
tenantCache[tenantPrefix] = makeReference<TCTenantInfo>(tenantInfo, tenant.prefix);
tenantCache[tenantPrefix]->updateCacheGeneration(generation);
if (tenant.tenantGroup.present()) {
tenantStorageMap[tenant.tenantGroup.get()].tenants.insert(tenantName);
}
}
void TenantCache::startRefresh() {
@ -289,13 +303,13 @@ Optional<Reference<TCTenantInfo>> TenantCache::tenantOwning(KeyRef key) const {
}
std::unordered_set<TenantName> TenantCache::getTenantsOverQuota() const {
std::unordered_set<TenantName> tenants;
for (const auto& [tenant, storage] : tenantStorageMap) {
std::unordered_set<TenantName> tenantsOverQuota;
for (const auto& [tenantGroup, storage] : tenantStorageMap) {
if (storage.usage > storage.quota) {
tenants.insert(tenant);
tenantsOverQuota.insert(storage.tenants.begin(), storage.tenants.end());
}
}
return tenants;
return tenantsOverQuota;
}
Future<Void> TenantCache::monitorTenantMap() {

6
fdbserver/include/fdbserver/TenantCache.h
View File

@ -35,8 +35,9 @@ typedef Map<KeyRef, Reference<TCTenantInfo>> TenantMapByPrefix;
struct Storage {
int64_t quota = std::numeric_limits<int64_t>::max();
int64_t usage = 0;
std::unordered_set<TenantName> tenants;
};
typedef std::unordered_map<TenantName, Storage> TenantStorageMap;
typedef std::unordered_map<TenantGroupName, Storage> TenantStorageMap;
struct TenantCacheTenantCreated {
KeyRange keys;
@ -56,7 +57,8 @@ private:
uint64_t generation;
TenantMapByPrefix tenantCache;
// Map from tenant names to storage quota and usage
// Map from tenant group names to the list of tenants, cumumlative storage used by
// all the tenants in the group, and its storage quota.
TenantStorageMap tenantStorageMap;
// mark the start of a new sweep of the tenant cache