diff --git a/doc/en/mooncake-store-preview.md b/doc/en/mooncake-store-preview.md index d721f815..9e4c9046 100644 --- a/doc/en/mooncake-store-preview.md +++ b/doc/en/mooncake-store-preview.md @@ -366,6 +366,9 @@ Currently, an approximate LRU policy is adopted, where the least recently used o Each time the eviction task is triggered, in default it will try to evict about 10% of objects. This ratio is configurable via a startup parameter of `master_service`. +To minimize put failures, you can set the eviction high watermark via the `master_service` startup parameter `-eviction_high_watermark_ratio=`(Default to 1). When the eviction thread detects that current space usage reaches the configured high watermark, +it initiates evict operations. The eviction target is to clean an additional `-eviction_ratio` specified proportion beyond the high watermark, thereby reaching the space low watermark. + ### Lease To avoid data conflicts, a per-object lease will be granted whenever an `ExistKey` request or a `GetReplicaListRequest` request succeeds. An object is guaranteed to be protected from `Remove` request, `RemoveAll` request and `Eviction` task until its lease expires. A `Remove` request on a leased object will fail. A `RemoveAll` request will only remove objects without a lease. diff --git a/doc/zh/mooncake-store-preview.md b/doc/zh/mooncake-store-preview.md index a9616635..c31019a6 100644 --- a/doc/zh/mooncake-store-preview.md +++ b/doc/zh/mooncake-store-preview.md @@ -378,6 +378,9 @@ virtual std::shared_ptr Allocate( 每次替换任务被触发时，会尝试换出大约 10% 的对象，这个比例可通过 `master_service` 的启动参数进行配置。 +为了尽力避免 Put 失败，还可以通过 `master_service` 的启动参数 `-eviction_high_watermark_ratio=`(默认为 1) 来设定 eviction 的高水位触发条件。当清理线程发现当前空间使用量达到了设定的高水位， +则开始进行清理工作，清理的目标在高水位基础上再多清理 `-eviction_ratio` 指定的清理比例，从而达到空间低水位。 + ### 租约机制 为避免数据冲突，每当 `ExistKey` 请求或 `GetReplicaListRequest` 请求成功时，系统会为对应对象授予一个租约。在租约过期前，该对象将受到保护，不会被 `Remove`、`RemoveAll` 或替换任务删除。对有租约的对象执行 `Remove` 请求会失败；`RemoveAll` 请求则只会删除没有租约的对象。 diff --git a/docs/source/design/mooncake-store-preview.md b/docs/source/design/mooncake-store-preview.md index 1e599756..ba3edf14 100644 --- a/docs/source/design/mooncake-store-preview.md +++ b/docs/source/design/mooncake-store-preview.md @@ -366,6 +366,9 @@ Currently, an approximate LRU policy is adopted, where the least recently used o Each time the eviction task is triggered, in default it will try to evict about 10% of objects. This ratio is configurable via a startup parameter of `master_service`. +To minimize put failures, you can set the eviction high watermark via the `master_service` startup parameter `-eviction_high_watermark_ratio=`(Default to 1). When the eviction thread detects that current space usage reaches the configured high watermark, +it initiates evict operations. The eviction target is to clean an additional `-eviction_ratio` specified proportion beyond the high watermark, thereby reaching the space low watermark. + ### Lease To avoid data conflicts, a per-object lease will be granted whenever an `ExistKey` request or a `GetReplicaListRequest` request succeeds. An object is guaranteed to be protected from `Remove` request, `RemoveAll` request and `Eviction` task until its lease expires. A `Remove` request on a leased object will fail. A `RemoveAll` request will only remove objects without a lease. diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 714f8340..858f20d3 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -92,7 +92,8 @@ class MasterService { public: MasterService(bool enable_gc = true, uint64_t default_kv_lease_ttl = DEFAULT_DEFAULT_KV_LEASE_TTL, - double eviction_ratio = DEFAULT_EVICTION_RATIO); + double eviction_ratio = DEFAULT_EVICTION_RATIO, + double eviction_high_watermark_ratio = DEFAULT_EVICTION_HIGH_WATERMARK_RATIO); ~MasterService(); /** @@ -183,7 +184,7 @@ class MasterService { void GCThreadFunc(); // Check all shards and try to evict some keys - void BatchEvict(); + void BatchEvict(double eviction_ratio); // Internal data structures struct ObjectMetadata { @@ -259,6 +260,7 @@ class MasterService { // Eviction related members std::atomic need_eviction_{false}; // Set to trigger eviction when not enough space left const double eviction_ratio_; // in range [0.0, 1.0] + const double eviction_high_watermark_ratio_; // in range [0.0, 1.0] // Helper class for accessing metadata with automatic locking and cleanup class MetadataAccessor { diff --git a/mooncake-store/include/rpc_service.h b/mooncake-store/include/rpc_service.h index 0a244df6..d7860afe 100644 --- a/mooncake-store/include/rpc_service.h +++ b/mooncake-store/include/rpc_service.h @@ -64,8 +64,9 @@ class WrappedMasterService { WrappedMasterService(bool enable_gc, uint64_t default_kv_lease_ttl, bool enable_metric_reporting = true, uint16_t http_port = 9003, - double eviction_ratio = DEFAULT_EVICTION_RATIO) - : master_service_(enable_gc, default_kv_lease_ttl, eviction_ratio), + double eviction_ratio = DEFAULT_EVICTION_RATIO, + double eviction_low_watermark_ratio = DEFAULT_EVICTION_HIGH_WATERMARK_RATIO) + : master_service_(enable_gc, default_kv_lease_ttl, eviction_ratio, eviction_low_watermark_ratio), http_server_(4, http_port), metric_report_running_(enable_metric_reporting) { // Initialize HTTP server for metrics diff --git a/mooncake-store/include/types.h b/mooncake-store/include/types.h index c9f45575..dc687ea1 100644 --- a/mooncake-store/include/types.h +++ b/mooncake-store/include/types.h @@ -21,6 +21,7 @@ static constexpr uint64_t DEFAULT_VALUE = UINT64_MAX; static constexpr uint64_t ERRNO_BASE = DEFAULT_VALUE - 1000; static constexpr uint64_t DEFAULT_DEFAULT_KV_LEASE_TTL = 200; // in milliseconds static constexpr double DEFAULT_EVICTION_RATIO = 0.1; +static constexpr double DEFAULT_EVICTION_HIGH_WATERMARK_RATIO = 1.0; // Forward declarations class BufferAllocator; diff --git a/mooncake-store/src/master.cpp b/mooncake-store/src/master.cpp index 903db949..ca31d385 100644 --- a/mooncake-store/src/master.cpp +++ b/mooncake-store/src/master.cpp @@ -19,6 +19,7 @@ DEFINE_int32(metrics_port, 9003, "Port for HTTP metrics server to listen on"); DEFINE_uint64(default_kv_lease_ttl, mooncake::DEFAULT_DEFAULT_KV_LEASE_TTL, "Default lease time for kv objects"); DEFINE_double(eviction_ratio, mooncake::DEFAULT_EVICTION_RATIO, "Ratio of objects to evict when storage space is full"); +DEFINE_double(eviction_high_watermark_ratio, mooncake::DEFAULT_EVICTION_HIGH_WATERMARK_RATIO, "Ratio of high watermark trigger eviction"); DEFINE_validator(eviction_ratio, [](const char* flagname, double value) { if (value < 0.0 || value > 1.0) { LOG(FATAL) << "Eviction ratio must be between 0.0 and 1.0"; @@ -44,11 +45,13 @@ int main(int argc, char* argv[]) { << ", enable_metric_reporting=" << FLAGS_enable_metric_reporting << ", metrics_port=" << FLAGS_metrics_port << ", default_kv_lease_ttl=" << FLAGS_default_kv_lease_ttl - << ", eviction_ratio=" << FLAGS_eviction_ratio; + << ", eviction_ratio=" << FLAGS_eviction_ratio + << ", eviction_high_watermark_ratio=" << FLAGS_eviction_high_watermark_ratio; mooncake::WrappedMasterService wrapped_master_service( FLAGS_enable_gc, FLAGS_default_kv_lease_ttl, - FLAGS_enable_metric_reporting, FLAGS_metrics_port, FLAGS_eviction_ratio); + FLAGS_enable_metric_reporting, FLAGS_metrics_port, + FLAGS_eviction_ratio, FLAGS_eviction_high_watermark_ratio); server.register_handler<&mooncake::WrappedMasterService::ExistKey>( &wrapped_master_service); server.register_handler<&mooncake::WrappedMasterService::GetReplicaList>( diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index f7eff74f..a8ddc812 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -67,17 +67,24 @@ ErrorCode BufferAllocatorManager::RemoveSegment( } MasterService::MasterService(bool enable_gc, uint64_t default_kv_lease_ttl, - double eviction_ratio) + double eviction_ratio, + double eviction_high_watermark_ratio) : buffer_allocator_manager_(std::make_shared()), allocation_strategy_(std::make_shared()), enable_gc_(enable_gc), default_kv_lease_ttl_(default_kv_lease_ttl), - eviction_ratio_(eviction_ratio) { + eviction_ratio_(eviction_ratio), + eviction_high_watermark_ratio_(eviction_high_watermark_ratio) { if (eviction_ratio_ < 0.0 || eviction_ratio_ > 1.0) { LOG(ERROR) << "Eviction ratio must be between 0.0 and 1.0, " << "current value: " << eviction_ratio_; throw std::invalid_argument("Invalid eviction ratio"); } + if (eviction_high_watermark_ratio_ < 0.0 || eviction_high_watermark_ratio_ > 1.0) { + LOG(ERROR) << "Eviction high watermark ratio must be between 0.0 and 1.0, " + << "current value: " << eviction_high_watermark_ratio_; + throw std::invalid_argument("Invalid eviction high watermark ratio"); + } gc_running_ = true; gc_thread_ = std::thread(&MasterService::GCThreadFunc, this); VLOG(1) << "action=start_gc_thread"; @@ -474,8 +481,11 @@ void MasterService::GCThreadFunc() { MasterMetricManager::instance().dec_key_count(gc_count); } - if (need_eviction_ && eviction_ratio_ > 0.0) { - BatchEvict(); + double used_ratio = MasterMetricManager::instance().get_global_used_ratio(); + if (used_ratio > eviction_high_watermark_ratio_ + || (need_eviction_ && eviction_ratio_ > 0.0)) { + BatchEvict(std::max(eviction_ratio_, + used_ratio - eviction_high_watermark_ratio_ + eviction_ratio_)); } std::this_thread::sleep_for( @@ -490,7 +500,7 @@ void MasterService::GCThreadFunc() { VLOG(1) << "action=gc_thread_stopped"; } -void MasterService::BatchEvict() { +void MasterService::BatchEvict(double eviction_ratio) { auto now = std::chrono::steady_clock::now(); long evicted_count = 0; long object_count = 0; @@ -509,7 +519,7 @@ void MasterService::BatchEvict() { // To achieve evicted_count / object_count = eviction_ration, // ideally how many object should be evicted in this shard - const long ideal_evict_num = std::ceil(object_count * eviction_ratio_) - evicted_count; + const long ideal_evict_num = std::ceil(object_count * eviction_ratio) - evicted_count; if (ideal_evict_num <= 0) { // No need to evict any object in this shard