From f76c92295bdbefb52b0de1c834312f543738ba1f Mon Sep 17 00:00:00 2001 From: Teng Ma Date: Mon, 25 Aug 2025 13:56:37 +0800 Subject: [PATCH] [Store] add c++ http metadata server in mooncake master (#766) * [Store] config: start http server from master Signed-off-by: Teng Ma * [Refactor] Use coro http server for metadata * fix cmake * fix merge * add doc and test * clear --------- Signed-off-by: Teng Ma --- doc/en/mooncake-store-preview.md | 29 +++++ doc/zh/mooncake-store-preview.md | 29 +++++ mooncake-store/include/http_metadata_server.h | 54 ++++++++ mooncake-store/include/master_config.h | 5 + mooncake-store/src/CMakeLists.txt | 1 + mooncake-store/src/http_metadata_server.cpp | 122 ++++++++++++++++++ mooncake-store/src/master.cpp | 85 +++++++++++- scripts/run_tests.sh | 10 ++ 8 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 mooncake-store/include/http_metadata_server.h create mode 100644 mooncake-store/src/http_metadata_server.cpp diff --git a/doc/en/mooncake-store-preview.md b/doc/en/mooncake-store-preview.md index 4e4321cd..8a75a7f9 100644 --- a/doc/en/mooncake-store-preview.md +++ b/doc/en/mooncake-store-preview.md @@ -535,6 +535,35 @@ After enabling the persistence feature: #### 3FS USRBIO Plugin If you need to use 3FS's native API (USRBIO) to achieve high-performance persistent file reads and writes, you can refer to the configuration instructions in this document [3FS USRBIO Plugin](/mooncake-store/src/hf3fs/README.md). +### Builtin Metadata Server + +Mooncake Store provides a built-in HTTP metadata server as an alternative to etcd for storing cluster metadata. This feature is particularly useful for development environments or scenarios where etcd is not available. + +#### Configuration Parameters + +The HTTP metadata server can be configured using the following parameters: + +- **`enable_http_metadata_server`** (boolean, default: `false`): Enables the built-in HTTP metadata server instead of using etcd. When set to `true`, the master service will start an embedded HTTP server that handles metadata operations. + +- **`http_metadata_server_port`** (integer, default: `8080`): Specifies the TCP port on which the HTTP metadata server will listen for incoming connections. This port must be available and not conflict with other services. + +- **`http_metadata_server_host`** (string, default: `"0.0.0.0"`): Specifies the host address for the HTTP metadata server to bind to. Use `"0.0.0.0"` to listen on all available network interfaces, or specify a specific IP address for security purposes. + +#### Usage Example + +To start the master service with the HTTP metadata server enabled: + +```bash +./build/mooncake-store/src/mooncake_master \ + --enable_http_metadata_server=true \ + --http_metadata_server_port=8080 \ + --http_metadata_server_host=0.0.0.0 +``` + +When enabled, the HTTP metadata server will start automatically and provide metadata services for the Mooncake Store cluster. This eliminates the need for an external etcd deployment, simplifying the setup process for development and testing environments. + +Note that the HTTP metadata server is designed for single-node deployments and does not provide the high availability features that etcd offers. For production environments requiring high availability, etcd is still the recommended choice. + ## Mooncake Store Python API **Complete Python API Documentation**: [https://kvcache-ai.github.io/Mooncake/mooncake-store-api/python-binding.html](https://kvcache-ai.github.io/Mooncake/mooncake-store-api/python-binding.html) diff --git a/doc/zh/mooncake-store-preview.md b/doc/zh/mooncake-store-preview.md index 3cbafe4d..17fb72b3 100644 --- a/doc/zh/mooncake-store-preview.md +++ b/doc/zh/mooncake-store-preview.md @@ -537,6 +537,35 @@ struct ReplicateConfig { #### 3FS USRBIO 插件 如需通过3FS原生接口（USRBIO）实现高性能持久化文件读写，请参阅本文档的配置说明。[3FS USRBIO 插件配置](/mooncake-store/src/hf3fs/README.md)。 +### 内置元数据服务器 + +Mooncake Store 提供了内置的 HTTP 元数据服务器作为 etcd 的替代方案，用于存储集群元数据。此功能特别适用于开发环境或 etcd 不可用的场景。 + +#### 配置参数 + +HTTP 元数据服务器可通过以下参数进行配置： + +- **`enable_http_metadata_server`**（布尔值，默认值：`false`）：启用内置的 HTTP 元数据服务器以替代 etcd。当设置为 `true` 时，主服务将启动一个嵌入式 HTTP 服务器来处理元数据操作。 + +- **`http_metadata_server_port`**（整型，默认值：`8080`）：指定 HTTP 元数据服务器监听的 TCP 端口。该端口必须可用且不能与其他服务冲突。 + +- **`http_metadata_server_host`**（字符串，默认值：`"0.0.0.0"`）：指定 HTTP 元数据服务器绑定的主机地址。使用 `"0.0.0.0"` 可监听所有可用网络接口，或指定特定 IP 地址以提高安全性。 + +#### 使用示例 + +要使用启用了 HTTP 元数据服务器的主服务，请运行： + +```bash +./build/mooncake-store/src/mooncake_master \ + --enable_http_metadata_server=true \ + --http_metadata_server_port=8080 \ + --http_metadata_server_host=0.0.0.0 +``` + +启用后，HTTP 元数据服务器将自动启动并为 Mooncake Store 集群提供元数据服务。这消除了对外部 etcd 部署的需求，简化了开发和测试环境的设置过程。 + +请注意，HTTP 元数据服务器专为单节点部署设计，不提供 etcd 所具备的高可用性功能。对于需要高可用性的生产环境，仍推荐使用 etcd。 + ## Mooncake Store Python API **完整的 Python API 文档**: [https://kvcache-ai.github.io/Mooncake/mooncake-store-api/python-binding.html](https://kvcache-ai.github.io/Mooncake/mooncake-store-api/python-binding.html) diff --git a/mooncake-store/include/http_metadata_server.h b/mooncake-store/include/http_metadata_server.h new file mode 100644 index 00000000..5613787e --- /dev/null +++ b/mooncake-store/include/http_metadata_server.h @@ -0,0 +1,54 @@ +#ifndef MOONCAKE_HTTP_METADATA_SERVER_H +#define MOONCAKE_HTTP_METADATA_SERVER_H + +#include +#include +#include + +#include + +namespace mooncake { + +enum class KVPoll { + Failed = 0, + Bootstrapping = 1, + WaitingForInput = 2, + Transferring = 3, + Success = 4 +}; + +class HttpMetadataServer { + public: + HttpMetadataServer(uint16_t port, const std::string& host = "0.0.0.0"); + ~HttpMetadataServer(); + + // Start the HTTP metadata server + bool start(); + + // Stop the HTTP metadata server + void stop(); + + // Poll the server status + KVPoll poll() const; + + // Check if the server is running + bool is_running() const { return running_; } + + // Non-copyable + HttpMetadataServer(const HttpMetadataServer&) = delete; + HttpMetadataServer& operator=(const HttpMetadataServer&) = delete; + + private: + void init_server(); + + uint16_t port_; + std::string host_; + std::unique_ptr server_; + std::unordered_map store_; + mutable std::mutex store_mutex_; + bool running_; +}; + +} // namespace mooncake + +#endif // MOONCAKE_HTTP_METADATA_SERVER_H diff --git a/mooncake-store/include/master_config.h b/mooncake-store/include/master_config.h index 5689cd52..5a118eb2 100644 --- a/mooncake-store/include/master_config.h +++ b/mooncake-store/include/master_config.h @@ -30,6 +30,11 @@ struct MasterConfig { std::string cluster_id; std::string root_fs_dir; std::string memory_allocator; + + // HTTP metadata server configuration + bool enable_http_metadata_server; + uint32_t http_metadata_server_port; + std::string http_metadata_server_host; }; class MasterServiceSupervisorConfig { diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 2e4182dc..f45e4d75 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -23,6 +23,7 @@ set(MOONCAKE_STORE_SOURCES posix_file.cpp client_buffer.cpp pybind_client.cpp + http_metadata_server.cpp ) set(EXTRA_LIBS "") diff --git a/mooncake-store/src/http_metadata_server.cpp b/mooncake-store/src/http_metadata_server.cpp new file mode 100644 index 00000000..dbb8517a --- /dev/null +++ b/mooncake-store/src/http_metadata_server.cpp @@ -0,0 +1,122 @@ +#include "http_metadata_server.h" + +#include +#include + +#include +#include + +namespace mooncake { + +HttpMetadataServer::HttpMetadataServer(uint16_t port, const std::string& host) + : port_(port), + host_(host), + server_(std::make_unique(4, port)), + running_(false) { + init_server(); +} + +HttpMetadataServer::~HttpMetadataServer() { stop(); } + +void HttpMetadataServer::init_server() { + using namespace coro_http; + + // GET /metadata?key= + server_->set_http_handler( + "/metadata", [this](coro_http_request& req, coro_http_response& resp) { + auto key = req.get_query_value("key"); + if (key.empty()) { + resp.set_status_and_content(status_type::bad_request, + "Missing key parameter"); + return; + } + + std::lock_guard lock(store_mutex_); + auto it = store_.find(std::string(key)); + if (it == store_.end()) { + resp.set_status_and_content(status_type::not_found, + "metadata not found"); + return; + } + + resp.add_header("Content-Type", "application/json"); + resp.set_status_and_content(status_type::ok, it->second); + }); + + // PUT /metadata?key= + server_->set_http_handler( + "/metadata", [this](coro_http_request& req, coro_http_response& resp) { + auto key = req.get_query_value("key"); + if (key.empty()) { + resp.set_status_and_content(status_type::bad_request, + "Missing key parameter"); + return; + } + + std::string body(req.get_body()); + { + std::lock_guard lock(store_mutex_); + store_[std::string(key)] = body; + } + + resp.set_status_and_content(status_type::ok, "metadata updated"); + }); + + // DELETE /metadata?key= + server_->set_http_handler( + "/metadata", [this](coro_http_request& req, coro_http_response& resp) { + auto key = req.get_query_value("key"); + if (key.empty()) { + resp.set_status_and_content(status_type::bad_request, + "Missing key parameter"); + return; + } + + std::lock_guard lock(store_mutex_); + auto it = store_.find(std::string(key)); + if (it == store_.end()) { + resp.set_status_and_content(status_type::not_found, + "metadata not found"); + return; + } + + store_.erase(it); + resp.set_status_and_content(status_type::ok, "metadata deleted"); + }); + + // Health check endpoint + server_->set_http_handler( + "/health", [](coro_http_request& req, coro_http_response& resp) { + resp.set_status_and_content(status_type::ok, "OK"); + }); +} + +bool HttpMetadataServer::start() { + if (running_) { + return true; + } + + server_->async_start(); + running_ = true; + LOG(INFO) << "HTTP metadata server started on " << host_ << ":" << port_; + return true; +} + +void HttpMetadataServer::stop() { + if (!running_) { + return; + } + + server_->stop(); + running_ = false; + LOG(INFO) << "HTTP metadata server stopped"; +} + +KVPoll HttpMetadataServer::poll() const { + if (!running_) { + return KVPoll::Failed; + } + return KVPoll::Success; +} + +} // namespace mooncake diff --git a/mooncake-store/src/master.cpp b/mooncake-store/src/master.cpp index 7fef3c2f..26d991c9 100644 --- a/mooncake-store/src/master.cpp +++ b/mooncake-store/src/master.cpp @@ -1,12 +1,14 @@ #include #include // For std::chrono +#include // For std::unique_ptr #include // For std::thread #include #include #include "default_config.h" #include "ha_helper.h" +#include "http_metadata_server.h" #include "rpc_service.h" #include "types.h" @@ -75,6 +77,12 @@ DEFINE_string(cluster_id, mooncake::DEFAULT_CLUSTER_ID, DEFINE_string(memory_allocator, "offset", "Memory allocator for global segments, cachelib | offset"); +DEFINE_bool(enable_http_metadata_server, false, + "Enable HTTP metadata server instead of etcd"); +DEFINE_int32(http_metadata_server_port, 8080, + "Port for HTTP metadata server to listen on"); +DEFINE_string(http_metadata_server_host, "0.0.0.0", + "Host for HTTP metadata server to bind to"); void InitMasterConf(const mooncake::DefaultConfig& default_config, mooncake::MasterConfig& master_config) { @@ -125,6 +133,15 @@ void InitMasterConf(const mooncake::DefaultConfig& default_config, default_config.GetString("memory_allocator", &master_config.memory_allocator, FLAGS_memory_allocator); + default_config.GetBool("enable_http_metadata_server", + &master_config.enable_http_metadata_server, + FLAGS_enable_http_metadata_server); + default_config.GetUInt32("http_metadata_server_port", + &master_config.http_metadata_server_port, + FLAGS_http_metadata_server_port); + default_config.GetString("http_metadata_server_host", + &master_config.http_metadata_server_host, + FLAGS_http_metadata_server_host); } void LoadConfigFromCmdline(mooncake::MasterConfig& master_config, @@ -258,6 +275,48 @@ void LoadConfigFromCmdline(mooncake::MasterConfig& master_config, !conf_set) { master_config.memory_allocator = FLAGS_memory_allocator; } + if ((google::GetCommandLineFlagInfo("enable_http_metadata_server", &info) && + !info.is_default) || + !conf_set) { + master_config.enable_http_metadata_server = + FLAGS_enable_http_metadata_server; + } + if ((google::GetCommandLineFlagInfo("http_metadata_server_port", &info) && + !info.is_default) || + !conf_set) { + master_config.http_metadata_server_port = + FLAGS_http_metadata_server_port; + } + if ((google::GetCommandLineFlagInfo("http_metadata_server_host", &info) && + !info.is_default) || + !conf_set) { + master_config.http_metadata_server_host = + FLAGS_http_metadata_server_host; + } +} + +// Function to start HTTP metadata server +std::unique_ptr StartHttpMetadataServer( + int port, const std::string& host) { + LOG(INFO) << "Starting C++ HTTP metadata server on " << host << ":" << port; + + try { + auto server = + std::make_unique(port, host); + server->start(); + + // Check if server started successfully + if (server->is_running()) { + LOG(INFO) << "C++ HTTP metadata server started successfully"; + return server; + } else { + LOG(ERROR) << "Failed to start C++ HTTP metadata server"; + return nullptr; + } + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to start C++ HTTP metadata server: " << e.what(); + return nullptr; + } } int main(int argc, char* argv[]) { @@ -321,7 +380,29 @@ int main(int argc, char* argv[]) { << master_config.rpc_enable_tcp_no_delay << ", cluster_id=" << master_config.cluster_id << ", root_fs_dir=" << master_config.root_fs_dir - << ", memory_allocator=" << master_config.memory_allocator; + << ", memory_allocator=" << master_config.memory_allocator + << ", enable_http_metadata_server=" + << master_config.enable_http_metadata_server + << ", http_metadata_server_port=" + << master_config.http_metadata_server_port + << ", http_metadata_server_host=" + << master_config.http_metadata_server_host; + + // Start HTTP metadata server if enabled + std::unique_ptr http_metadata_server; + if (master_config.enable_http_metadata_server) { + http_metadata_server = + StartHttpMetadataServer(master_config.http_metadata_server_port, + master_config.http_metadata_server_host); + + if (!http_metadata_server) { + LOG(FATAL) << "Failed to start HTTP metadata server"; + return 1; + } + + // Give the server some time to start + std::this_thread::sleep_for(std::chrono::seconds(1)); + } if (master_config.enable_ha) { mooncake::MasterServiceSupervisor supervisor( @@ -341,4 +422,4 @@ int main(int argc, char* argv[]) { mooncake::RegisterRpcService(server, wrapped_master_service); return server.start(); } -} \ No newline at end of file +} diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 7ce19b74..d314b993 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -56,5 +56,15 @@ fi echo "Running CLI entry point tests..." python test_cli.py +killall mooncake_http_metadata_server +killall mooncake_master +mooncake_master --default_kv_lease_ttl=500 --enable_http_metadata_server=true & +MASTER_PID=$! +sleep 1 +MC_METADATA_SERVER=http://127.0.0.1:8080/metadata DEFAULT_KV_LEASE_TTL=500 python test_distributed_object_store.py +sleep 1 +kill $MASTER_PID || true + + echo "All tests completed successfully!" cd ../..