mooncake-track
/
Mooncake
0
0
Fork 12
Code Issues Pull Requests 13 Actions Packages Projects Releases Wiki Activity

[Store] Hot Standby and Oplog interface for master service HA (#1515)

This commit is contained in:
Bo Lee 2026-03-17 11:26:17 +08:00 committed by GitHub
parent f55a59f83e
commit a811c288ea
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
30 changed files with 8659 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dependencies.sh
View File

@ -127,6 +127,7 @@ SYSTEM_PACKAGES="build-essential \
libmsgpack-dev \
libzstd-dev \
libasio-dev \
libxxhash-dev \
pkg-config \
patchelf \
libc6-dev \

10
mooncake-store/include/etcd_helper.h
View File

@ -52,6 +52,16 @@ class EtcdHelper {
EtcdLeaseId lease_id,
EtcdRevisionId& revision_id);
/*
* @brief Batch create key-value pairs in a single transaction.
* Fails if any key already exists.
* @param keys: The vector of keys.
* @param values: The vector of values.
* @return: Error code.
*/
static ErrorCode BatchCreate(const std::vector<std::string>& keys,
const std::vector<std::string>& values);
/*
* @brief Grant a lease from the etcd.
* @param lease_ttl: The ttl of the lease, in seconds.

248
mooncake-store/include/etcd_oplog_store.h Normal file
View File

@ -0,0 +1,248 @@
#pragma once
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstdint>
#include <optional>
#include <mutex>
#include <string>
#include <thread>
#include <vector>
#include "oplog_manager.h"
#include "types.h"
namespace mooncake {
/**
* @brief Store for OpLog entries in etcd.
*
* This class is responsible for writing OpLog entries to etcd and reading them
* back. OpLog entries are stored with keys in the format:
* /oplog/{cluster_id}/{sequence_id}
*
* The latest sequence_id is also stored at:
* /oplog/{cluster_id}/latest
*/
class EtcdOpLogStore {
public:
/**
* @brief Constructor.
* @param cluster_id: The cluster ID for this OpLog store.
* @param enable_latest_seq_batch_update: Whether to start background thread
* to batch-update `/latest`. Readers (Standby) should set this to
* false to avoid unnecessary thread creation.
* @param enable_batch_write: Whether to start the OpLog batch-write
* background thread. Readers (Standby) that only call Read*
* methods should set this to false to avoid unnecessary thread
* creation and /latest key initialization overhead.
*/
explicit EtcdOpLogStore(const std::string& cluster_id,
bool enable_latest_seq_batch_update = false,
bool enable_batch_write = false);
~EtcdOpLogStore();
/**
* @brief Initialize the store.
* Must be called after construction and before use.
* Performs necessary I/O (e.g. initializing /latest key) and starts
* background threads if enabled.
* @return: Error code.
*/
ErrorCode Init();
/**
* @brief Write an OpLog entry to etcd.
* @param entry: The OpLog entry to write.
* @param sync: If true, wait until the entry is persisted to etcd.
* If false, buffer it and return immediately (Group Commit).
* @return: Error code.
*/
ErrorCode WriteOpLog(const OpLogEntry& entry, bool sync = true);
/**
* @brief Read an OpLog entry from etcd by sequence_id.
* @param sequence_id: The sequence ID of the entry to read.
* @param entry: Output param, the OpLog entry.
* @return: Error code.
*/
ErrorCode ReadOpLog(uint64_t sequence_id, OpLogEntry& entry);
/**
* @brief Read OpLog entries starting from a given sequence_id.
* @param start_sequence_id: The starting sequence ID (exclusive).
* @param limit: Maximum number of entries to read (default: 1000).
* @param entries: Output param, vector of OpLog entries.
* @return: Error code.
*/
ErrorCode ReadOpLogSince(uint64_t start_sequence_id, size_t limit,
std::vector<OpLogEntry>& entries);
// Like ReadOpLogSince, but also returns the etcd revision for consistent
// "read then watch(from revision+1)" startup.
ErrorCode ReadOpLogSinceWithRevision(uint64_t start_sequence_id,
size_t limit,
std::vector<OpLogEntry>& entries,
EtcdRevisionId& revision_id);
/**
* @brief Get the latest sequence_id from etcd.
* @param sequence_id: Output param, the latest sequence_id.
* @return: Error code. ETCD_KEY_NOT_EXIST if no OpLog exists yet.
*/
ErrorCode GetLatestSequenceId(uint64_t& sequence_id);
// Stronger (than `/latest`) best-effort query: return the maximum existing
// sequence_id by scanning etcd keys under /oplog/{cluster_id}/ with
// descending key order.
// Return ETCD_KEY_NOT_EXIST if no OpLog exists yet.
ErrorCode GetMaxSequenceId(uint64_t& sequence_id);
/**
* @brief Update the latest sequence_id in etcd.
* @param sequence_id: The latest sequence_id to update.
* @return: Error code.
*/
ErrorCode UpdateLatestSequenceId(uint64_t sequence_id);
/**
* @brief Record the sequence_id corresponding to a snapshot.
* @param snapshot_id: The snapshot ID.
* @param sequence_id: The sequence_id at which the snapshot was taken.
* @return: Error code.
*/
ErrorCode RecordSnapshotSequenceId(const std::string& snapshot_id,
uint64_t sequence_id);
/**
* @brief Get the sequence_id for a given snapshot.
* @param snapshot_id: The snapshot ID.
* @param sequence_id: Output param, the sequence_id.
* @return: Error code. ETCD_KEY_NOT_EXIST if snapshot not found.
*/
ErrorCode GetSnapshotSequenceId(const std::string& snapshot_id,
uint64_t& sequence_id);
/**
* @brief Clean up OpLog entries before a given sequence_id.
* @param before_sequence_id: All entries with sequence_id <
* before_sequence_id will be deleted.
* @return: Error code.
*/
ErrorCode CleanupOpLogBefore(uint64_t before_sequence_id);
private:
/**
* @brief Build the etcd key for an OpLog entry.
* @param sequence_id: The sequence ID.
* @return: The etcd key.
*/
std::string BuildOpLogKey(uint64_t sequence_id) const;
/**
* @brief Build the etcd key for the latest sequence_id.
* @return: The etcd key.
*/
std::string BuildLatestKey() const;
/**
* @brief Build the etcd key for a snapshot sequence_id.
* @param snapshot_id: The snapshot ID.
* @return: The etcd key.
*/
std::string BuildSnapshotKey(const std::string& snapshot_id) const;
// Best-effort: find the minimum existing OpLog sequence_id in etcd.
// Used for robust cleanup (Scheme 3) so we don't rely on a persisted
// "cleaned_upto" marker.
std::optional<uint64_t> GetMinSequenceId() const;
// Best-effort: find the maximum existing OpLog sequence_id in etcd.
std::optional<uint64_t> GetMaxSequenceIdInternal() const;
/**
* @brief Serialize an OpLogEntry to JSON string.
* @param entry: The OpLog entry to serialize.
* @return: The JSON string.
*/
std::string SerializeOpLogEntry(const OpLogEntry& entry) const;
/**
* @brief Deserialize a JSON string to OpLogEntry.
* @param json_str: The JSON string.
* @param entry: Output param, the OpLog entry.
* @return: true if successful, false otherwise.
*/
bool DeserializeOpLogEntry(const std::string& json_str,
OpLogEntry& entry) const;
/**
* @brief Batch update thread function.
* Periodically updates latest_sequence_id in etcd.
*/
void BatchUpdateThread();
/**
* @brief Trigger immediate batch update if threshold is reached.
*/
void TriggerBatchUpdateIfNeeded();
/**
* @brief Perform the actual batch update to etcd.
*/
void DoBatchUpdate();
std::string cluster_id_;
static constexpr const char* kOpLogPrefix = "/oplog/";
static constexpr const char* kLatestSuffix = "/latest";
static constexpr const char* kSnapshotPrefix = "/oplog/";
static constexpr const char* kSnapshotSuffix = "/snapshot/";
// Batch update mechanism for latest_sequence_id
const bool enable_latest_seq_batch_update_{false};
const bool enable_batch_write_{false};
std::atomic<uint64_t> pending_latest_seq_id_{0};
std::atomic<size_t> pending_count_{0};
std::atomic<bool> batch_update_running_{false};
std::mutex batch_update_mutex_;
std::thread batch_update_thread_;
std::chrono::steady_clock::time_point last_update_time_;
// Batch update configuration
static constexpr size_t kBatchSize = 100; // Update every 100 entries
static constexpr int kBatchIntervalMs = 1000; // Or every 1 second
// Group Commit / Batch Write support
struct BatchEntry {
std::string key;
std::string value;
uint64_t sequence_id;
bool is_sync; // Track if entry requires sync
};
void BatchWriteThread();
void FlushBatch();
mutable std::mutex batch_mutex_;
std::deque<BatchEntry> pending_batch_;
std::condition_variable cv_batch_updated_; // Notify background thread
std::condition_variable cv_sync_completed_; // Notify sync waiters
std::atomic<bool> batch_write_running_{false};
std::thread batch_write_thread_;
std::atomic<uint64_t> last_persisted_seq_id_{0};
// Configs for OpLog batching
static constexpr size_t kOpLogBatchSizeLimit =
1 * 1024 * 1024; // 1MB payload limit (soft)
static constexpr size_t kOpLogBatchCountLimit = 100; // 100 entries
static constexpr int kOpLogBatchTimeoutMs =
10; // 10ms max latency for Async
static constexpr int kSyncWaitTimeoutMs =
3000; // 3s timeout for Sync writes
static constexpr int kFlushRetryCount = 3; // Retries for failed flush
static constexpr int kFlushRetryIntervalMs = 50; // Retry interval
};
} // namespace mooncake

222
mooncake-store/include/ha_metric_manager.h Normal file
View File

@ -0,0 +1,222 @@
#pragma once
#include <atomic>
#include <chrono>
#include <mutex>
#include <string>
#include "ylt/metric/counter.hpp"
#include "ylt/metric/gauge.hpp"
#include "ylt/metric/histogram.hpp"
namespace mooncake {
/**
* @brief Singleton manager for High Availability (HA) related metrics.
*
* This class provides metrics for monitoring the health and performance
* of the OpLog replication system, including:
* - OpLog sequence tracking
* - Standby replication lag
* - Error counters (checksum failures, skipped entries)
* - Performance histograms (etcd write latency)
* - Queue sizes (pending mutations)
*/
class HAMetricManager {
public:
// --- Singleton Access ---
static HAMetricManager& instance();
/**
* @brief Explicitly initialize the singleton instance.
* Use this at startup (e.g. main) to ensure thread-safe initialization
* of the underlying metric library components.
*/
static void Init() { instance(); }
HAMetricManager(const HAMetricManager&) = delete;
HAMetricManager& operator=(const HAMetricManager&) = delete;
HAMetricManager(HAMetricManager&&) = delete;
HAMetricManager& operator=(HAMetricManager&&) = delete;
// ========== OpLog Sequence Metrics (Gauge) ==========
/**
* @brief Set the latest OpLog sequence ID on Primary
*/
void set_oplog_last_sequence_id(int64_t seq_id);
int64_t get_oplog_last_sequence_id();
/**
* @brief Set the Standby's applied sequence ID
*/
void set_oplog_applied_sequence_id(int64_t seq_id);
int64_t get_oplog_applied_sequence_id();
/**
* @brief Set the replication lag (entries behind Primary)
*/
void set_oplog_standby_lag(int64_t lag);
int64_t get_oplog_standby_lag();
/**
* @brief Set the number of pending (out-of-order) entries in OpLogApplier
*/
void set_oplog_pending_entries(int64_t count);
int64_t get_oplog_pending_entries();
/**
* @brief Set the pending mutation queue size (retry queue)
*/
void set_pending_mutation_queue_size(int64_t size);
int64_t get_pending_mutation_queue_size();
// ========== Error Counters ==========
/**
* @brief Increment counter for skipped OpLog entries
*/
void inc_oplog_skipped_entries(int64_t val = 1);
int64_t get_oplog_skipped_entries_total();
/**
* @brief Increment counter for checksum verification failures
*/
void inc_oplog_checksum_failures(int64_t val = 1);
int64_t get_oplog_checksum_failures_total();
/**
* @brief Increment counter for gap resolve attempts
*/
void inc_oplog_gap_resolve_attempts(int64_t val = 1);
int64_t get_oplog_gap_resolve_attempts_total();
/**
* @brief Increment counter for successful gap resolves
*/
void inc_oplog_gap_resolve_success(int64_t val = 1);
int64_t get_oplog_gap_resolve_success_total();
/**
* @brief Increment counter for etcd write failures
*/
void inc_oplog_etcd_write_failures(int64_t val = 1);
int64_t get_oplog_etcd_write_failures_total();
/**
* @brief Increment counter for etcd write retries
*/
void inc_oplog_etcd_write_retries(int64_t val = 1);
int64_t get_oplog_etcd_write_retries_total();
/**
* @brief Increment counter for watch disconnections
*/
void inc_oplog_watch_disconnections(int64_t val = 1);
int64_t get_oplog_watch_disconnections_total();
/**
* @brief Increment counter for successfully applied OpLog entries
*/
void inc_oplog_applied_entries(int64_t val = 1);
int64_t get_oplog_applied_entries_total();
/**
* @brief Increment counter for dropped PUT_END operations (late arrival
* after skip)
*/
void inc_oplog_dropped_put_end(int64_t val = 1);
int64_t get_oplog_dropped_put_end_total();
/**
* @brief Increase the total number of OpLog batch commits (Group Commit)
*/
void inc_oplog_batch_commits(int64_t count = 1);
int64_t get_oplog_batch_commits_total();
/**
* @brief Increase the number of sync batch commits (triggered by
* DELETE/Sync ops)
*/
void inc_oplog_sync_batch_commits(int64_t count = 1);
int64_t get_oplog_sync_batch_commits_total();
// ========== Latency Histograms ==========
/**
* @brief Record etcd write latency in microseconds
*/
void observe_oplog_etcd_write_latency_us(int64_t latency_us);
/**
* @brief Record OpLog apply latency in microseconds
*/
void observe_oplog_apply_latency_us(int64_t latency_us);
// ========== State Machine Metrics ==========
/**
* @brief Set the current Standby state (as integer for Prometheus)
* @param state_value Integer representation of StandbyState
*/
void set_standby_state(int64_t state_value);
int64_t get_standby_state();
/**
* @brief Increment state transition counter
*/
void inc_state_transitions(int64_t val = 1);
int64_t get_state_transitions_total();
// ========== Serialization ==========
/**
* @brief Serializes all HA metrics into Prometheus text format.
* @return A string containing the metrics in Prometheus format.
*/
std::string serialize_metrics();
/**
* @brief Generates a concise, human-readable summary of HA metrics.
* @return A string containing the formatted summary.
*/
std::string get_summary_string();
private:
// --- Private Constructor & Destructor ---
HAMetricManager();
~HAMetricManager() = default;
// --- Metric Members ---
// OpLog Sequence Gauges
ylt::metric::gauge_t oplog_last_sequence_id_;
ylt::metric::gauge_t oplog_applied_sequence_id_;
ylt::metric::gauge_t oplog_standby_lag_;
ylt::metric::gauge_t oplog_pending_entries_;
ylt::metric::gauge_t pending_mutation_queue_size_;
// Error Counters
ylt::metric::counter_t oplog_skipped_entries_total_;
ylt::metric::counter_t oplog_checksum_failures_total_;
ylt::metric::counter_t oplog_gap_resolve_attempts_total_;
ylt::metric::counter_t oplog_gap_resolve_success_total_;
ylt::metric::counter_t oplog_etcd_write_failures_total_;
ylt::metric::counter_t oplog_etcd_write_retries_total_;
ylt::metric::counter_t oplog_watch_disconnections_total_;
ylt::metric::counter_t oplog_applied_entries_total_;
ylt::metric::counter_t oplog_dropped_put_end_total_;
ylt::metric::counter_t oplog_batch_commits_total_;
ylt::metric::counter_t oplog_sync_batch_commits_total_;
// Latency Histograms (buckets in microseconds: 100us, 500us, 1ms, 5ms,
// 10ms, 50ms, 100ms, 500ms, 1s)
ylt::metric::histogram_t oplog_etcd_write_latency_us_;
ylt::metric::histogram_t oplog_apply_latency_us_;
// State Machine
ylt::metric::gauge_t standby_state_;
ylt::metric::counter_t state_transitions_total_;
};
} // namespace mooncake

249
mooncake-store/include/hot_standby_service.h Normal file
View File

@ -0,0 +1,249 @@
#pragma once
#include <atomic>
#include <chrono>
#include <cstdint>
#include <memory>
#include <mutex>
#include <string>
#include <thread>
#include <unordered_map>
#include <utility>
#include <vector>
#include "metadata_store.h"
#include "oplog_applier.h"
#include "oplog_manager.h"
#include "oplog_watcher.h"
#include "snapshot_provider.h"
#include "standby_state_machine.h"
#include "types.h"
namespace mooncake {
// Forward declarations
class MasterService;
class ReplicationStream;
/**
* @brief Configuration for HotStandbyService
*/
struct HotStandbyConfig {
std::string standby_id;
std::string primary_address;
uint32_t replication_port{0};
uint32_t verification_interval_sec{30};
uint32_t max_replication_lag_entries{1000};
bool enable_verification{true};
// Snapshot bootstrap (optional):
// If provided, Standby will try to load a snapshot first, then replay OpLog
// from snapshot_sequence_id.
bool enable_snapshot_bootstrap{false};
};
/**
* @brief Sync status information for HotStandbyService
*/
struct StandbySyncStatus {
uint64_t applied_seq_id{0};
uint64_t primary_seq_id{0};
uint64_t lag_entries{0};
std::chrono::milliseconds lag_time{0};
bool is_syncing{false};
bool is_connected{false};
StandbyState state{StandbyState::STOPPED};
std::chrono::milliseconds time_in_state{0};
};
/**
* @brief HotStandbyService manages standby replication and promotion
*
* This service runs on Standby Master nodes and is responsible for:
* - Connecting to Primary and receiving OpLog entries
* - Applying OpLog entries to local metadata store
* - Periodically verifying data consistency with Primary
* - Promoting to Primary when elected as new Leader
*
* For now, this is a skeleton implementation without actual network
* communication. The gRPC integration will be added later.
*/
class HotStandbyService {
public:
explicit HotStandbyService(const HotStandbyConfig& config);
~HotStandbyService();
/**
* @brief Start connecting to Primary and begin replication
* @param primary_address Address of the Primary Master (not used with
* etcd-based sync)
* @param etcd_endpoints Comma-separated etcd endpoints
* @param cluster_id Cluster identifier for OpLog path
* @return ErrorCode::OK on success
*/
ErrorCode Start(const std::string& primary_address,
const std::string& etcd_endpoints,
const std::string& cluster_id);
/**
* @brief Stop replication and disconnect from Primary
*/
void Stop();
/**
* @brief Get current synchronization status
* @return StandbySyncStatus with current sync state
*/
StandbySyncStatus GetSyncStatus() const;
/**
* @brief Check if standby is ready for promotion
* @return true if replication lag is within threshold
*/
bool IsReadyForPromotion() const;
/**
* @brief Promote this standby to Primary
*
* This method should be called after successful leader election.
* It ensures all Op Logs are applied and transitions the state machine.
* The caller is responsible for creating the MasterService separately.
*
* @return ErrorCode::OK on success, other codes on failure
*/
ErrorCode Promote();
/**
* @brief Get the number of metadata entries in the local store
*/
size_t GetMetadataCount() const;
/**
* @brief Get the latest applied sequence ID after promotion
*
* This should be called after Promote() to get the sequence_id
* that the new Primary's OpLogManager should start from.
*
* @return Latest applied sequence ID, or 0 if not available
*/
uint64_t GetLatestAppliedSequenceId() const;
// Export a point-in-time snapshot of all replicated metadata.
// This is used by MasterServiceSupervisor to initialize the new Primary
// after leader election (fast recovery).
bool ExportMetadataSnapshot(
std::vector<std::pair<std::string, StandbyObjectMetadata>>& out) const;
// Inject a snapshot provider (from external snapshot implementation).
void SetSnapshotProvider(std::unique_ptr<SnapshotProvider> provider);
/**
* @brief Get current state from state machine
*/
StandbyState GetState() const { return state_machine_.GetState(); }
/**
* @brief Get state machine for monitoring/debugging
*/
const StandbyStateMachine& GetStateMachine() const {
return state_machine_;
}
/**
* @brief Callback for OpLogWatcher state changes
* @param event The event to process
*/
void OnWatcherEvent(StandbyEvent event);
private:
/**
* @brief Main replication loop (runs in background thread)
*/
void ReplicationLoop();
/**
* @brief Verification loop (runs in background thread)
*/
void VerificationLoop();
/**
* @brief Apply a single OpLog entry to local metadata store
* @param entry The OpLog entry to apply
* @deprecated Use OpLogApplier instead
*/
void ApplyOpLogEntry(const OpLogEntry& entry);
/**
* @brief Connect to Primary and establish replication stream
* @return true on success, false on failure
*/
bool ConnectToPrimary();
/**
* @brief Disconnect from Primary
*/
void DisconnectFromPrimary();
/**
* @brief Process a batch of OpLog entries received from Primary
* @param entries Batch of OpLog entries
*/
void ProcessOpLogBatch(const std::vector<OpLogEntry>& entries);
HotStandbyConfig config_;
// Simple in-memory metadata store implementation
class StandbyMetadataStore : public MetadataStore {
public:
bool PutMetadata(const std::string& key,
const StandbyObjectMetadata& metadata) override;
bool Put(const std::string& key,
const std::string& payload = std::string()) override;
std::optional<StandbyObjectMetadata> GetMetadata(
const std::string& key) const override;
bool Remove(const std::string& key) override;
bool Exists(const std::string& key) const override;
size_t GetKeyCount() const override;
// Snapshot for promotion/restore.
void Snapshot(
std::vector<std::pair<std::string, StandbyObjectMetadata>>& out)
const;
private:
mutable std::mutex mutex_;
std::unordered_map<std::string, StandbyObjectMetadata> store_;
};
std::unique_ptr<StandbyMetadataStore> metadata_store_;
std::unique_ptr<SnapshotProvider> snapshot_provider_{
std::make_unique<NoopSnapshotProvider>()};
// OpLog replication components
std::unique_ptr<OpLogApplier> oplog_applier_;
std::unique_ptr<OpLogWatcher> oplog_watcher_;
// Configuration for etcd-based OpLog sync
std::string etcd_endpoints_;
std::string cluster_id_;
// Replication state
std::shared_ptr<ReplicationStream> replication_stream_;
std::atomic<uint64_t> applied_seq_id_{0};
std::atomic<uint64_t> primary_seq_id_{0};
// State machine for managing service lifecycle
StandbyStateMachine state_machine_;
// Helper methods for state machine
bool IsRunning() const { return state_machine_.IsRunning(); }
bool IsConnected() const { return state_machine_.IsConnected(); }
// Background threads
std::thread replication_thread_;
std::thread verification_thread_;
// Synchronization
mutable std::mutex mutex_;
};
} // namespace mooncake

120
mooncake-store/include/metadata_store.h Normal file
View File

@ -0,0 +1,120 @@
#pragma once
#include <cstddef>
#include <optional>
#include <string>
#include <vector>
#include "replica.h"
#include "types.h"
namespace mooncake {
/**
* @brief Metadata structure for Standby to store and restore object information
*
* This structure contains all essential metadata information needed by Standby
* to immediately serve as Primary when promoted.
*/
struct StandbyObjectMetadata {
UUID client_id{0, 0};
uint64_t size{0};
std::vector<Replica::Descriptor> replicas;
// NOTE: Lease information is NOT stored because:
// 1. Standby does not perform eviction, so lease info is not used
// 2. After promotion, new Primary should grant fresh leases, not restore
// old ones
uint64_t last_sequence_id{
0}; // Last OpLog sequence ID that modified this key
StandbyObjectMetadata() = default;
// Check if this metadata has valid replicas
bool HasReplicas() const { return !replicas.empty(); }
};
/**
* @brief Payload structure for struct_pack serialization (msgpack binary
* format)
*
* Now uses UUID directly since struct_pack natively supports std::pair.
*/
struct MetadataPayload {
UUID client_id{0, 0};
uint64_t size{0};
std::vector<Replica::Descriptor> replicas;
// NOTE: Lease information removed - not needed by Standby
YLT_REFL(MetadataPayload, client_id, size, replicas);
// Convert to StandbyObjectMetadata
StandbyObjectMetadata ToStandbyMetadata(uint64_t sequence_id) const {
StandbyObjectMetadata meta;
meta.client_id = client_id;
meta.size = size;
meta.replicas = replicas;
meta.last_sequence_id = sequence_id;
return meta;
}
};
/**
* @brief Abstract interface for metadata storage on Standby
*
* This interface provides basic operations for storing and managing object
* metadata. In a full implementation, this would mirror MasterService's
* metadata_shards_ structure.
*/
class MetadataStore {
public:
virtual ~MetadataStore() = default;
/**
* @brief Put or update metadata for a key with structured metadata
* @param key Object key
* @param metadata Structured metadata object
* @return true on success, false on failure
*/
virtual bool PutMetadata(const std::string& key,
const StandbyObjectMetadata& metadata) = 0;
/**
* @brief Put or update metadata for a key (legacy interface for backward
* compatibility)
* @param key Object key
* @param payload Optional payload data (JSON serialized metadata)
* @return true on success, false on failure
*/
virtual bool Put(const std::string& key,
const std::string& payload = std::string()) = 0;
/**
* @brief Get metadata for a key
* @param key Object key
* @return Copy of metadata if found, std::nullopt otherwise
*/
virtual std::optional<StandbyObjectMetadata> GetMetadata(
const std::string& key) const = 0;
/**
* @brief Remove metadata for a key
* @param key Object key
* @return true if key was found and removed, false otherwise
*/
virtual bool Remove(const std::string& key) = 0;
/**
* @brief Check if a key exists
* @param key Object key
* @return true if key exists, false otherwise
*/
virtual bool Exists(const std::string& key) const = 0;
/**
* @brief Get the count of keys in the store
* @return Number of keys
*/
virtual size_t GetKeyCount() const = 0;
};
} // namespace mooncake

176
mooncake-store/include/oplog_applier.h Normal file
View File

@ -0,0 +1,176 @@
#pragma once
#include <chrono>
#include <atomic>
#include <cstdint>
#include <map>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
#include "oplog_manager.h"
#include "metadata_store.h"
namespace mooncake {
// Forward declaration
class EtcdOpLogStore;
/**
* @brief Apply OpLog entries to Standby metadata store with ordering guarantee
*
* This class applies OpLog entries to the Standby metadata store,
* ensuring both global and per-key ordering.
*/
class OpLogApplier {
public:
/**
* @brief Constructor
* @param metadata_store Metadata store to apply changes to
* @param cluster_id Cluster ID for accessing etcd OpLog (optional, for
* requesting missing OpLog)
*/
explicit OpLogApplier(MetadataStore* metadata_store,
const std::string& cluster_id = std::string());
/**
* @brief Apply a single OpLog entry (with ordering checks)
* @param entry OpLog entry to apply
* @return true on success, false on failure or ordering violation
*/
bool ApplyOpLogEntry(const OpLogEntry& entry);
/**
* @brief Apply multiple OpLog entries
* @param entries OpLog entries to apply
* @return Number of successfully applied entries
*/
size_t ApplyOpLogEntries(const std::vector<OpLogEntry>& entries);
/**
* @brief Get the current sequence ID for a key (DEPRECATED)
* @param key Object key
* @return Always returns 0 - global sequence_id is used for ordering
* @deprecated Use global sequence_id for ordering
*/
uint64_t GetKeySequenceId(const std::string& key) const;
/**
* @brief Get the expected global sequence ID
* @return Expected global sequence ID
*/
uint64_t GetExpectedSequenceId() const;
/**
* @brief Recover from a given sequence ID
* @param last_applied_sequence_id Last applied sequence ID
*/
void Recover(uint64_t last_applied_sequence_id);
/**
* @brief Process pending entries (entries with non-continuous sequence IDs)
* @return Number of entries processed
*/
size_t ProcessPendingEntries();
// Promotion helper:
// Try to resolve current gaps ONCE (no waiting) by fetching missing/skipped
// sequence_ids from etcd. If an entry arrives late:
// - REMOVE / PUT_REVOKE: delete the key
// - PUT_END: discard
//
// This is used during Standby promotion so we don't block promotion on
// gaps, but still best-effort clean up potentially stale metadata.
struct GapResolveResult {
size_t attempted{0};
size_t fetched{0};
size_t applied_deletes{0};
};
GapResolveResult TryResolveGapsOnceForPromotion(size_t max_ids = 1024);
private:
/**
* @brief Check if the entry's sequence order is valid
* @param entry OpLog entry
* @return true if order is valid, false otherwise
*/
bool CheckSequenceOrder(const OpLogEntry& entry);
/**
* @brief Apply PUT_END operation
* @param entry OpLog entry
*/
void ApplyPutEnd(const OpLogEntry& entry);
/**
* @brief Apply PUT_REVOKE operation
* @param entry OpLog entry
*/
void ApplyPutRevoke(const OpLogEntry& entry);
/**
* @brief Apply REMOVE operation
* @param entry OpLog entry
*/
void ApplyRemove(const OpLogEntry& entry);
/**
* @brief Request missing OpLog entry from etcd
* @param missing_seq_id Missing sequence ID
* @return true if entry was found and applied, false otherwise
*/
bool RequestMissingOpLog(uint64_t missing_seq_id);
/**
* @brief Schedule wait for missing entries
* @param missing_seq_id Missing sequence ID
*/
void ScheduleWaitForMissingEntries(uint64_t missing_seq_id);
MetadataStore* metadata_store_;
// EtcdOpLogStore for requesting missing OpLog entries (optional)
std::string cluster_id_;
mutable std::mutex etcd_oplog_store_mutex_;
mutable std::unique_ptr<EtcdOpLogStore> etcd_oplog_store_;
/**
* @brief Get or create EtcdOpLogStore instance (lazy initialization)
* @return Pointer to EtcdOpLogStore, or nullptr if cluster_id is not set
*/
EtcdOpLogStore* GetEtcdOpLogStore() const;
// Note: key_sequence_map_ has been removed.
// Global sequence_id is sufficient for ordering guarantee.
// Track pending entries (entries with non-continuous sequence IDs)
mutable std::mutex pending_mutex_;
std::map<uint64_t, OpLogEntry> pending_entries_;
// Track missing sequence IDs that we're waiting for
std::map<uint64_t, std::chrono::steady_clock::time_point>
missing_sequence_ids_;
// Sequence IDs we chose to skip (gap-timeout). If the late entry arrives:
// - REMOVE / PUT_REVOKE: delete the key (safe)
// - PUT_END: discard (do not resurrect potentially stale metadata)
std::map<uint64_t, std::chrono::steady_clock::time_point>
skipped_sequence_ids_;
// Next expected global sequence_id. Read frequently from monitoring thread,
// updated by watch/apply thread. Use atomic to avoid data races.
std::atomic<uint64_t> expected_sequence_id_{1};
// Constants for missing entry handling
// IMPORTANT: request must happen BEFORE skip, otherwise we will never
// request.
static constexpr int kMissingEntryRequestSeconds =
1; // request from etcd after 1s
static constexpr int kMissingEntrySkipSeconds =
3; // skip after 3s (avoid global stall)
static constexpr int kMaxPendingEntries =
1000; // Max pending entries before giving up
};
} // namespace mooncake

150
mooncake-store/include/oplog_manager.h Normal file
View File

@ -0,0 +1,150 @@
#pragma once
#include <cstdint>
#include <deque>
#include <memory>
#include <mutex>
#include <shared_mutex>
#include <string>
#include <vector>
#include <ylt/util/tl/expected.hpp>
#include "types.h"
namespace mooncake {
// Forward declaration
class EtcdOpLogStore;
// Operation types for hot-standby replication.
// This is a minimal subset that can be extended later.
enum class OpType : uint8_t {
PUT_END = 1,
PUT_REVOKE = 2,
REMOVE = 3,
// Deprecated: LEASE_RENEW is intentionally not recorded in OpLog in the
// current etcd-based hot-standby design (Standby relies on Primary DELETE
// operations).
LEASE_RENEW = 4,
};
// A single operation log entry.
// Note: Payload contains JSON serialized MetadataPayload (defined in
// metadata_store.h) for PUT_END operations, allowing Standby to restore
// complete metadata.
struct OpLogEntry {
uint64_t sequence_id{0}; // Monotonically increasing global sequence
uint64_t timestamp_ms{0}; // Logical timestamp in milliseconds
OpType op_type{OpType::PUT_END};
std::string object_key; // Target object key
std::string payload; // Serialized extra data (optional)
uint32_t checksum{0}; // Checksum of payload (implementation-defined)
uint32_t prefix_hash{
0}; // Hash of the entire key (for verification and optimization)
};
/**
* @brief In-memory operation log manager.
*
* This class is intentionally simple: it keeps a bounded deque of OpLogEntry
* and provides append / get-since primitives. It can later be extended to
* or to spill to disk if needed. In the new etcd-based design, OpLog will be
* written to etcd.
*/
class OpLogManager {
public:
OpLogManager();
// Set the EtcdOpLogStore for writing OpLog to etcd (optional).
// If not set, OpLog will only be stored in memory buffer.
void SetEtcdOpLogStore(std::shared_ptr<EtcdOpLogStore> etcd_oplog_store);
// Append a new entry and return the assigned sequence_id.
// This is a best-effort (async) path: the entry is buffered in memory
// and enqueued to etcd without waiting for persistence.
// Only suitable for idempotent, lag-tolerant operations (PUT_END).
// For operations that MUST be durable before returning (REMOVE, etc.),
// use AppendAndPersist() instead.
uint64_t Append(OpType type, const std::string& key,
const std::string& payload = std::string());
// Allocate a new OpLogEntry with a reserved sequence_id, append it to the
// in-memory buffer, and return the full entry.
//
// IMPORTANT: This will advance last_seq_id_ even if the caller later fails
// to persist it to etcd. This supports "seq pre-allocation" semantics where
// retries use the same (smaller) sequence_id.
OpLogEntry AllocateEntry(OpType type, const std::string& key,
const std::string& payload = std::string());
// Persist an already-allocated entry to etcd using its sequence_id.
// Does NOT modify sequence counters.
ErrorCode PersistEntryToEtcd(const OpLogEntry& entry) const;
// Append a new entry and durably persist it to etcd (if EtcdOpLogStore is
// set).
//
// This is intended for operations that may free/reuse memory (e.g. REMOVE),
// where best-effort replication is unsafe: Standby must observe the DELETE
// before promotion, otherwise it may return stale descriptors that point to
// reused memory and cause silent data corruption.
//
// Design (updated for seq pre-allocation):
// - sequence_id is allocated first and never reused.
// - If etcd write fails, caller may retry PersistEntryToEtcd with the same
// entry (sequence_id fixed and "smaller" than later entries).
tl::expected<uint64_t, ErrorCode> AppendAndPersist(
OpType type, const std::string& key,
const std::string& payload = std::string());
// Get the latest assigned sequence id. Returns 0 if no entry exists.
uint64_t GetLastSequenceId() const;
// Set the initial sequence ID (used when promoting Standby to Primary).
// This ensures the new Primary's OpLogManager continues from the correct
// sequence_id.
void SetInitialSequenceId(uint64_t sequence_id);
// Current number of entries in the buffer.
size_t GetEntryCount() const;
// Verify checksum of an OpLogEntry payload.
// Returns true if checksum matches, false otherwise.
// This is public so OpLogWatcher and OpLogApplier can validate entries.
static bool VerifyChecksum(const OpLogEntry& entry);
// Basic DoS protection for externally sourced OpLog entries (etcd watch /
// reads). Enforce conservative bounds on key/payload sizes before
// parsing/applying.
static constexpr size_t kMaxObjectKeySize = 4096; // 4 KiB
static constexpr size_t kMaxPayloadSize = 10 * 1024 * 1024; // 10 MiB
// Validate OpLogEntry key/payload sizes. If invalid, returns false and
// optionally sets a human-readable reason.
static bool ValidateEntrySize(const OpLogEntry& entry,
std::string* reason = nullptr);
private:
static uint64_t NowMs();
static uint32_t ComputeChecksum(const std::string& data);
static uint32_t ComputePrefixHash(const std::string& key);
mutable std::shared_mutex mutex_;
std::deque<OpLogEntry> buffer_;
uint64_t first_seq_id_{1}; // sequence_id of buffer_.front()
uint64_t last_seq_id_{0}; // last assigned sequence_id
// Note: We removed key_sequence_map_ and key_remove_time_map_.
// Global sequence_id is sufficient for ordering guarantee.
// All operations are applied in sequence_id order, which ensures
// consistency.
// Optional etcd OpLog store for persistent storage
std::shared_ptr<EtcdOpLogStore> etcd_oplog_store_;
// Simple bounds to avoid unbounded memory growth.
static constexpr size_t kMaxBufferEntries_ = 100000;
};
} // namespace mooncake

189
mooncake-store/include/oplog_watcher.h Normal file
View File

@ -0,0 +1,189 @@
#pragma once
#include <atomic>
#include <cstdint>
#include <functional>
#include <memory>
#include <string>
#include <thread>
#include <vector>
#include "oplog_manager.h"
#include "etcd_oplog_store.h"
#include "standby_state_machine.h"
#include "types.h"
namespace mooncake {
// Forward declarations
class OpLogApplier;
class OpLogWatcher;
// Callback type for state events
using WatcherStateCallback = std::function<void(StandbyEvent)>;
/**
* @brief Shared control block for safe C-style watch callbacks.
*
* Because the etcd Watch goroutine can deliver callbacks after
* OpLogWatcher::Stop() returns (or even after ~OpLogWatcher), we
* cannot pass a raw `this` pointer as the callback context.
*
* Instead we heap‐allocate a WatchCallbackContext whose lifetime is
* decoupled from the watcher. The callback locks the mutex and checks
* `watcher != nullptr` before touching any watcher state.
*
* Stop() sets `watcher = nullptr` under the same mutex, then cancels
* the Go goroutine and waits. If the wait times out the context is
* intentionally leaked (a few bytes) rather than risking UAF.
*/
struct WatchCallbackContext {
std::mutex mutex;
OpLogWatcher* watcher{nullptr};
WatchCallbackContext() = default;
WatchCallbackContext(const WatchCallbackContext&) = delete;
WatchCallbackContext& operator=(const WatchCallbackContext&) = delete;
};
/**
* @brief Watch etcd for OpLog changes and apply them to Standby
*
* This class watches etcd for new OpLog entries and forwards them
* to OpLogApplier for processing.
*/
class OpLogWatcher {
public:
/**
* @brief Constructor
* @param etcd_endpoints Comma-separated etcd endpoints
* @param cluster_id Cluster identifier
* @param applier OpLog applier to process entries
*/
OpLogWatcher(const std::string& etcd_endpoints,
const std::string& cluster_id, OpLogApplier* applier);
~OpLogWatcher();
/**
* @brief Start watching etcd for OpLog changes
*/
void Start();
/**
* @brief Start from a known last-applied sequence_id.
*
* It will read historical OpLogs at a consistent etcd revision, then start
* watch from revision+1 to close the gap between "read" and "watch".
*/
bool StartFromSequenceId(uint64_t start_seq_id);
/**
* @brief Stop watching
*/
void Stop();
/**
* @brief Get the last processed sequence ID
* @return Last processed sequence ID
*/
uint64_t GetLastProcessedSequenceId() const;
/**
* @brief Set callback for state events
* @param callback Callback function to invoke on state events
*/
void SetStateCallback(WatcherStateCallback callback) {
state_callback_ = std::move(callback);
}
/**
* @brief Check if watch is healthy
*/
bool IsWatchHealthy() const { return watch_healthy_.load(); }
private:
/**
* @brief Notify state callback
*/
void NotifyStateEvent(StandbyEvent event) {
if (state_callback_) {
state_callback_(event);
}
}
bool ReadOpLogSince(uint64_t start_seq_id, std::vector<OpLogEntry>& entries,
EtcdRevisionId& revision_id);
// Callback includes etcd KV mod_revision for precise resume.
static void WatchCallback(void* context, const char* key, size_t key_size,
const char* value, size_t value_size,
int event_type, int64_t mod_revision);
/**
* @brief Watch etcd OpLog changes (runs in background thread)
*/
void WatchOpLog();
/**
* @brief Process a Watch event
* @param key etcd key
* @param value etcd value (JSON string for PUT events, empty for DELETE
* events)
* @param event_type Event type (0 = PUT, 1 = DELETE)
*/
void HandleWatchEvent(const std::string& key, const std::string& value,
int event_type);
void HandleWatchEvent(const std::string& key, const std::string& value,
int event_type, int64_t mod_revision);
/**
* @brief Deserialize OpLogEntry from JSON string
* @param json_str JSON string
* @param entry Output OpLog entry
* @return true on success, false on failure
*/
bool DeserializeOpLogEntry(const std::string& json_str, OpLogEntry& entry);
/**
* @brief Attempt to reconnect after watch failure
*/
void TryReconnect();
/**
* @brief Sync missed OpLog entries after reconnection
* @return true if sync was successful
*/
bool SyncMissedEntries();
// Next watch revision (0 means from now). Updated by consistent reads.
std::atomic<int64_t> next_watch_revision_{0};
std::string etcd_endpoints_;
std::string cluster_id_;
OpLogApplier* applier_;
std::unique_ptr<EtcdOpLogStore> op_log_store_;
std::atomic<bool> running_{false};
std::thread watch_thread_;
std::atomic<uint64_t> last_processed_sequence_id_{0};
// Shared control block for the C-style watch callback. Allocated on the
// heap; ownership is transferred to the Go goroutine if Stop() cannot
// confirm the goroutine has exited (intentional leak to prevent UAF).
WatchCallbackContext* watch_callback_ctx_{nullptr};
// Error handling and recovery
std::atomic<int> consecutive_errors_{0};
std::atomic<int> reconnect_count_{0};
std::atomic<bool> watch_healthy_{false};
// State callback for notifying HotStandbyService
WatcherStateCallback state_callback_;
// Constants for error handling
static constexpr int kMaxConsecutiveErrors = 10;
static constexpr int kReconnectDelayMs = 1000;
static constexpr int kMaxReconnectDelayMs = 30000;
static constexpr int kSyncBatchSize = 1000;
};
} // namespace mooncake

56
mooncake-store/include/snapshot_provider.h Normal file
View File

@ -0,0 +1,56 @@
#pragma once
#include <cstdint>
#include <string>
#include <utility>
#include <vector>
#include "metadata_store.h"
namespace mooncake {
/**
* @brief SnapshotProvider is an abstraction for loading metadata snapshots.
*
* Assumption: snapshot functionality exists (implemented by another team), but
* may not be synced into this repo yet. We keep Mooncake-store code progressing
* by depending on this narrow interface.
*
* Snapshot semantics for hot-standby:
* - A snapshot represents a consistent metadata baseline at
* `snapshot_sequence_id`.
* - Standby should: load snapshot -> recover applier to snapshot_sequence_id ->
* replay OpLog entries with sequence_id > snapshot_sequence_id.
*/
class SnapshotProvider {
public:
virtual ~SnapshotProvider() = default;
// Load the latest available snapshot for `cluster_id`.
// Returns true on success and fills:
// - snapshot_id: opaque identifier (e.g. timestamp/version)
// - snapshot_sequence_id: global OpLog sequence_id at snapshot boundary
// - snapshot: full metadata baseline as key -> StandbyObjectMetadata
virtual bool LoadLatestSnapshot(
const std::string& cluster_id, std::string& snapshot_id,
uint64_t& snapshot_sequence_id,
std::vector<std::pair<std::string, StandbyObjectMetadata>>&
snapshot) = 0;
};
// Default no-op provider: behaves as if "no snapshot available".
class NoopSnapshotProvider final : public SnapshotProvider {
public:
bool LoadLatestSnapshot(
const std::string& /*cluster_id*/, std::string& snapshot_id,
uint64_t& snapshot_sequence_id,
std::vector<std::pair<std::string, StandbyObjectMetadata>>& snapshot)
override {
snapshot_id.clear();
snapshot_sequence_id = 0;
snapshot.clear();
return false;
}
};
} // namespace mooncake

343
mooncake-store/include/standby_state_machine.h Normal file
View File

@ -0,0 +1,343 @@
#pragma once
#include <atomic>
#include <chrono>
#include <functional>
#include <mutex>
#include <string>
#include <vector>
namespace mooncake {
/**
* @brief Standby service states
*
* State transition diagram:
*
* ┌─────────┐
* │ STOPPED │◄───────────────────────────────────────┐
* └────┬────┘ │
* │ Start() │ Stop()/Error
* ▼ │
* ┌─────────────┐ │
* │ CONNECTING │ │
* └──────┬──────┘ │
* │ Connected │
* ▼ Connected │
* ┌─────────────┐ Error/Disconnect ┌────────────┐ │
* │ SYNCING │────────────────► │RECONNECTING│────┤
* └──────┬──────┘◄──────────────── └──────┬─────┘ │
* │ Sync complete ▲ │ │
* ▼ │ │Watch │
* ┌─────────────┐ Watch broken/ ────┘ │healthy │
* │ WATCHING │── Disconnect │ │
* └──────┬──────┘◄────────────────────────┘ │
* │ Max errors ┌────────────┐ │
* │────────────────────────►│ RECOVERING │────┤
* │ └────────────┘ │
* │ Promote() │
* ▼ │
* ┌─────────────┐ │
* │ PROMOTING │────────────────────────────────────┤
* └──────┬──────┘ │
* │ Success │
* ▼ │
* ┌─────────────┐ │
* │ PROMOTED │────────────────────────────────────┘
* └─────────────┘
*/
enum class StandbyState : uint8_t {
// Initial state, service not started
STOPPED = 0,
// Connecting to etcd cluster
CONNECTING = 1,
// Initial sync: reading historical OpLog entries
SYNCING = 2,
// Normal operation: watching for new OpLog entries
WATCHING = 3,
// Recovering from error: re-syncing missed entries
RECOVERING = 4,
// Reconnecting after watch failure
RECONNECTING = 5,
// Promotion in progress: final catch-up before becoming Primary
PROMOTING = 6,
// Successfully promoted to Primary
PROMOTED = 7,
// Fatal error, cannot recover
FAILED = 8,
};
/**
* @brief Get human-readable state name
*/
inline const char* StandbyStateToString(StandbyState state) {
switch (state) {
case StandbyState::STOPPED:
return "STOPPED";
case StandbyState::CONNECTING:
return "CONNECTING";
case StandbyState::SYNCING:
return "SYNCING";
case StandbyState::WATCHING:
return "WATCHING";
case StandbyState::RECOVERING:
return "RECOVERING";
case StandbyState::RECONNECTING:
return "RECONNECTING";
case StandbyState::PROMOTING:
return "PROMOTING";
case StandbyState::PROMOTED:
return "PROMOTED";
case StandbyState::FAILED:
return "FAILED";
default:
return "UNKNOWN";
}
}
/**
* @brief Events that trigger state transitions
*/
enum class StandbyEvent : uint8_t {
// User/system actions
START, // Start() called
STOP, // Stop() called
PROMOTE, // Promote() called
// Connection events
CONNECTED, // Successfully connected to etcd
CONNECTION_FAILED, // Failed to connect to etcd
DISCONNECTED, // Connection lost
// Sync events
SYNC_COMPLETE, // Initial sync completed
SYNC_FAILED, // Sync failed
// Watch events
WATCH_HEALTHY, // Watch is healthy and receiving events
WATCH_BROKEN, // Watch connection broken
// Recovery events
RECOVERY_SUCCESS, // Successfully recovered from error
RECOVERY_FAILED, // Recovery failed
// Promotion events
PROMOTION_SUCCESS, // Successfully promoted
PROMOTION_FAILED, // Promotion failed
// Error events
MAX_ERRORS_REACHED, // Too many consecutive errors
FATAL_ERROR, // Unrecoverable error
};
inline const char* StandbyEventToString(StandbyEvent event) {
switch (event) {
case StandbyEvent::START:
return "START";
case StandbyEvent::STOP:
return "STOP";
case StandbyEvent::PROMOTE:
return "PROMOTE";
case StandbyEvent::CONNECTED:
return "CONNECTED";
case StandbyEvent::CONNECTION_FAILED:
return "CONNECTION_FAILED";
case StandbyEvent::DISCONNECTED:
return "DISCONNECTED";
case StandbyEvent::SYNC_COMPLETE:
return "SYNC_COMPLETE";
case StandbyEvent::SYNC_FAILED:
return "SYNC_FAILED";
case StandbyEvent::WATCH_HEALTHY:
return "WATCH_HEALTHY";
case StandbyEvent::WATCH_BROKEN:
return "WATCH_BROKEN";
case StandbyEvent::RECOVERY_SUCCESS:
return "RECOVERY_SUCCESS";
case StandbyEvent::RECOVERY_FAILED:
return "RECOVERY_FAILED";
case StandbyEvent::PROMOTION_SUCCESS:
return "PROMOTION_SUCCESS";
case StandbyEvent::PROMOTION_FAILED:
return "PROMOTION_FAILED";
case StandbyEvent::MAX_ERRORS_REACHED:
return "MAX_ERRORS_REACHED";
case StandbyEvent::FATAL_ERROR:
return "FATAL_ERROR";
default:
return "UNKNOWN";
}
}
/**
* @brief State transition result
*/
struct StateTransitionResult {
bool allowed{false};
StandbyState old_state{StandbyState::STOPPED};
StandbyState new_state{StandbyState::STOPPED};
std::string reason;
};
/**
* @brief Callback for state transition notifications
*/
using StateChangeCallback = std::function<void(
StandbyState old_state, StandbyState new_state, StandbyEvent event)>;
/**
* @brief Standby State Machine
*
* Thread-safe state machine for managing Standby service lifecycle.
* All state transitions are explicit and logged.
*/
class StandbyStateMachine {
public:
StandbyStateMachine();
/**
* @brief Get current state (thread-safe)
*/
StandbyState GetState() const {
return current_state_.load(std::memory_order_acquire);
}
/**
* @brief Check if in a specific state
*/
bool IsInState(StandbyState state) const { return GetState() == state; }
/**
* @brief Check if service is running (SYNCING, WATCHING, RECOVERING,
* RECONNECTING, PROMOTING)
*/
bool IsRunning() const {
StandbyState s = GetState();
return s == StandbyState::SYNCING || s == StandbyState::WATCHING ||
s == StandbyState::RECOVERING ||
s == StandbyState::RECONNECTING || s == StandbyState::PROMOTING;
}
/**
* @brief Check if connected to etcd
*/
bool IsConnected() const {
StandbyState s = GetState();
return s == StandbyState::SYNCING || s == StandbyState::WATCHING ||
s == StandbyState::RECOVERING || s == StandbyState::PROMOTING;
}
/**
* @brief Check if watch is healthy
*/
bool IsWatchHealthy() const { return GetState() == StandbyState::WATCHING; }
/**
* @brief Check if ready for promotion
*/
bool IsReadyForPromotion() const {
return GetState() == StandbyState::WATCHING;
}
/**
* @brief Process an event and perform state transition
* @param event The event to process
* @return Result indicating if transition was allowed and new state
*/
StateTransitionResult ProcessEvent(StandbyEvent event);
/**
* @brief Register a callback for state change notifications
*/
void RegisterCallback(StateChangeCallback callback);
/**
* @brief State transition record for debugging
*/
struct TransitionRecord {
std::chrono::steady_clock::time_point timestamp;
StandbyState from_state;
StandbyState to_state;
StandbyEvent event;
};
/**
* @brief Get state transition history (for debugging)
*/
std::vector<TransitionRecord> GetTransitionHistory(
size_t max_records = 100) const;
/**
* @brief Get time spent in current state
*/
std::chrono::milliseconds GetTimeInCurrentState() const;
/**
* @brief Get consecutive error count
*/
int GetConsecutiveErrors() const { return consecutive_errors_.load(); }
/**
* @brief Increment consecutive error count
* @return New error count
*/
int IncrementErrors();
/**
* @brief Reset consecutive error count
*/
void ResetErrors() { consecutive_errors_.store(0); }
/**
* @brief Get reconnect attempt count
*/
int GetReconnectCount() const { return reconnect_count_.load(); }
/**
* @brief Increment reconnect count
*/
void IncrementReconnectCount() { reconnect_count_.fetch_add(1); }
/**
* @brief Reset reconnect count
*/
void ResetReconnectCount() { reconnect_count_.store(0); }
// Constants
static constexpr int kMaxConsecutiveErrors = 10;
static constexpr int kMaxReconnectAttempts = 100;
private:
/**
* @brief Check if a transition is valid and get new state
*/
StateTransitionResult ValidateTransition(StandbyState from,
StandbyEvent event) const;
/**
* @brief Notify all registered callbacks
*/
void NotifyCallbacks(StandbyState old_state, StandbyState new_state,
StandbyEvent event);
std::atomic<StandbyState> current_state_{StandbyState::STOPPED};
std::atomic<int> consecutive_errors_{0};
std::atomic<int> reconnect_count_{0};
std::chrono::steady_clock::time_point state_enter_time_;
mutable std::mutex mutex_;
std::vector<StateChangeCallback> callbacks_;
std::vector<TransitionRecord> transition_history_;
static constexpr size_t kMaxHistorySize = 1000;
};
} // namespace mooncake

59
mooncake-store/include/types.h
View File

@ -7,11 +7,13 @@
#include <string>
#include <limits>
#include <unordered_map>
#include <utility>
#include <vector>
#include "Slab.h"
#include "ylt/struct_json/json_reader.h"
#include "ylt/struct_json/json_writer.h"
#include "ylt/struct_pack.hpp"
#ifdef STORE_USE_ETCD
#include "libetcd_wrapper.h"
@ -22,6 +24,63 @@ namespace mooncake {
static constexpr uint64_t WRONG_VERSION = 0;
static constexpr uint64_t DEFAULT_VALUE = UINT64_MAX;
static constexpr uint64_t ERRNO_BASE = DEFAULT_VALUE - 1000;
// Sequence ID comparison utilities for wrap-around safety.
// These functions use signed difference to correctly handle uint64_t overflow
// (from UINT64_MAX wrapping to 0). Assumes sequence IDs won't differ by more
// than 2^63, which is reasonable for practical systems.
//
// Example: If sequence_id wraps from UINT64_MAX to 0, then:
// IsSequenceNewer(0, UINT64_MAX) = true (0 is newer after wrap)
// IsSequenceNewer(UINT64_MAX, 0) = false (UINT64_MAX is older before wrap)
//
// Note: We use 'inline' here to allow multiple definition but keep external
// linkage.
inline bool IsSequenceNewer(uint64_t a, uint64_t b) {
// Cast to int64_t to get signed difference, then check if positive.
// This correctly handles wrap-around: if a wrapped from UINT64_MAX to 0,
// then (int64_t)(a - b) will be positive (assuming gap < 2^63).
return static_cast<int64_t>(a - b) > 0;
}
inline bool IsSequenceOlder(uint64_t a, uint64_t b) {
return static_cast<int64_t>(a - b) < 0;
}
inline bool IsSequenceEqual(uint64_t a, uint64_t b) { return a == b; }
inline bool IsSequenceNewerOrEqual(uint64_t a, uint64_t b) {
return a == b || static_cast<int64_t>(a - b) > 0;
}
inline bool IsSequenceOlderOrEqual(uint64_t a, uint64_t b) {
return a == b || static_cast<int64_t>(a - b) < 0;
}
// Cluster ID validation utilities.
//
// cluster_id is used to construct etcd key prefixes (e.g.
// "/oplog/<cluster_id>/..."). To avoid key-prefix injection / accidental
// cross-cluster overlap, we restrict the allowed characters to a conservative
// safe set. We validate the "component" form (without trailing slash). Trailing
// slashes should be normalized away before validation.
inline bool IsValidClusterIdComponent(const std::string& cluster_id) {
if (cluster_id.empty()) {
return false;
}
if (cluster_id.size() > 128) {
return false;
}
for (unsigned char c : cluster_id) {
const bool ok = (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') ||
(c >= 'a' && c <= 'z') || c == '_' || c == '-' ||
c == '.';
if (!ok) {
return false;
}
}
return true;
}
static constexpr uint64_t DEFAULT_DEFAULT_KV_LEASE_TTL =
5000; // in milliseconds
static constexpr uint64_t DEFAULT_KV_SOFT_PIN_TTL_MS =

121
mooncake-store/include/utils/base64.h Normal file
View File

@ -0,0 +1,121 @@
#pragma once
#include <cstdint>
#include <string>
namespace mooncake {
namespace base64 {
// Base64 encoding for binary payload.
// JsonCpp treats strings as UTF-8, so we must encode binary data.
inline std::string Encode(const std::string& data) {
static const char base64_chars[] =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
std::string result;
result.reserve(((data.size() + 2) / 3) * 4);
size_t i = 0;
size_t data_len = data.size();
// Process 3 bytes at a time
while (i + 2 < data_len) {
uint32_t octet_a = static_cast<unsigned char>(data[i++]);
uint32_t octet_b = static_cast<unsigned char>(data[i++]);
uint32_t octet_c = static_cast<unsigned char>(data[i++]);
uint32_t triple = (octet_a << 16) | (octet_b << 8) | octet_c;
result.push_back(base64_chars[(triple >> 18) & 0x3F]);
result.push_back(base64_chars[(triple >> 12) & 0x3F]);
result.push_back(base64_chars[(triple >> 6) & 0x3F]);
result.push_back(base64_chars[triple & 0x3F]);
}
// Handle remaining bytes
size_t remaining = data_len - i;
if (remaining > 0) {
uint32_t octet_a = static_cast<unsigned char>(data[i++]);
uint32_t octet_b =
(remaining > 1) ? static_cast<unsigned char>(data[i++]) : 0;
uint32_t octet_c = 0;
uint32_t triple = (octet_a << 16) | (octet_b << 8) | octet_c;
result.push_back(base64_chars[(triple >> 18) & 0x3F]);
result.push_back(base64_chars[(triple >> 12) & 0x3F]);
result.push_back((remaining > 1) ? base64_chars[(triple >> 6) & 0x3F]
: '=');
result.push_back('=');
}
return result;
}
// Base64 decoding for binary payload.
inline std::string Decode(const std::string& encoded) {
static const unsigned char decode_table[256] = {
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
64, 64, 64, 64, 64, 64, 64, 62, 64, 64, 64, 63, 52, 53, 54, 55, 56, 57,
58, 59, 60, 61, 64, 64, 64, 64, 64, 64, 64, 0, 1, 2, 3, 4, 5, 6,
7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
25, 64, 64, 64, 64, 64, 64, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 64, 64, 64,
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
64, 64, 64, 64};
std::string result;
result.reserve((encoded.size() * 3) / 4);
size_t i = 0;
while (i < encoded.size()) {
// Skip whitespace and invalid chars
while (i < encoded.size() &&
(encoded[i] == ' ' || encoded[i] == '\n' || encoded[i] == '\r' ||
encoded[i] == '\t')) {
i++;
}
if (i >= encoded.size()) break;
uint32_t sextet_a =
decode_table[static_cast<unsigned char>(encoded[i++])];
if (i >= encoded.size() || sextet_a == 64) break;
uint32_t sextet_b =
decode_table[static_cast<unsigned char>(encoded[i++])];
if (sextet_b == 64) break;
uint32_t sextet_c =
(i < encoded.size())
? decode_table[static_cast<unsigned char>(encoded[i++])]
: 64;
uint32_t sextet_d =
(i < encoded.size())
? decode_table[static_cast<unsigned char>(encoded[i++])]
: 64;
uint32_t triple = (sextet_a << 18) | (sextet_b << 12) |
((sextet_c != 64) ? (sextet_c << 6) : 0) |
((sextet_d != 64) ? sextet_d : 0);
result.push_back(static_cast<char>((triple >> 16) & 0xFF));
if (sextet_c != 64) {
result.push_back(static_cast<char>((triple >> 8) & 0xFF));
}
if (sextet_d != 64) {
result.push_back(static_cast<char>(triple & 0xFF));
}
}
return result;
}
} // namespace base64
} // namespace mooncake

150
mooncake-store/src/CMakeLists.txt
View File

@ -16,8 +16,6 @@ set(MOONCAKE_STORE_SOURCES
ha_helper.cpp
segment.cpp
transfer_task.cpp
etcd_helper.cpp
ha_helper.cpp
rpc_service.cpp
offset_allocator.cpp
posix_file.cpp
@ -34,44 +32,73 @@ set(MOONCAKE_STORE_SOURCES
utils/file_util.cpp
task_manager.cpp
local_hot_cache.cpp
)
oplog_manager.cpp
etcd_oplog_store.cpp
oplog_watcher.cpp
oplog_applier.cpp
hot_standby_service.cpp
standby_state_machine.cpp
ha_metric_manager.cpp)
set(EXTRA_LIBS "")
# Find AWS SDK
find_package(AWSSDK QUIET COMPONENTS s3)
if(AWSSDK_FOUND)
message(STATUS "AWS SDK found: ${AWSSDK_VERSION}")
set(HAVE_AWS_SDK TRUE)
# Add S3 related source files
list(APPEND MOONCAKE_STORE_SOURCES utils/s3_helper.cpp)
message(STATUS "AWS SDK found: ${AWSSDK_VERSION}")
set(HAVE_AWS_SDK TRUE)
# Add S3 related source files
list(APPEND MOONCAKE_STORE_SOURCES utils/s3_helper.cpp)
else()
message(STATUS "AWS SDK not found, S3 functionality will be disabled")
set(HAVE_AWS_SDK FALSE)
message(STATUS "AWS SDK not found, S3 functionality will be disabled")
set(HAVE_AWS_SDK FALSE)
endif()
# Find zstd library
find_library(ZSTD_LIBRARY NAMES zstd)
if(NOT ZSTD_LIBRARY)
message(FATAL_ERROR "zstd library not found")
message(FATAL_ERROR "zstd library not found")
endif()
set(EXTRA_LIBS ${ZSTD_LIBRARY})
# If AWS SDK is found, add it to EXTRA_LIBS
if(HAVE_AWS_SDK)
list(APPEND EXTRA_LIBS ${AWSSDK_LINK_LIBRARIES})
list(APPEND EXTRA_LIBS ${AWSSDK_LINK_LIBRARIES})
add_definitions(-DHAVE_AWS_SDK)
if(AWS_S3_LIB AND AWS_CORE_LIB)
list(APPEND EXTRA_LIBS ${AWS_S3_LIB} ${AWS_CORE_LIB})
add_definitions(-DHAVE_AWS_SDK)
if(AWS_S3_LIB AND AWS_CORE_LIB)
list(APPEND EXTRA_LIBS ${AWS_S3_LIB} ${AWS_CORE_LIB})
add_definitions(-DHAVE_AWS_SDK)
endif()
endif()
endif()
# Find xxHash (required for ComputeChecksum)
find_path(
XXHASH_INCLUDE_DIR
NAMES xxhash.h
PATHS /usr/include /usr/local/include)
find_library(
XXHASH_LIBRARY
NAMES xxhash libxxhash
PATHS /usr/lib /usr/local/lib /usr/lib64)
if(XXHASH_INCLUDE_DIR AND XXHASH_LIBRARY)
message(
STATUS "Found xxHash: include=${XXHASH_INCLUDE_DIR} lib=${XXHASH_LIBRARY}")
list(APPEND MASTER_EXTRA_INCS ${XXHASH_INCLUDE_DIR})
else()
message(
FATAL_ERROR
"xxHash library/header not found. Please install xxhash (development headers) and try again."
)
endif()
if(USE_3FS)
add_subdirectory(hf3fs)
list(APPEND MOONCAKE_STORE_SOURCES ${HF3FS_SOURCES})
find_library(HF3FS_API_LIB hf3fs_api_shared PATHS /usr/lib NO_DEFAULT_PATH)
list(APPEND MOONCAKE_STORE_SOURCES ${HF3FS_SOURCES})
find_library(
HF3FS_API_LIB hf3fs_api_shared
PATHS /usr/lib
NO_DEFAULT_PATH)
if(NOT HF3FS_API_LIB)
message(FATAL_ERROR "hf3fs_api_shared library not found in /usr/lib")
endif()
@ -79,7 +106,8 @@ if(USE_3FS)
endif()
# io_uring support (auto-detected)
find_library(URING_LIB uring PATHS /usr/lib /usr/lib64 /usr/local/lib /usr/local/lib64)
find_library(URING_LIB uring PATHS /usr/lib /usr/lib64 /usr/local/lib
/usr/local/lib64)
find_path(URING_INCLUDE liburing.h PATHS /usr/include /usr/local/include)
if(URING_LIB AND URING_INCLUDE)
message(STATUS "io_uring: Enabled for mooncake_store")
@ -92,30 +120,41 @@ endif()
# The cache_allocator library
include_directories(${Python3_INCLUDE_DIRS})
add_library(mooncake_store ${MOONCAKE_STORE_SOURCES})
# Note: transfer_engine is PRIVATE to avoid propagating its dependencies (e.g., vendor-specific hardware)
# to targets that don't need it (e.g., mooncake_master).
# Targets that need transfer_engine should link it explicitly.
target_link_libraries(mooncake_store
PUBLIC
cachelib_memory_allocator
${ETCD_WRAPPER_LIB}
glog::glog
gflags::gflags
${EXTRA_LIBS}
asio_shared
PRIVATE
transfer_engine
)
if (STORE_USE_ETCD)
add_dependencies(mooncake_store build_etcd_wrapper)
target_include_directories(mooncake_store PUBLIC ${XXHASH_INCLUDE_DIR})
target_link_libraries(mooncake_store PUBLIC ${XXHASH_LIBRARY})
# Note: transfer_engine is PRIVATE to avoid propagating its dependencies (e.g.,
# vendor-specific hardware) to targets that don't need it (e.g.,
# mooncake_master). Targets that need transfer_engine should link it explicitly.
target_link_libraries(
mooncake_store
PUBLIC cachelib_memory_allocator ${ETCD_WRAPPER_LIB} glog::glog gflags::gflags
${EXTRA_LIBS} asio_shared
PRIVATE transfer_engine)
if(STORE_USE_ETCD)
add_dependencies(mooncake_store build_etcd_wrapper)
endif()
if(URING_LIB AND URING_INCLUDE)
target_compile_definitions(mooncake_store PUBLIC USE_URING)
target_include_directories(mooncake_store PRIVATE ${URING_INCLUDE})
target_compile_definitions(mooncake_store PUBLIC USE_URING)
target_include_directories(mooncake_store PRIVATE ${URING_INCLUDE})
endif()
if (BUILD_SHARED_LIBS)
if(USE_ASCEND_DIRECT)
set(ACL_RUNTIME_HEADER_PATH "${ASCEND_INCLUDE_DIR}/acl/acl_rt.h")
if(EXISTS "${ACL_RUNTIME_HEADER_PATH}")
file(READ "${ACL_RUNTIME_HEADER_PATH}" ACL_RUNTIME_CONTENT)
if("${ACL_RUNTIME_CONTENT}" MATCHES "ACL_MEM_P2P_HUGE1G")
message(STATUS "ACL_MEM_P2P_HUGE1G exist.")
add_compile_definitions(ASCEND_SUPPORT_FABRIC_MEM)
else()
message(STATUS "ACL_MEM_P2P_HUGE1G is not exist.")
endif()
else()
message(WARNING "Acl runtime header file is not exist")
endif()
endif()
if(BUILD_SHARED_LIBS)
install(TARGETS mooncake_store DESTINATION lib)
endif()
@ -125,27 +164,27 @@ add_executable(mooncake_master master.cpp)
set(MASTER_EXTRA_INCS)
set(MASTER_EXTRA_LIBS)
if (STORE_USE_JEMALLOC)
find_package(PkgConfig REQUIRED)
pkg_check_modules(JEMALLOC REQUIRED jemalloc)
list(APPEND MASTER_EXTRA_INCS ${JEMALLOC_INCLUDE_DIRS})
list(APPEND MASTER_EXTRA_LIBS ${JEMALLOC_STATIC_LIBRARIES})
if(STORE_USE_JEMALLOC)
find_package(PkgConfig REQUIRED)
pkg_check_modules(JEMALLOC REQUIRED jemalloc)
list(APPEND MASTER_EXTRA_INCS ${JEMALLOC_INCLUDE_DIRS})
list(APPEND MASTER_EXTRA_LIBS ${JEMALLOC_STATIC_LIBRARIES})
endif()
target_include_directories(mooncake_master PRIVATE ${MASTER_EXTRA_INCS})
target_link_libraries(mooncake_master PRIVATE
mooncake_store
cachelib_memory_allocator
pthread
ibverbs
mooncake_common
${ETCD_WRAPPER_LIB}
${MASTER_EXTRA_LIBS}
asio_shared
)
target_link_libraries(
mooncake_master
PRIVATE mooncake_store
cachelib_memory_allocator
pthread
ibverbs
mooncake_common
${ETCD_WRAPPER_LIB}
${MASTER_EXTRA_LIBS}
asio_shared)
if (STORE_USE_ETCD)
add_dependencies(mooncake_master build_etcd_wrapper)
if(STORE_USE_ETCD)
add_dependencies(mooncake_master build_etcd_wrapper)
endif()
target_compile_options(mooncake_master PRIVATE -Os)
@ -154,7 +193,8 @@ target_link_options(mooncake_master PRIVATE -Os -s)
# Client server binary
add_executable(mooncake_client real_client_main.cpp)
# Client needs transfer_engine for data transfer operations
target_link_libraries(mooncake_client PRIVATE mooncake_store transfer_engine asio_shared)
target_link_libraries(mooncake_client PRIVATE mooncake_store transfer_engine
asio_shared)
target_compile_options(mooncake_client PRIVATE -Os)
target_link_options(mooncake_client PRIVATE -Os -s)

105
mooncake-store/src/etcd_helper.cpp
View File

@ -26,7 +26,8 @@ ErrorCode EtcdHelper::ConnectToEtcdStoreClient(
return ErrorCode::OK;
} else {
char* err_msg = nullptr;
int ret = NewStoreEtcdClient((char*)etcd_endpoints.c_str(), &err_msg);
int ret = NewStoreEtcdClient(const_cast<char*>(etcd_endpoints.c_str()),
&err_msg);
// ret == -2 means the etcd client has already been initialized
if (ret != 0 && ret != -2) {
LOG(ERROR) << "Failed to initialize etcd client: " << err_msg;
@ -46,8 +47,9 @@ ErrorCode EtcdHelper::Get(const char* key, const size_t key_size,
char* err_msg = nullptr;
char* value_ptr = nullptr;
int value_size = 0;
int ret = EtcdStoreGetWrapper((char*)key, (int)key_size, &value_ptr,
&value_size, &revision_id, &err_msg);
int ret =
EtcdStoreGetWrapper(const_cast<char*>(key), (int)key_size, &value_ptr,
&value_size, &revision_id, &err_msg);
if (ret == -2) {
LOG(ERROR) << "key=" << std::string(key, key_size)
<< ", error=" << err_msg;
@ -71,9 +73,9 @@ ErrorCode EtcdHelper::CreateWithLease(const char* key, const size_t key_size,
EtcdLeaseId lease_id,
EtcdRevisionId& revision_id) {
char* err_msg = nullptr;
int ret = EtcdStoreCreateWithLeaseWrapper((char*)key, (int)key_size,
(char*)value, (int)value_size,
lease_id, &revision_id, &err_msg);
int ret = EtcdStoreCreateWithLeaseWrapper(
const_cast<char*>(key), (int)key_size, const_cast<char*>(value),
(int)value_size, lease_id, &revision_id, &err_msg);
if (ret == -2) {
VLOG(1) << "key=" << std::string(key, key_size)
<< ", lease_id=" << lease_id << ", error=" << err_msg;
@ -89,6 +91,46 @@ ErrorCode EtcdHelper::CreateWithLease(const char* key, const size_t key_size,
}
}
ErrorCode EtcdHelper::BatchCreate(const std::vector<std::string>& keys,
const std::vector<std::string>& values) {
if (keys.size() != values.size()) {
return ErrorCode::INVALID_PARAMS;
}
if (keys.empty()) {
return ErrorCode::OK;
}
std::vector<char*> c_keys;
std::vector<char*> c_values;
c_keys.reserve(keys.size());
c_values.reserve(values.size());
for (const auto& key : keys) {
c_keys.push_back(const_cast<char*>(key.c_str()));
}
for (const auto& val : values) {
c_values.push_back(const_cast<char*>(val.c_str()));
}
char* err_msg = nullptr;
int ret = EtcdStoreBatchCreateWrapper(c_keys.data(), c_values.data(),
(int)keys.size(), &err_msg);
if (ret == -2) {
if (err_msg) {
LOG(ERROR) << "BatchCreate transaction failed: " << err_msg;
free(err_msg);
}
return ErrorCode::ETCD_TRANSACTION_FAIL;
} else if (ret != 0) {
if (err_msg) {
LOG(ERROR) << "BatchCreate failed: " << err_msg;
free(err_msg);
}
return ErrorCode::ETCD_OPERATION_ERROR;
}
return ErrorCode::OK;
}
ErrorCode EtcdHelper::GrantLease(int64_t lease_ttl, EtcdLeaseId& lease_id) {
char* err_msg = nullptr;
if (0 != EtcdStoreGrantLeaseWrapper(lease_ttl, &lease_id, &err_msg)) {
@ -102,8 +144,8 @@ ErrorCode EtcdHelper::GrantLease(int64_t lease_ttl, EtcdLeaseId& lease_id) {
ErrorCode EtcdHelper::WatchUntilDeleted(const char* key,
const size_t key_size) {
char* err_msg = nullptr;
int err_code =
EtcdStoreWatchUntilDeletedWrapper((char*)key, (int)key_size, &err_msg);
int err_code = EtcdStoreWatchUntilDeletedWrapper(const_cast<char*>(key),
(int)key_size, &err_msg);
if (err_code != 0) {
LOG(ERROR) << "key=" << std::string(key, key_size)
<< ", error=" << err_msg;
@ -119,7 +161,8 @@ ErrorCode EtcdHelper::WatchUntilDeleted(const char* key,
ErrorCode EtcdHelper::CancelWatch(const char* key, const size_t key_size) {
char* err_msg = nullptr;
if (0 != EtcdStoreCancelWatchWrapper((char*)key, (int)key_size, &err_msg)) {
if (0 != EtcdStoreCancelWatchWrapper(const_cast<char*>(key), (int)key_size,
&err_msg)) {
LOG(ERROR) << "key=" << std::string(key, key_size)
<< ", error=" << err_msg;
free(err_msg);
@ -156,8 +199,9 @@ ErrorCode EtcdHelper::CancelKeepAlive(EtcdLeaseId lease_id) {
ErrorCode EtcdHelper::Put(const char* key, const size_t key_size,
const char* value, const size_t value_size) {
char* err_msg = nullptr;
int ret = EtcdStorePutWrapper((char*)key, (int)key_size, (char*)value,
(int)value_size, &err_msg);
int ret = EtcdStorePutWrapper(const_cast<char*>(key), (int)key_size,
const_cast<char*>(value), (int)value_size,
&err_msg);
if (ret != 0) {
LOG(ERROR) << "key=" << std::string(key, key_size)
<< ", error=" << err_msg;
@ -170,8 +214,9 @@ ErrorCode EtcdHelper::Put(const char* key, const size_t key_size,
ErrorCode EtcdHelper::Create(const char* key, const size_t key_size,
const char* value, const size_t value_size) {
char* err_msg = nullptr;
int ret = EtcdStoreCreateWrapper((char*)key, (int)key_size, (char*)value,
(int)value_size, &err_msg);
int ret = EtcdStoreCreateWrapper(const_cast<char*>(key), (int)key_size,
const_cast<char*>(value), (int)value_size,
&err_msg);
if (ret == -2) {
free(err_msg);
return ErrorCode::ETCD_TRANSACTION_FAIL;
@ -196,9 +241,9 @@ ErrorCode EtcdHelper::GetRangeAsJson(const char* start_key,
int json_size = 0;
// Go wrapper takes int limit.
int ret = EtcdStoreGetRangeAsJsonWrapper(
(char*)start_key, (int)start_key_size, (char*)end_key,
(int)end_key_size, (int)limit, &json_ptr, &json_size,
(GoInt64*)&revision_id, &err_msg);
const_cast<char*>(start_key), (int)start_key_size,
const_cast<char*>(end_key), (int)end_key_size, (int)limit, &json_ptr,
&json_size, (GoInt64*)&revision_id, &err_msg);
if (ret != 0) {
LOG(ERROR) << "start_key=" << std::string(start_key, start_key_size)
<< ", end_key=" << std::string(end_key, end_key_size)
@ -218,8 +263,8 @@ ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix,
char* first_key_ptr = nullptr;
int first_key_size = 0;
int ret = EtcdStoreGetFirstKeyWithPrefixWrapper(
(char*)prefix, (int)prefix_size, &first_key_ptr, &first_key_size,
&err_msg);
const_cast<char*>(prefix), (int)prefix_size, &first_key_ptr,
&first_key_size, &err_msg);
if (ret == -2) {
free(err_msg);
return ErrorCode::ETCD_KEY_NOT_EXIST;
@ -242,8 +287,8 @@ ErrorCode EtcdHelper::GetLastKeyWithPrefix(const char* prefix,
char* last_key_ptr = nullptr;
int last_key_size = 0;
int ret = EtcdStoreGetLastKeyWithPrefixWrapper(
(char*)prefix, (int)prefix_size, &last_key_ptr, &last_key_size,
&err_msg);
const_cast<char*>(prefix), (int)prefix_size, &last_key_ptr,
&last_key_size, &err_msg);
if (ret == -2) {
free(err_msg);
return ErrorCode::ETCD_KEY_NOT_EXIST;
@ -264,9 +309,9 @@ ErrorCode EtcdHelper::DeleteRange(const char* start_key,
const char* end_key,
const size_t end_key_size) {
char* err_msg = nullptr;
int ret = EtcdStoreDeleteRangeWrapper((char*)start_key, (int)start_key_size,
(char*)end_key, (int)end_key_size,
&err_msg);
int ret = EtcdStoreDeleteRangeWrapper(
const_cast<char*>(start_key), (int)start_key_size,
const_cast<char*>(end_key), (int)end_key_size, &err_msg);
if (ret != 0) {
LOG(ERROR) << "start_key=" << std::string(start_key, start_key_size)
<< ", end_key=" << std::string(end_key, end_key_size)
@ -285,7 +330,7 @@ ErrorCode EtcdHelper::WatchWithPrefixFromRevision(
char* err_msg = nullptr;
void* callback_func_ptr = reinterpret_cast<void*>(callback_func);
int ret = EtcdStoreWatchWithPrefixFromRevisionWrapper(
(char*)prefix, (int)prefix_size, (GoInt64)start_revision,
const_cast<char*>(prefix), (int)prefix_size, (GoInt64)start_revision,
callback_context, callback_func_ptr, &err_msg);
if (ret != 0) {
LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size)
@ -300,7 +345,7 @@ ErrorCode EtcdHelper::WatchWithPrefixFromRevision(
ErrorCode EtcdHelper::CancelWatchWithPrefix(const char* prefix,
const size_t prefix_size) {
char* err_msg = nullptr;
int ret = EtcdStoreCancelWatchWithPrefixWrapper((char*)prefix,
int ret = EtcdStoreCancelWatchWithPrefixWrapper(const_cast<char*>(prefix),
(int)prefix_size, &err_msg);
if (ret != 0) {
LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size)
@ -316,7 +361,7 @@ ErrorCode EtcdHelper::WaitWatchWithPrefixStopped(const char* prefix,
int timeout_ms) {
char* err_msg = nullptr;
int ret = EtcdStoreWaitWatchWithPrefixStoppedWrapper(
(char*)prefix, (int)prefix_size, timeout_ms, &err_msg);
const_cast<char*>(prefix), (int)prefix_size, timeout_ms, &err_msg);
if (ret != 0) {
LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size)
<< ", timeout_ms=" << timeout_ms
@ -361,6 +406,14 @@ ErrorCode EtcdHelper::CreateWithLease(const char* key, const size_t key_size,
return ErrorCode::ETCD_OPERATION_ERROR;
}
ErrorCode EtcdHelper::BatchCreate(const std::vector<std::string>& keys,
const std::vector<std::string>& values) {
(void)keys;
(void)values;
LOG(FATAL) << "Etcd is not enabled in compilation";
return ErrorCode::ETCD_OPERATION_ERROR;
}
ErrorCode EtcdHelper::GrantLease(int64_t lease_ttl, EtcdLeaseId& lease_id) {
(void)lease_ttl;
(void)lease_id;

747
mooncake-store/src/etcd_oplog_store.cpp Normal file
View File

@ -0,0 +1,747 @@
#include "etcd_oplog_store.h"
#include <glog/logging.h>
#include <sstream>
#include <iomanip>
#include "ha_metric_manager.h"
#include "utils/base64.h"
#if __has_include(<jsoncpp/json/json.h>)
#include <jsoncpp/json/json.h> // Ubuntu
#else
#include <json/json.h> // CentOS
#endif
#include "etcd_helper.h"
namespace mooncake {
EtcdOpLogStore::EtcdOpLogStore(const std::string& cluster_id,
bool enable_latest_seq_batch_update,
bool enable_batch_write)
: cluster_id_(cluster_id),
enable_latest_seq_batch_update_(enable_latest_seq_batch_update),
enable_batch_write_(enable_batch_write),
last_update_time_(std::chrono::steady_clock::now()) {
// Normalize cluster_id to avoid accidental double slashes in etcd keys when
// caller passes a trailing '/' (master_view_key uses trailing '/', OpLog
// keys don't).
while (!cluster_id_.empty() && cluster_id_.back() == '/') {
cluster_id_.pop_back();
}
if (!cluster_id_.empty() && !IsValidClusterIdComponent(cluster_id_)) {
LOG(FATAL)
<< "Invalid cluster_id for EtcdOpLogStore: '" << cluster_id_
<< "'. Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes.";
}
}
ErrorCode EtcdOpLogStore::Init() {
// Initialize /latest key to 0 if it doesn't exist (first startup).
// This avoids "key not found" errors when querying the latest sequence ID.
// Important: Only initialize if the key doesn't exist to avoid overwriting
// existing data.
// Skip for read-only instances to avoid unnecessary etcd writes.
if (enable_batch_write_ && !cluster_id_.empty()) {
std::string latest_key = BuildLatestKey();
std::string existing_value;
EtcdRevisionId revision_id;
ErrorCode get_err = EtcdHelper::Get(
latest_key.c_str(), latest_key.size(), existing_value, revision_id);
if (get_err == ErrorCode::ETCD_KEY_NOT_EXIST) {
// Key doesn't exist, safe to initialize to 0
std::string initial_value = "0";
ErrorCode create_err =
EtcdHelper::Create(latest_key.c_str(), latest_key.size(),
initial_value.c_str(), initial_value.size());
if (create_err == ErrorCode::OK) {
LOG(INFO) << "Initialized /latest key to 0 for cluster_id="
<< cluster_id_;
} else if (create_err == ErrorCode::ETCD_TRANSACTION_FAIL) {
// Race condition: another instance created it between Get and
// Create
LOG(INFO) << "/latest key was created by another instance for "
"cluster_id="
<< cluster_id_;
} else {
// Other errors (e.g., etcd not connected) are logged but don't
// fail initialization. The key will be created when the first
// OpLog entry is written
LOG(WARNING)
<< "Failed to initialize /latest key (error=" << create_err
<< "), will be created on first OpLog write";
}
} else if (get_err == ErrorCode::OK) {
// Key already exists, do nothing - preserve existing value
LOG(INFO) << "/latest key already exists (value=" << existing_value
<< ") for cluster_id=" << cluster_id_;
} else {
// Other errors (e.g., etcd not connected) are logged but don't fail
// initialization
LOG(WARNING) << "Failed to check /latest key existence (error="
<< get_err
<< "), will be created on first OpLog write";
}
}
// Start batch update thread only for writers.
if (enable_latest_seq_batch_update_) {
// Prevent double start
if (!batch_update_running_.exchange(true)) {
batch_update_thread_ =
std::thread(&EtcdOpLogStore::BatchUpdateThread, this);
}
}
// Start OpLog batch write thread only for writers.
if (enable_batch_write_) {
// Prevent double start
if (!batch_write_running_.exchange(true)) {
batch_write_thread_ =
std::thread(&EtcdOpLogStore::BatchWriteThread, this);
}
}
return ErrorCode::OK;
}
EtcdOpLogStore::~EtcdOpLogStore() {
// Stop OpLog batch write thread (only started for writers)
if (enable_batch_write_) {
batch_write_running_.store(false);
cv_batch_updated_.notify_all();
if (batch_write_thread_.joinable()) {
batch_write_thread_.join();
}
// Attempt final flush (FlushBatch manages its own locking)
FlushBatch();
}
if (!enable_latest_seq_batch_update_) {
return;
}
// Stop batch update thread
batch_update_running_.store(false);
if (batch_update_thread_.joinable()) {
batch_update_thread_.join();
}
// Perform final update if there are pending updates
if (pending_count_.load() > 0) {
DoBatchUpdate();
}
}
ErrorCode EtcdOpLogStore::WriteOpLog(const OpLogEntry& entry, bool sync) {
if (!enable_batch_write_) {
LOG(ERROR) << "WriteOpLog called on a read-only EtcdOpLogStore "
<< "(enable_batch_write=false), cluster_id=" << cluster_id_;
return ErrorCode::INVALID_PARAMS;
}
std::string key = BuildOpLogKey(entry.sequence_id);
std::string value = SerializeOpLogEntry(entry);
{
std::unique_lock<std::mutex> lock(batch_mutex_);
pending_batch_.push_back(
{std::move(key), std::move(value), entry.sequence_id, sync});
bool should_notify = false;
if (sync) {
// Strategy 2+: Sync writes (DELETE) trigger immediate flush
should_notify = true;
} else {
// Async writes (PUT_END): trigger if threshold reached
if (pending_batch_.size() >= kOpLogBatchCountLimit) {
should_notify = true;
}
}
if (should_notify) {
cv_batch_updated_.notify_one();
}
if (sync) {
// Wait for persistence
uint64_t target_seq = entry.sequence_id;
bool success = cv_sync_completed_.wait_for(
lock, std::chrono::milliseconds(kSyncWaitTimeoutMs),
[&] { return last_persisted_seq_id_.load() >= target_seq; });
if (!success) {
LOG(ERROR) << "Timeout waiting for OpLog persistence, seq="
<< target_seq;
return ErrorCode::ETCD_OPERATION_ERROR;
}
}
}
// Update /latest pointer logic
// We defer this to the batch flush or just queue it up here?
// Original logic:
if (!enable_latest_seq_batch_update_) {
// Direct update (may be slow, but it's what config asked for)
// Warning: This is now done AFTER op log write, which is correct order.
return UpdateLatestSequenceId(entry.sequence_id);
}
// For batch update, we update the pending counter
pending_latest_seq_id_.store(entry.sequence_id);
size_t count = pending_count_.fetch_add(1) + 1;
if (count >= kBatchSize) {
DoBatchUpdate();
}
return ErrorCode::OK;
}
void EtcdOpLogStore::BatchWriteThread() {
while (batch_write_running_.load()) {
{
std::unique_lock<std::mutex> lock(batch_mutex_);
if (pending_batch_.empty()) {
// Wait for signal or timeout (Group Commit time window)
cv_batch_updated_.wait_for(
lock, std::chrono::milliseconds(kOpLogBatchTimeoutMs));
}
if (!batch_write_running_.load() && pending_batch_.empty()) {
break;
}
}
FlushBatch();
}
}
void EtcdOpLogStore::FlushBatch() {
// Step 1: Take pending batch under lock.
std::deque<BatchEntry> batch_to_write;
{
std::lock_guard<std::mutex> lock(batch_mutex_);
batch_to_write.swap(pending_batch_);
}
if (batch_to_write.empty()) {
return;
}
// Step 2: Perform IO without holding the lock.
std::vector<std::string> keys;
std::vector<std::string> values;
keys.reserve(batch_to_write.size());
values.reserve(batch_to_write.size());
uint64_t max_seq = 0;
bool has_sync_entry = false;
for (const auto& entry : batch_to_write) {
keys.push_back(entry.key);
if (entry.is_sync) {
has_sync_entry = true;
}
values.push_back(entry.value);
if (entry.sequence_id > max_seq) {
max_seq = entry.sequence_id;
}
}
ErrorCode err = ErrorCode::OK;
for (int i = 0; i <= kFlushRetryCount; ++i) {
err = EtcdHelper::BatchCreate(keys, values);
if (err == ErrorCode::OK) {
break;
}
if (err == ErrorCode::ETCD_TRANSACTION_FAIL) {
// BatchCreate uses Txn(If all keys CreateRevision==0).
// Transaction failure means some keys already exist — likely
// from a previous attempt that timed out but actually succeeded
// on the etcd side. Since OpLog entries are idempotent (same
// sequence_id → same key/value), we can safely fall back to
// individual Put (overwrite) for the remaining keys.
LOG(WARNING)
<< "BatchCreate transaction failed (keys already exist), "
<< "falling back to per-key Put for " << keys.size()
<< " entries";
bool all_ok = true;
for (size_t j = 0; j < keys.size(); ++j) {
ErrorCode put_err =
EtcdHelper::Put(keys[j].c_str(), keys[j].size(),
values[j].c_str(), values[j].size());
if (put_err != ErrorCode::OK) {
LOG(ERROR) << "Fallback Put failed for key=" << keys[j];
all_ok = false;
}
}
if (all_ok) {
err = ErrorCode::OK;
}
break; // Do not retry further; fallback already handled it.
}
if (i < kFlushRetryCount) {
LOG(WARNING) << "Failed to flush OpLog batch (attempt " << i + 1
<< "/" << kFlushRetryCount + 1 << "), retrying...";
std::this_thread::sleep_for(
std::chrono::milliseconds(kFlushRetryIntervalMs));
}
}
// Step 3: Update state under lock.
{
std::lock_guard<std::mutex> lock(batch_mutex_);
if (err == ErrorCode::OK) {
if (max_seq > last_persisted_seq_id_.load()) {
last_persisted_seq_id_.store(max_seq);
}
// Update HA metrics
HAMetricManager::instance().inc_oplog_batch_commits();
if (has_sync_entry) {
HAMetricManager::instance().inc_oplog_sync_batch_commits();
}
if (batch_to_write.size() > 1) {
LOG(INFO)
<< "HA Strategy: Group Commit flush success. batch_size="
<< batch_to_write.size() << ", max_seq=" << max_seq;
} else {
VLOG(3)
<< "HA Strategy: Group Commit flush success. batch_size=1, "
"max_seq="
<< max_seq;
if (!has_sync_entry) {
LOG_EVERY_N(INFO, 1000)
<< "Note: Frequent single-entry async "
"flushes detected (sample).";
}
}
} else {
LOG(ERROR) << "Failed to flush OpLog batch, count="
<< batch_to_write.size();
}
}
// Wake up all waiting threads (Strategy 2+: DELETE waiters)
cv_sync_completed_.notify_all();
}
ErrorCode EtcdOpLogStore::ReadOpLog(uint64_t sequence_id, OpLogEntry& entry) {
std::string key = BuildOpLogKey(sequence_id);
std::string value;
EtcdRevisionId revision_id;
ErrorCode err =
EtcdHelper::Get(key.c_str(), key.size(), value, revision_id);
if (err != ErrorCode::OK) {
return err;
}
if (!DeserializeOpLogEntry(value, entry)) {
LOG(ERROR) << "Failed to deserialize OpLog entry, sequence_id="
<< sequence_id;
return ErrorCode::INTERNAL_ERROR;
}
return ErrorCode::OK;
}
ErrorCode EtcdOpLogStore::ReadOpLogSince(uint64_t start_sequence_id,
size_t limit,
std::vector<OpLogEntry>& entries) {
EtcdRevisionId rev = 0;
return ReadOpLogSinceWithRevision(start_sequence_id, limit, entries, rev);
}
ErrorCode EtcdOpLogStore::ReadOpLogSinceWithRevision(
uint64_t start_sequence_id, size_t limit, std::vector<OpLogEntry>& entries,
EtcdRevisionId& revision_id) {
entries.clear();
entries.reserve(limit);
// Range is limited to OpLog entry keys only.
const std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/";
std::string current_start_key = BuildOpLogKey(start_sequence_id + 1);
// Compute prefix range end (etcd prefix end).
auto prefix_end = [](std::string p) -> std::string {
for (int i = static_cast<int>(p.size()) - 1; i >= 0; --i) {
unsigned char c = static_cast<unsigned char>(p[i]);
if (c < 0xFF) {
p[i] = static_cast<char>(c + 1);
p.resize(i + 1);
return p;
}
}
return std::string(1, '\0');
};
const std::string end_key = prefix_end(prefix);
// Pagination:
// - Use range-get with limit
// - Start next page from lastKey + '\0' (lexicographically just after
// lastKey) This avoids repeating the last key without adding new Go/C++
// APIs.
revision_id = 0;
while (entries.size() < limit) {
const size_t page_limit = limit - entries.size();
std::string json;
EtcdRevisionId page_rev = 0;
ErrorCode err = EtcdHelper::GetRangeAsJson(
current_start_key.c_str(), current_start_key.size(),
end_key.c_str(), end_key.size(), page_limit, json, page_rev);
if (err != ErrorCode::OK) {
return err;
}
if (page_rev > revision_id) {
revision_id = page_rev;
}
// Parse kv list: [{"key":"...","value":"..."}]
Json::Value root;
Json::CharReaderBuilder reader;
std::string errs;
std::istringstream s(json);
if (!Json::parseFromStream(reader, s, &root, &errs)) {
LOG(ERROR) << "Failed to parse range JSON: " << errs;
return ErrorCode::INTERNAL_ERROR;
}
if (!root.isArray()) {
return ErrorCode::INTERNAL_ERROR;
}
if (root.empty()) {
break; // no more data
}
std::string last_key_in_page;
for (const auto& kv : root) {
const std::string key = kv.get("key", "").asString();
last_key_in_page = key;
if (key.empty() || key.find("/latest") != std::string::npos ||
key.find("/snapshot/") != std::string::npos) {
continue;
}
// Parse seq from key suffix and filter (handles legacy keys too).
size_t pos = key.rfind('/');
if (pos == std::string::npos || pos + 1 >= key.size()) {
continue;
}
uint64_t seq = 0;
try {
seq = static_cast<uint64_t>(std::stoull(key.substr(pos + 1)));
} catch (...) {
continue;
}
if (IsSequenceOlderOrEqual(seq, start_sequence_id)) {
continue;
}
OpLogEntry entry;
const std::string value = kv.get("value", "").asString();
if (!DeserializeOpLogEntry(value, entry)) {
LOG(ERROR) << "Failed to deserialize OpLog entry from key="
<< key;
return ErrorCode::INTERNAL_ERROR;
}
entries.push_back(std::move(entry));
if (entries.size() >= limit) {
break;
}
}
// Advance start key for next page.
if (last_key_in_page.empty()) {
break;
}
current_start_key = last_key_in_page;
current_start_key.push_back('\0');
}
return ErrorCode::OK;
}
ErrorCode EtcdOpLogStore::GetLatestSequenceId(uint64_t& sequence_id) {
std::string key = BuildLatestKey();
std::string value;
EtcdRevisionId revision_id;
ErrorCode err =
EtcdHelper::Get(key.c_str(), key.size(), value, revision_id);
if (err != ErrorCode::OK) {
return err;
}
try {
sequence_id = std::stoull(value);
} catch (const std::exception& e) {
LOG(ERROR) << "Failed to parse latest sequence_id: " << e.what();
return ErrorCode::INTERNAL_ERROR;
}
return ErrorCode::OK;
}
ErrorCode EtcdOpLogStore::GetMaxSequenceId(uint64_t& sequence_id) {
auto max_seq_opt = GetMaxSequenceIdInternal();
if (!max_seq_opt.has_value()) {
return ErrorCode::ETCD_KEY_NOT_EXIST;
}
sequence_id = max_seq_opt.value();
return ErrorCode::OK;
}
ErrorCode EtcdOpLogStore::UpdateLatestSequenceId(uint64_t sequence_id) {
std::string key = BuildLatestKey();
std::string value = std::to_string(sequence_id);
return EtcdHelper::Put(key.c_str(), key.size(), value.c_str(),
value.size());
}
ErrorCode EtcdOpLogStore::RecordSnapshotSequenceId(
const std::string& snapshot_id, uint64_t sequence_id) {
std::string key = BuildSnapshotKey(snapshot_id);
std::string value = std::to_string(sequence_id);
return EtcdHelper::Put(key.c_str(), key.size(), value.c_str(),
value.size());
}
ErrorCode EtcdOpLogStore::GetSnapshotSequenceId(const std::string& snapshot_id,
uint64_t& sequence_id) {
std::string key = BuildSnapshotKey(snapshot_id);
std::string value;
EtcdRevisionId revision_id;
ErrorCode err =
EtcdHelper::Get(key.c_str(), key.size(), value, revision_id);
if (err != ErrorCode::OK) {
return err;
}
try {
sequence_id = std::stoull(value);
} catch (const std::exception& e) {
LOG(ERROR) << "Failed to parse snapshot sequence_id: " << e.what();
return ErrorCode::INTERNAL_ERROR;
}
return ErrorCode::OK;
}
ErrorCode EtcdOpLogStore::CleanupOpLogBefore(uint64_t before_sequence_id) {
// Robust cleanup (Scheme 3):
// - Determine current minimum sequence_id in etcd
// - DeleteRange [min_key, before_key)
//
// IMPORTANT: This relies on lexicographical ordering of keys, so the
// sequence_id portion MUST be fixed-width (zero-padded).
auto min_seq_opt = GetMinSequenceId();
if (!min_seq_opt.has_value()) {
return ErrorCode::OK; // nothing to cleanup
}
uint64_t min_seq = min_seq_opt.value();
if (before_sequence_id <= min_seq) {
return ErrorCode::OK;
}
std::string start_key = BuildOpLogKey(min_seq);
std::string end_key =
BuildOpLogKey(before_sequence_id); // delete < before_sequence_id
return EtcdHelper::DeleteRange(start_key.c_str(), start_key.size(),
end_key.c_str(), end_key.size());
}
std::string EtcdOpLogStore::BuildOpLogKey(uint64_t sequence_id) const {
std::ostringstream oss;
// Fixed-width encoding for correct etcd lexicographical range operations.
// 20 digits is enough for uint64_t max (18446744073709551615).
oss << kOpLogPrefix << cluster_id_ << "/" << std::setw(20)
<< std::setfill('0') << sequence_id;
return oss.str();
}
std::optional<uint64_t> EtcdOpLogStore::GetMinSequenceId() const {
std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/";
std::string first_key;
ErrorCode err = EtcdHelper::GetFirstKeyWithPrefix(prefix.c_str(),
prefix.size(), first_key);
if (err != ErrorCode::OK) {
return std::nullopt;
}
// Skip non-entry keys if any (e.g. "/latest" or "/snapshot/...").
// Entries are expected to be ".../<20-digit-seq>".
// If the first key isn't an entry key, fall back to nullopt (safe no-op).
if (first_key.find("/latest") != std::string::npos ||
first_key.find("/snapshot/") != std::string::npos) {
return std::nullopt;
}
size_t pos = first_key.rfind('/');
if (pos == std::string::npos || pos + 1 >= first_key.size()) {
return std::nullopt;
}
std::string seq_str = first_key.substr(pos + 1);
try {
return static_cast<uint64_t>(std::stoull(seq_str));
} catch (...) {
return std::nullopt;
}
}
std::optional<uint64_t> EtcdOpLogStore::GetMaxSequenceIdInternal() const {
// Entry keys are fixed-width 20-digit numbers, which (in practice) start
// with '0'. Use "/0" to avoid picking up "/latest" which is
// lexicographically after digits.
std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/0";
std::string last_key;
ErrorCode err = EtcdHelper::GetLastKeyWithPrefix(prefix.c_str(),
prefix.size(), last_key);
if (err != ErrorCode::OK) {
return std::nullopt;
}
size_t pos = last_key.rfind('/');
if (pos == std::string::npos || pos + 1 >= last_key.size()) {
return std::nullopt;
}
std::string seq_str = last_key.substr(pos + 1);
try {
return static_cast<uint64_t>(std::stoull(seq_str));
} catch (...) {
return std::nullopt;
}
}
std::string EtcdOpLogStore::BuildLatestKey() const {
std::ostringstream oss;
oss << kOpLogPrefix << cluster_id_ << kLatestSuffix;
return oss.str();
}
std::string EtcdOpLogStore::BuildSnapshotKey(
const std::string& snapshot_id) const {
std::ostringstream oss;
oss << kOpLogPrefix << cluster_id_ << kSnapshotSuffix << snapshot_id
<< "/sequence_id";
return oss.str();
}
std::string EtcdOpLogStore::SerializeOpLogEntry(const OpLogEntry& entry) const {
Json::Value root;
root["sequence_id"] = static_cast<Json::UInt64>(entry.sequence_id);
root["timestamp_ms"] = static_cast<Json::UInt64>(entry.timestamp_ms);
root["op_type"] = static_cast<int>(entry.op_type);
root["object_key"] = entry.object_key;
// CRITICAL: Base64 encode binary payload to prevent UTF-8 corruption in
// JSON
root["payload"] = base64::Encode(entry.payload);
root["checksum"] = static_cast<Json::UInt>(entry.checksum);
root["prefix_hash"] = static_cast<Json::UInt>(entry.prefix_hash);
Json::StreamWriterBuilder builder;
builder["indentation"] = ""; // Compact format
std::unique_ptr<Json::StreamWriter> writer(builder.newStreamWriter());
std::ostringstream oss;
writer->write(root, &oss);
return oss.str();
}
bool EtcdOpLogStore::DeserializeOpLogEntry(const std::string& json_str,
OpLogEntry& entry) const {
Json::Value root;
Json::CharReaderBuilder builder;
std::unique_ptr<Json::CharReader> reader(builder.newCharReader());
std::string errors;
if (!reader->parse(json_str.data(), json_str.data() + json_str.size(),
&root, &errors)) {
LOG(ERROR) << "Failed to parse JSON: " << errors;
return false;
}
try {
entry.sequence_id = root["sequence_id"].asUInt64();
entry.timestamp_ms = root["timestamp_ms"].asUInt64();
entry.op_type = static_cast<OpType>(root["op_type"].asInt());
entry.object_key = root["object_key"].asString();
// CRITICAL: Base64 decode payload to restore binary data
entry.payload = base64::Decode(root["payload"].asString());
entry.checksum = root["checksum"].asUInt();
entry.prefix_hash = root["prefix_hash"].asUInt();
} catch (const std::exception& e) {
LOG(ERROR) << "Failed to deserialize OpLogEntry: " << e.what();
return false;
}
std::string size_reason;
if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) {
LOG(ERROR) << "EtcdOpLogStore: entry size rejected, sequence_id="
<< entry.sequence_id << ", key=" << entry.object_key
<< ", reason=" << size_reason;
return false;
}
return true;
}
void EtcdOpLogStore::BatchUpdateThread() {
if (!enable_latest_seq_batch_update_) {
return;
}
while (batch_update_running_.load()) {
std::this_thread::sleep_for(
std::chrono::milliseconds(kBatchIntervalMs));
// Check if we need to update based on time interval
auto now = std::chrono::steady_clock::now();
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
now - last_update_time_)
.count();
if (pending_count_.load() > 0 && elapsed >= kBatchIntervalMs) {
DoBatchUpdate();
}
}
}
void EtcdOpLogStore::TriggerBatchUpdateIfNeeded() {
// This method is kept for potential future use (e.g., manual trigger)
// Currently, DoBatchUpdate() is called directly from WriteOpLog
// when batch size threshold is reached
if (pending_count_.load() >= kBatchSize) {
DoBatchUpdate();
}
}
void EtcdOpLogStore::DoBatchUpdate() {
if (!enable_latest_seq_batch_update_) {
return;
}
std::lock_guard<std::mutex> lock(batch_update_mutex_);
// Get the pending sequence_id and reset counters
uint64_t seq_id_to_update = pending_latest_seq_id_.load();
size_t count = pending_count_.exchange(0);
if (count == 0) {
return; // Nothing to update
}
// Update latest_sequence_id in etcd
ErrorCode err = UpdateLatestSequenceId(seq_id_to_update);
if (err != ErrorCode::OK) {
LOG(WARNING) << "Failed to batch update latest_sequence_id="
<< seq_id_to_update << ", error=" << err
<< ". Will retry in next batch.";
// Restore the count so it will be retried
pending_count_.fetch_add(count);
} else {
last_update_time_ = std::chrono::steady_clock::now();
VLOG(2) << "Batch updated latest_sequence_id=" << seq_id_to_update
<< " (count=" << count << " entries)";
}
}
} // namespace mooncake

314
mooncake-store/src/ha_metric_manager.cpp Normal file
View File

@ -0,0 +1,314 @@
#include "ha_metric_manager.h"
#include <glog/logging.h>
#include <iomanip>
#include <sstream>
namespace mooncake {
// --- Singleton Instance ---
HAMetricManager& HAMetricManager::instance() {
static HAMetricManager static_instance;
return static_instance;
}
// --- Constructor ---
HAMetricManager::HAMetricManager()
// OpLog Sequence Gauges
: oplog_last_sequence_id_("ha_oplog_last_sequence_id",
"Latest OpLog sequence ID written by Primary"),
oplog_applied_sequence_id_("ha_oplog_applied_sequence_id",
"Latest OpLog sequence ID applied by Standby"),
oplog_standby_lag_("ha_oplog_standby_lag",
"Number of OpLog entries Standby is behind Primary"),
oplog_pending_entries_(
"ha_oplog_pending_entries",
"Number of out-of-order entries waiting in OpLogApplier"),
pending_mutation_queue_size_(
"ha_pending_mutation_queue_size",
"Number of mutations pending etcd write retry"),
// Error Counters
oplog_skipped_entries_total_(
"ha_oplog_skipped_entries_total",
"Total number of OpLog entries skipped due to timeout"),
oplog_checksum_failures_total_(
"ha_oplog_checksum_failures_total",
"Total number of OpLog entries with checksum verification failures"),
oplog_gap_resolve_attempts_total_(
"ha_oplog_gap_resolve_attempts_total",
"Total number of attempts to resolve missing OpLog entries"),
oplog_gap_resolve_success_total_(
"ha_oplog_gap_resolve_success_total",
"Total number of successfully resolved missing OpLog entries"),
oplog_etcd_write_failures_total_(
"ha_oplog_etcd_write_failures_total",
"Total number of failed etcd write operations"),
oplog_etcd_write_retries_total_(
"ha_oplog_etcd_write_retries_total",
"Total number of etcd write retry attempts"),
oplog_watch_disconnections_total_(
"ha_oplog_watch_disconnections_total",
"Total number of OpLog watch disconnections"),
oplog_applied_entries_total_(
"ha_oplog_applied_entries_total",
"Total number of OpLog entries successfully applied"),
oplog_dropped_put_end_total_("ha_oplog_dropped_put_end_total",
"Total number of dropped PUT_END operations "
"due to late arrival after "
"skip"),
oplog_batch_commits_total_(
"ha_oplog_batch_commits_total",
"Total number of Group Commit batches flushed to etcd"),
oplog_sync_batch_commits_total_(
"ha_oplog_sync_batch_commits_total",
"Total number of sync batches (triggered by DELETE/Sync ops)"),
// Latency Histograms (buckets in microseconds)
// 100us, 500us, 1ms, 5ms, 10ms, 50ms, 100ms, 500ms, 1s, 5s
oplog_etcd_write_latency_us_(
"ha_oplog_etcd_write_latency_us",
"Latency of etcd write operations in microseconds",
{100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000,
5000000}),
oplog_apply_latency_us_(
"ha_oplog_apply_latency_us",
"Latency of OpLog entry application in microseconds",
{10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000}),
// State Machine
standby_state_(
"ha_standby_state",
"Current state of the Standby service (0=STOPPED, 1=CONNECTING, "
"2=SYNCING, 3=WATCHING, 4=RECOVERING, 5=RECONNECTING, "
"6=PROMOTING, 7=PROMOTED, 8=FAILED)"),
state_transitions_total_(
"ha_state_transitions_total",
"Total number of Standby state machine transitions") {
// Initialize gauges to 0 for proper Prometheus output
oplog_last_sequence_id_.update(0);
oplog_applied_sequence_id_.update(0);
oplog_standby_lag_.update(0);
oplog_pending_entries_.update(0);
pending_mutation_queue_size_.update(0);
standby_state_.update(0);
}
// ========== OpLog Sequence Metrics (Gauge) ==========
void HAMetricManager::set_oplog_last_sequence_id(int64_t seq_id) {
oplog_last_sequence_id_.update(seq_id);
}
int64_t HAMetricManager::get_oplog_last_sequence_id() {
return static_cast<int64_t>(oplog_last_sequence_id_.value());
}
void HAMetricManager::set_oplog_applied_sequence_id(int64_t seq_id) {
oplog_applied_sequence_id_.update(seq_id);
}
int64_t HAMetricManager::get_oplog_applied_sequence_id() {
return static_cast<int64_t>(oplog_applied_sequence_id_.value());
}
void HAMetricManager::set_oplog_standby_lag(int64_t lag) {
oplog_standby_lag_.update(lag);
}
int64_t HAMetricManager::get_oplog_standby_lag() {
return static_cast<int64_t>(oplog_standby_lag_.value());
}
void HAMetricManager::set_oplog_pending_entries(int64_t count) {
oplog_pending_entries_.update(count);
}
int64_t HAMetricManager::get_oplog_pending_entries() {
return static_cast<int64_t>(oplog_pending_entries_.value());
}
void HAMetricManager::set_pending_mutation_queue_size(int64_t size) {
pending_mutation_queue_size_.update(size);
}
int64_t HAMetricManager::get_pending_mutation_queue_size() {
return static_cast<int64_t>(pending_mutation_queue_size_.value());
}
// ========== Error Counters ==========
void HAMetricManager::inc_oplog_skipped_entries(int64_t val) {
oplog_skipped_entries_total_.inc(val);
}
int64_t HAMetricManager::get_oplog_skipped_entries_total() {
return static_cast<int64_t>(oplog_skipped_entries_total_.value());
}
void HAMetricManager::inc_oplog_checksum_failures(int64_t val) {
oplog_checksum_failures_total_.inc(val);
}
int64_t HAMetricManager::get_oplog_checksum_failures_total() {
return static_cast<int64_t>(oplog_checksum_failures_total_.value());
}
void HAMetricManager::inc_oplog_gap_resolve_attempts(int64_t val) {
oplog_gap_resolve_attempts_total_.inc(val);
}
int64_t HAMetricManager::get_oplog_gap_resolve_attempts_total() {
return static_cast<int64_t>(oplog_gap_resolve_attempts_total_.value());
}
void HAMetricManager::inc_oplog_gap_resolve_success(int64_t val) {
oplog_gap_resolve_success_total_.inc(val);
}
int64_t HAMetricManager::get_oplog_gap_resolve_success_total() {
return static_cast<int64_t>(oplog_gap_resolve_success_total_.value());
}
void HAMetricManager::inc_oplog_etcd_write_failures(int64_t val) {
oplog_etcd_write_failures_total_.inc(val);
}
int64_t HAMetricManager::get_oplog_etcd_write_failures_total() {
return static_cast<int64_t>(oplog_etcd_write_failures_total_.value());
}
void HAMetricManager::inc_oplog_etcd_write_retries(int64_t val) {
oplog_etcd_write_retries_total_.inc(val);
}
int64_t HAMetricManager::get_oplog_etcd_write_retries_total() {
return static_cast<int64_t>(oplog_etcd_write_retries_total_.value());
}
void HAMetricManager::inc_oplog_watch_disconnections(int64_t val) {
oplog_watch_disconnections_total_.inc(val);
}
int64_t HAMetricManager::get_oplog_watch_disconnections_total() {
return static_cast<int64_t>(oplog_watch_disconnections_total_.value());
}
void HAMetricManager::inc_oplog_applied_entries(int64_t val) {
oplog_applied_entries_total_.inc(val);
}
int64_t HAMetricManager::get_oplog_applied_entries_total() {
return static_cast<int64_t>(oplog_applied_entries_total_.value());
}
void HAMetricManager::inc_oplog_dropped_put_end(int64_t val) {
oplog_dropped_put_end_total_.inc(val);
}
int64_t HAMetricManager::get_oplog_dropped_put_end_total() {
return static_cast<int64_t>(oplog_dropped_put_end_total_.value());
}
void HAMetricManager::inc_oplog_batch_commits(int64_t val) {
oplog_batch_commits_total_.inc(val);
}
void HAMetricManager::inc_oplog_sync_batch_commits(int64_t val) {
oplog_sync_batch_commits_total_.inc(val);
}
int64_t HAMetricManager::get_oplog_batch_commits_total() {
return static_cast<int64_t>(oplog_batch_commits_total_.value());
}
int64_t HAMetricManager::get_oplog_sync_batch_commits_total() {
return static_cast<int64_t>(oplog_sync_batch_commits_total_.value());
}
// ========== Latency Histograms ==========
void HAMetricManager::observe_oplog_etcd_write_latency_us(int64_t latency_us) {
oplog_etcd_write_latency_us_.observe(latency_us);
}
void HAMetricManager::observe_oplog_apply_latency_us(int64_t latency_us) {
oplog_apply_latency_us_.observe(latency_us);
}
// ========== State Machine Metrics ==========
void HAMetricManager::set_standby_state(int64_t state_value) {
standby_state_.update(state_value);
}
int64_t HAMetricManager::get_standby_state() {
return static_cast<int64_t>(standby_state_.value());
}
void HAMetricManager::inc_state_transitions(int64_t val) {
state_transitions_total_.inc(val);
}
int64_t HAMetricManager::get_state_transitions_total() {
return static_cast<int64_t>(state_transitions_total_.value());
}
// ========== Serialization ==========
std::string HAMetricManager::serialize_metrics() {
std::stringstream ss;
// Helper lambda to serialize a metric
auto serialize_metric = [&ss](auto& metric) {
std::string metric_str;
metric.serialize(metric_str);
ss << metric_str;
};
// Gauges
serialize_metric(oplog_last_sequence_id_);
serialize_metric(oplog_applied_sequence_id_);
serialize_metric(oplog_standby_lag_);
serialize_metric(oplog_pending_entries_);
serialize_metric(pending_mutation_queue_size_);
serialize_metric(standby_state_);
// Counters
serialize_metric(oplog_skipped_entries_total_);
serialize_metric(oplog_checksum_failures_total_);
serialize_metric(oplog_gap_resolve_attempts_total_);
serialize_metric(oplog_gap_resolve_success_total_);
serialize_metric(oplog_etcd_write_failures_total_);
serialize_metric(oplog_etcd_write_retries_total_);
serialize_metric(oplog_watch_disconnections_total_);
serialize_metric(oplog_applied_entries_total_);
serialize_metric(state_transitions_total_);
// Histograms
serialize_metric(oplog_etcd_write_latency_us_);
serialize_metric(oplog_apply_latency_us_);
return ss.str();
}
std::string HAMetricManager::get_summary_string() {
std::stringstream ss;
ss << "HA Metrics Summary: ";
ss << "last_seq=" << get_oplog_last_sequence_id();
ss << ", applied_seq=" << get_oplog_applied_sequence_id();
ss << ", lag=" << get_oplog_standby_lag();
ss << ", pending=" << get_oplog_pending_entries();
ss << ", mutation_queue=" << get_pending_mutation_queue_size();
ss << ", batch_commits=" << get_oplog_batch_commits_total();
ss << ", sync_commits=" << get_oplog_sync_batch_commits_total();
ss << ", skipped=" << get_oplog_skipped_entries_total();
ss << ", checksum_fail=" << get_oplog_checksum_failures_total();
ss << ", etcd_fail=" << get_oplog_etcd_write_failures_total();
ss << ", watch_disconn=" << get_oplog_watch_disconnections_total();
ss << ", state=" << get_standby_state();
return ss.str();
}
} // namespace mooncake

686
mooncake-store/src/hot_standby_service.cpp Normal file
View File

@ -0,0 +1,686 @@
#include "hot_standby_service.h"
#include <glog/logging.h>
#include <chrono>
#include <thread>
#include "etcd_helper.h"
#include "etcd_oplog_store.h"
#include "ha_metric_manager.h"
#include "master_service.h"
#include "oplog_applier.h"
#include "oplog_manager.h"
#include "oplog_watcher.h"
namespace mooncake {
HotStandbyService::HotStandbyService(const HotStandbyConfig& config)
: config_(config) {
// Explicitly initialize HA metric manager to ensure thread safety
// during metric registration.
HAMetricManager::Init();
metadata_store_ = std::make_unique<StandbyMetadataStore>();
// OpLogApplier will be re-created in Start() with the resolved cluster_id
// to enable etcd-based operations (e.g. requesting missing OpLog entries).
// Here we construct a minimal instance so that local metadata operations
// are available before etcd wiring is completed.
oplog_applier_ = std::make_unique<OpLogApplier>(metadata_store_.get());
// Register callback for state change logging and metrics.
state_machine_.RegisterCallback(
[](StandbyState old_state, StandbyState new_state, StandbyEvent event) {
LOG(INFO) << "HotStandbyService state changed: "
<< StandbyStateToString(old_state) << " -> "
<< StandbyStateToString(new_state)
<< " (event: " << StandbyEventToString(event) << ")";
// Update HA metrics
HAMetricManager::instance().set_standby_state(
static_cast<int64_t>(new_state));
HAMetricManager::instance().inc_state_transitions();
// Track watch disconnections
if (event == StandbyEvent::WATCH_BROKEN ||
event == StandbyEvent::DISCONNECTED) {
HAMetricManager::instance().inc_oplog_watch_disconnections();
}
});
}
// StandbyMetadataStore implementation
bool HotStandbyService::StandbyMetadataStore::PutMetadata(
const std::string& key, const StandbyObjectMetadata& metadata) {
std::lock_guard<std::mutex> lock(mutex_);
store_[key] = metadata;
VLOG(2) << "StandbyMetadataStore: stored metadata for key=" << key
<< ", replicas=" << metadata.replicas.size()
<< ", size=" << metadata.size;
return true;
}
bool HotStandbyService::StandbyMetadataStore::Put(const std::string& key,
const std::string& payload) {
// Legacy interface - create empty metadata
StandbyObjectMetadata metadata;
std::lock_guard<std::mutex> lock(mutex_);
store_[key] = metadata;
return true;
}
std::optional<StandbyObjectMetadata>
HotStandbyService::StandbyMetadataStore::GetMetadata(
const std::string& key) const {
std::lock_guard<std::mutex> lock(mutex_);
auto it = store_.find(key);
if (it != store_.end()) {
return it->second;
}
return std::nullopt;
}
bool HotStandbyService::StandbyMetadataStore::Remove(const std::string& key) {
std::lock_guard<std::mutex> lock(mutex_);
auto it = store_.find(key);
if (it != store_.end()) {
store_.erase(it);
return true;
}
return false;
}
bool HotStandbyService::StandbyMetadataStore::Exists(
const std::string& key) const {
std::lock_guard<std::mutex> lock(mutex_);
return store_.find(key) != store_.end();
}
size_t HotStandbyService::StandbyMetadataStore::GetKeyCount() const {
std::lock_guard<std::mutex> lock(mutex_);
return store_.size();
}
void HotStandbyService::StandbyMetadataStore::Snapshot(
std::vector<std::pair<std::string, StandbyObjectMetadata>>& out) const {
std::lock_guard<std::mutex> lock(mutex_);
out.clear();
out.reserve(store_.size());
for (const auto& kv : store_) {
out.emplace_back(kv.first, kv.second);
}
}
HotStandbyService::~HotStandbyService() {
// Always ensure threads are joined, regardless of state
// This prevents std::terminate() if threads are still joinable
Stop();
// Double-check: ensure all threads are joined even if Stop() had early
// return
if (replication_thread_.joinable()) {
replication_thread_.join();
}
if (verification_thread_.joinable()) {
verification_thread_.join();
}
}
ErrorCode HotStandbyService::Start(const std::string& primary_address,
const std::string& etcd_endpoints,
const std::string& cluster_id) {
std::lock_guard<std::mutex> lock(mutex_);
// Use state machine to check if already running
if (IsRunning()) {
LOG(WARNING) << "HotStandbyService is already running";
return ErrorCode::OK;
}
// Trigger START event
auto result = state_machine_.ProcessEvent(StandbyEvent::START);
if (!result.allowed) {
LOG(ERROR) << "Cannot start HotStandbyService: " << result.reason;
return ErrorCode::INTERNAL_ERROR; // State machine rejected START
}
config_.primary_address = primary_address;
etcd_endpoints_ = etcd_endpoints;
cluster_id_ = cluster_id;
#ifdef STORE_USE_ETCD
// Connect to etcd
ErrorCode err =
EtcdHelper::ConnectToEtcdStoreClient(etcd_endpoints.c_str());
if (err != ErrorCode::OK) {
LOG(ERROR) << "Failed to connect to etcd: " << etcd_endpoints;
state_machine_.ProcessEvent(StandbyEvent::CONNECTION_FAILED);
return err;
}
// Transition to SYNCING state
state_machine_.ProcessEvent(StandbyEvent::CONNECTED);
// Preserve existing local state if HotStandbyService is restarted
// in-process:
// - metadata_store_ may already contain real-time metadata
// - oplog_applier_ may already have expected_sequence_id_
uint64_t local_last_seq_id = 0;
if (oplog_applier_) {
uint64_t expected = oplog_applier_->GetExpectedSequenceId();
local_last_seq_id = expected > 0 ? expected - 1 : 0;
}
const bool has_local_metadata =
metadata_store_ && metadata_store_->GetKeyCount() > 0;
const bool has_local_state = has_local_metadata && local_last_seq_id > 0;
// Recreate OpLogApplier with cluster_id (for requesting missing OpLog).
// If we had local state, recover to keep sequence continuity.
oplog_applier_ =
std::make_unique<OpLogApplier>(metadata_store_.get(), cluster_id);
if (has_local_state) {
LOG(INFO) << "Standby warm start: reuse local metadata (keys="
<< metadata_store_->GetKeyCount()
<< "), recover last_seq_id=" << local_last_seq_id;
oplog_applier_->Recover(local_last_seq_id);
}
// Create OpLogWatcher with state machine callback
oplog_watcher_ = std::make_unique<OpLogWatcher>(etcd_endpoints, cluster_id,
oplog_applier_.get());
// Register callback for watcher events
oplog_watcher_->SetStateCallback(
[this](StandbyEvent event) { OnWatcherEvent(event); });
// Bootstrap:
// - If we already have local state (warm start), do NOT reload snapshot.
// - Otherwise (cold start/new standby), try snapshot (if enabled) then
// replay OpLog.
uint64_t baseline_seq_id = has_local_state ? local_last_seq_id : 0;
if (!has_local_state && config_.enable_snapshot_bootstrap &&
snapshot_provider_) {
std::string snapshot_id;
uint64_t snapshot_seq_id = 0;
std::vector<std::pair<std::string, StandbyObjectMetadata>> snapshot;
if (snapshot_provider_->LoadLatestSnapshot(cluster_id_, snapshot_id,
snapshot_seq_id, snapshot)) {
LOG(INFO) << "Loaded snapshot: snapshot_id=" << snapshot_id
<< ", snapshot_seq_id=" << snapshot_seq_id
<< ", keys=" << snapshot.size();
// Apply snapshot into local standby store.
for (const auto& kv : snapshot) {
metadata_store_->PutMetadata(kv.first, kv.second);
}
// Align applier to snapshot boundary.
oplog_applier_->Recover(snapshot_seq_id);
baseline_seq_id = snapshot_seq_id;
} else {
LOG(INFO) << "No snapshot available (or provider not ready), "
"falling back to OpLog-only bootstrap";
}
}
// Read historical OpLog entries since baseline_seq_id.
uint64_t last_applied_seq_id = baseline_seq_id;
// Start OpLogWatcher with a consistent "read then watch(from revision+1)"
// sequence. Retry with exponential backoff to avoid getting stuck in
// RECONNECTING state where nothing drives recovery.
static constexpr int kMaxStartRetries = 3;
static constexpr int kStartRetryBaseMs = 500;
bool watcher_started = false;
for (int attempt = 0; attempt < kMaxStartRetries; ++attempt) {
if (oplog_watcher_->StartFromSequenceId(last_applied_seq_id)) {
watcher_started = true;
break;
}
LOG(WARNING) << "Failed to start OpLogWatcher from sequence_id="
<< last_applied_seq_id << " (attempt " << (attempt + 1)
<< "/" << kMaxStartRetries << ")";
if (attempt + 1 < kMaxStartRetries) {
std::this_thread::sleep_for(
std::chrono::milliseconds(kStartRetryBaseMs * (1 << attempt)));
}
}
if (!watcher_started) {
LOG(ERROR) << "Failed to start OpLogWatcher after " << kMaxStartRetries
<< " attempts, aborting Start()";
state_machine_.ProcessEvent(StandbyEvent::FATAL_ERROR);
return ErrorCode::INTERNAL_ERROR;
}
// Transition to WATCHING state after successful sync
state_machine_.ProcessEvent(StandbyEvent::SYNC_COMPLETE);
// Start background threads
replication_thread_ =
std::thread(&HotStandbyService::ReplicationLoop, this);
if (config_.enable_verification) {
verification_thread_ =
std::thread(&HotStandbyService::VerificationLoop, this);
}
LOG(INFO) << "HotStandbyService started, watching etcd OpLog for cluster: "
<< cluster_id << ", state=" << StandbyStateToString(GetState());
return ErrorCode::OK;
#else
state_machine_.ProcessEvent(StandbyEvent::FATAL_ERROR);
LOG(ERROR)
<< "STORE_USE_ETCD is not enabled, cannot start HotStandbyService";
return ErrorCode::INTERNAL_ERROR;
#endif
}
void HotStandbyService::OnWatcherEvent(StandbyEvent event) {
state_machine_.ProcessEvent(event);
}
void HotStandbyService::Stop() {
std::lock_guard<std::mutex> lock(mutex_);
// Check if already stopped (to avoid duplicate processing)
bool was_running = IsRunning();
StandbyState current_state = GetState();
if (!was_running && current_state != StandbyState::PROMOTING &&
!replication_thread_.joinable() && !verification_thread_.joinable()) {
// Already fully stopped and threads are joined
return;
}
// Trigger STOP event
state_machine_.ProcessEvent(StandbyEvent::STOP);
// Stop OpLogWatcher
if (oplog_watcher_) {
oplog_watcher_->Stop();
oplog_watcher_.reset();
}
// Wait for threads to finish
if (replication_thread_.joinable()) {
replication_thread_.join();
}
if (verification_thread_.joinable()) {
verification_thread_.join();
}
LOG(INFO) << "HotStandbyService stopped, final_state="
<< StandbyStateToString(GetState());
}
StandbySyncStatus HotStandbyService::GetSyncStatus() const {
StandbySyncStatus status;
// Get applied sequence ID from OpLogApplier
if (oplog_applier_) {
uint64_t expected = oplog_applier_->GetExpectedSequenceId();
status.applied_seq_id = (expected > 0) ? (expected - 1) : 0;
if (status.applied_seq_id == 0) {
status.applied_seq_id = applied_seq_id_.load(); // Fallback
}
} else {
status.applied_seq_id = applied_seq_id_.load();
}
// Primary sequence ID (best-effort): updated by ReplicationLoop via etcd
// `/latest`.
status.primary_seq_id = primary_seq_id_.load();
// Use state machine for connection status
status.is_connected = IsConnected();
status.state = GetState();
status.time_in_state = state_machine_.GetTimeInCurrentState();
if (status.primary_seq_id > status.applied_seq_id) {
status.lag_entries = status.primary_seq_id - status.applied_seq_id;
} else {
status.lag_entries = 0;
}
// Lag time is currently reported as 0; if needed we can extend the
// protocol to propagate primary timestamps and compute a real value.
status.lag_time = std::chrono::milliseconds(0);
status.is_syncing = IsRunning() && IsConnected();
return status;
}
bool HotStandbyService::IsReadyForPromotion() const {
// Use state machine to check if ready for promotion
if (!state_machine_.IsReadyForPromotion()) {
return false;
}
StandbySyncStatus status = GetSyncStatus();
// Allow promotion even with large lag - the new Primary can continue
// syncing remaining OpLog entries from etcd after promotion.
// Log a warning if lag is large, but don't block promotion.
if (status.lag_entries > config_.max_replication_lag_entries) {
LOG(WARNING)
<< "Standby has large replication lag: " << status.lag_entries
<< " entries (threshold: " << config_.max_replication_lag_entries
<< "). Promotion will proceed, but remaining OpLog entries "
<< "will be synced after promotion.";
}
return true;
}
ErrorCode HotStandbyService::Promote() {
std::unique_lock<std::mutex> lock(mutex_);
if (!IsReadyForPromotion()) {
LOG(ERROR) << "Standby is not ready for promotion, state="
<< StandbyStateToString(GetState());
return ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS;
}
// Trigger PROMOTE event
auto result = state_machine_.ProcessEvent(StandbyEvent::PROMOTE);
if (!result.allowed) {
LOG(ERROR) << "Cannot promote: " << result.reason;
return ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS;
}
StandbySyncStatus status = GetSyncStatus();
uint64_t current_applied_seq_id = status.applied_seq_id;
LOG(INFO) << "Promoting Standby to Primary. Applied seq_id: "
<< current_applied_seq_id << ", lag: " << status.lag_entries
<< " entries"
<< ", state: " << StandbyStateToString(GetState());
// Final catch-up sync before promotion.
// IMPORTANT:
// - Do NOT rely on `lag_entries` here because primary_seq_id_ is
// best-effort.
// - Stop OpLogWatcher first to avoid concurrent Apply from watch callbacks.
if (oplog_watcher_) {
oplog_watcher_->Stop();
}
// Best-effort: resolve any outstanding gaps with retry before promotion.
// Do NOT block promotion if gaps cannot be fetched after max retries.
static constexpr int kMaxGapResolveRetries = 3;
if (oplog_applier_) {
for (int retry = 0; retry < kMaxGapResolveRetries; ++retry) {
auto res = oplog_applier_->TryResolveGapsOnceForPromotion(
/*max_ids=*/1024);
if (res.attempted == 0) {
// No gaps to resolve
break;
}
LOG(INFO) << "Promotion gap resolve (attempt " << (retry + 1) << "/"
<< kMaxGapResolveRetries
<< "): attempted=" << res.attempted
<< ", fetched=" << res.fetched
<< ", applied_deletes=" << res.applied_deletes;
if (res.fetched == res.attempted) {
// All gaps resolved successfully
break;
}
// Some gaps failed, retry after short delay
if (retry + 1 < kMaxGapResolveRetries) {
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
}
}
LOG(INFO) << "Final catch-up sync from etcd before promotion...";
EtcdOpLogStore oplog_store(cluster_id_,
/*enable_latest_seq_batch_update=*/false);
if (oplog_store.Init() != ErrorCode::OK) {
LOG(ERROR) << "Failed to initialize oplog_store for final catch-up";
return ErrorCode::ETCD_OPERATION_ERROR;
}
const size_t batch_size = 1000;
// P0 fix: Prevent underflow when current_applied_seq_id is 0
// ReadOpLogSince reads entries with seq > given_seq, so we pass
// current_applied_seq_id directly
uint64_t read_from_seq =
current_applied_seq_id; // Will read entries with seq > read_from_seq
// P1 fix: Add timeout control to prevent infinite blocking
static constexpr size_t kMaxCatchUpBatches =
100; // Max 100 batches * 1000 = 100k entries
static constexpr auto kMaxCatchUpDuration = std::chrono::seconds(30);
auto catch_up_start = std::chrono::steady_clock::now();
size_t total_applied = 0;
size_t batch_count = 0;
for (;;) {
// Check timeout
auto elapsed = std::chrono::steady_clock::now() - catch_up_start;
if (elapsed > kMaxCatchUpDuration) {
LOG(WARNING) << "Final catch-up: timeout after "
<< std::chrono::duration_cast<std::chrono::seconds>(
elapsed)
.count()
<< "s. Proceeding with promotion. total_applied="
<< total_applied;
break;
}
// Check batch limit
if (batch_count >= kMaxCatchUpBatches) {
LOG(WARNING) << "Final catch-up: reached max batch limit ("
<< kMaxCatchUpBatches
<< "). Proceeding with promotion. total_applied="
<< total_applied;
break;
}
std::vector<OpLogEntry> batch;
ErrorCode read_err =
oplog_store.ReadOpLogSince(read_from_seq, batch_size, batch);
if (read_err != ErrorCode::OK) {
LOG(WARNING) << "Final catch-up: failed to read OpLog since seq="
<< read_from_seq
<< ", err=" << static_cast<int>(read_err)
<< ". Proceeding with promotion.";
break;
}
if (batch.empty()) {
break;
}
size_t applied = oplog_applier_->ApplyOpLogEntries(batch);
total_applied += applied;
read_from_seq =
batch.back().sequence_id; // Next read will get entries > this seq
++batch_count;
}
LOG(INFO) << "Final catch-up sync done. total_applied=" << total_applied
<< ", batches=" << batch_count;
// Transition to PROMOTED state
state_machine_.ProcessEvent(StandbyEvent::PROMOTION_SUCCESS);
// Stop replication (OpLogWatcher will stop watching).
// Note: This will trigger STOP event, transitioning to STOPPED.
lock.unlock();
Stop();
// Design note: MasterService creation and initialization are handled by
// MasterServiceSupervisor::Start() after leader election. The
// responsibility of HotStandbyService::Promote() is limited to ensuring
// that all remaining OpLog entries are applied before the new Primary
// starts serving requests.
LOG(INFO) << "Standby promoted to Primary successfully. "
<< "All remaining OpLog entries have been synced.";
// Return ErrorCode - actual MasterService creation happens externally
// The caller (MasterServiceSupervisor) will create the MasterService
// with the appropriate configuration.
return ErrorCode::OK;
}
size_t HotStandbyService::GetMetadataCount() const {
std::lock_guard<std::mutex> lock(mutex_);
return metadata_store_ ? metadata_store_->GetKeyCount() : 0;
}
uint64_t HotStandbyService::GetLatestAppliedSequenceId() const {
std::lock_guard<std::mutex> lock(mutex_);
if (oplog_applier_) {
uint64_t expected_seq = oplog_applier_->GetExpectedSequenceId();
// GetExpectedSequenceId returns the next expected sequence_id,
// so the latest applied is expected_seq - 1
return expected_seq > 0 ? expected_seq - 1 : 0;
}
return applied_seq_id_.load();
}
bool HotStandbyService::ExportMetadataSnapshot(
std::vector<std::pair<std::string, StandbyObjectMetadata>>& out) const {
std::lock_guard<std::mutex> lock(mutex_);
if (!metadata_store_) {
out.clear();
return false;
}
metadata_store_->Snapshot(out);
return true;
}
void HotStandbyService::SetSnapshotProvider(
std::unique_ptr<SnapshotProvider> provider) {
std::lock_guard<std::mutex> lock(mutex_);
if (provider) {
snapshot_provider_ = std::move(provider);
} else {
snapshot_provider_ = std::make_unique<NoopSnapshotProvider>();
}
}
void HotStandbyService::ReplicationLoop() {
LOG(INFO) << "Replication loop started (etcd-based OpLog sync)";
// With etcd-based OpLog sync, OpLogWatcher handles the actual watching
// in its own thread. This loop now just monitors the status and updates
// metrics.
// Create EtcdOpLogStore once before the loop to avoid repeated
// construction/destruction overhead (constructor does etcd I/O and
// spawns background threads).
#ifdef STORE_USE_ETCD
std::unique_ptr<EtcdOpLogStore> oplog_store;
if (!cluster_id_.empty()) {
oplog_store = std::make_unique<EtcdOpLogStore>(
cluster_id_, /*enable_latest_seq_batch_update=*/false);
if (oplog_store->Init() != ErrorCode::OK) {
LOG(ERROR)
<< "Failed to initialize oplog_store in replication loop";
oplog_store.reset();
}
}
#endif
while (IsRunning()) {
if (!IsConnected()) {
// Not connected - wait a bit before checking again
std::this_thread::sleep_for(std::chrono::seconds(1));
continue;
}
// Update applied_seq_id from OpLogApplier
if (oplog_applier_) {
uint64_t expected = oplog_applier_->GetExpectedSequenceId();
uint64_t current_applied = (expected > 0) ? (expected - 1) : 0;
if (current_applied > 0) {
applied_seq_id_.store(current_applied);
}
}
// Update primary_seq_id by querying etcd `/latest` (best-effort).
// Note: `/latest` is batch-updated on Primary, so this is for
// monitoring only.
#ifdef STORE_USE_ETCD
if (oplog_store) {
uint64_t latest_seq = 0;
ErrorCode err = oplog_store->GetLatestSequenceId(latest_seq);
if (err == ErrorCode::OK) {
primary_seq_id_.store(latest_seq);
}
}
#endif
// Sleep and check again
std::this_thread::sleep_for(std::chrono::milliseconds(1000));
}
LOG(INFO) << "Replication loop stopped";
}
void HotStandbyService::VerificationLoop() {
LOG(INFO) << "Verification loop started";
while (IsRunning()) {
std::this_thread::sleep_for(
std::chrono::seconds(config_.verification_interval_sec));
if (!IsConnected()) {
continue;
}
// Verification is not yet implemented. When enabled, this loop is
// expected to:
// 1) sample keys from the local metadata store,
// 2) calculate checksums,
// 3) send a verification request to the Primary, and
// 4) handle any mismatches that are detected.
VLOG(1)
<< "Verification check skipped (feature not implemented), state="
<< StandbyStateToString(GetState());
}
LOG(INFO) << "Verification loop stopped";
}
void HotStandbyService::ApplyOpLogEntry(const OpLogEntry& entry) {
// NOTE: This method is deprecated. OpLog entries are now applied via
// OpLogApplier, which is called by OpLogWatcher. This method is kept
// for backward compatibility but should not be used in the new etcd-based
// implementation.
// Update applied_seq_id for status tracking
applied_seq_id_.store(entry.sequence_id);
// The actual application is handled by OpLogApplier via OpLogWatcher
VLOG(2) << "ApplyOpLogEntry called (deprecated), sequence_id="
<< entry.sequence_id
<< ", op_type=" << static_cast<int>(entry.op_type)
<< ", key=" << entry.object_key;
}
void HotStandbyService::ProcessOpLogBatch(
const std::vector<OpLogEntry>& entries) {
for (const auto& entry : entries) {
ApplyOpLogEntry(entry);
}
}
bool HotStandbyService::ConnectToPrimary() {
// With etcd-based OpLog sync, connection is handled by OpLogWatcher
// This method is kept for compatibility but is no longer used
LOG(INFO) << "ConnectToPrimary called (no-op with etcd-based sync)";
return true;
}
void HotStandbyService::DisconnectFromPrimary() {
// With etcd-based OpLog sync, disconnection is handled by OpLogWatcher
// This method is kept for compatibility
if (IsConnected()) {
state_machine_.ProcessEvent(StandbyEvent::DISCONNECTED);
replication_stream_.reset();
LOG(INFO) << "Disconnected from Primary (etcd-based sync), state="
<< StandbyStateToString(GetState());
}
}
} // namespace mooncake

621
mooncake-store/src/oplog_applier.cpp Normal file
View File

@ -0,0 +1,621 @@
#include "oplog_applier.h"
#include <glog/logging.h>
#include <algorithm>
#include <chrono>
#include "etcd_oplog_store.h"
#include "ha_metric_manager.h"
#include "metadata_store.h"
#include "oplog_manager.h"
namespace mooncake {
OpLogApplier::OpLogApplier(MetadataStore* metadata_store,
const std::string& cluster_id)
: metadata_store_(metadata_store),
cluster_id_(cluster_id),
expected_sequence_id_(1) {
if (metadata_store_ == nullptr) {
LOG(FATAL) << "OpLogApplier: metadata_store cannot be null";
}
// Validate cluster_id if provided (required for etcd operations).
// Normalize by stripping trailing slashes for validation.
std::string normalized = cluster_id_;
while (!normalized.empty() && normalized.back() == '/') {
normalized.pop_back();
}
if (!normalized.empty() && !IsValidClusterIdComponent(normalized)) {
LOG(FATAL)
<< "Invalid cluster_id for OpLogApplier: '" << cluster_id_
<< "' (normalized: '" << normalized
<< "'). Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes.";
}
}
EtcdOpLogStore* OpLogApplier::GetEtcdOpLogStore() const {
#ifdef STORE_USE_ETCD
if (cluster_id_.empty()) {
return nullptr;
}
std::lock_guard<std::mutex> lock(etcd_oplog_store_mutex_);
if (!etcd_oplog_store_) {
// Reader: do not start `/latest` batch update thread.
auto new_store = std::make_unique<EtcdOpLogStore>(
cluster_id_, /*enable_latest_seq_batch_update=*/false);
if (new_store->Init() != ErrorCode::OK) {
LOG(ERROR) << "Failed to initialize EtcdOpLogStore for cluster: "
<< cluster_id_;
return nullptr;
}
etcd_oplog_store_ = std::move(new_store);
}
return etcd_oplog_store_.get();
#else
return nullptr;
#endif
}
bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) {
// Basic DoS protection: validate key/payload sizes before parsing/applying.
std::string size_reason;
if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) {
LOG(ERROR) << "OpLogApplier: entry size rejected, sequence_id="
<< entry.sequence_id << ", key=" << entry.object_key
<< ", reason=" << size_reason;
return false;
}
// Verify checksum to detect data corruption or tampering.
if (!OpLogManager::VerifyChecksum(entry)) {
LOG(ERROR)
<< "OpLogApplier: checksum mismatch, sequence_id="
<< entry.sequence_id << ", key=" << entry.object_key
<< ". Possible data corruption or tampering. Discarding entry.";
HAMetricManager::instance().inc_oplog_checksum_failures();
return false;
}
// Global ordering only.
//
// IMPORTANT:
// - Watch callbacks / retries may deliver duplicate or already-applied
// entries.
// - Those must be treated as no-op, not as "out-of-order pending",
// otherwise
// pending_entries_ can grow and the applier may appear stuck.
const uint64_t expected = expected_sequence_id_.load();
if (IsSequenceOlder(entry.sequence_id, expected)) {
// Late arrival of a previously-skipped gap entry: apply only if it's a
// delete/revoke.
bool was_skipped = false;
{
std::lock_guard<std::mutex> lock(pending_mutex_);
auto it = skipped_sequence_ids_.find(entry.sequence_id);
if (it != skipped_sequence_ids_.end()) {
was_skipped = true;
skipped_sequence_ids_.erase(it);
}
}
if (was_skipped) {
if (entry.op_type == OpType::REMOVE ||
entry.op_type == OpType::PUT_REVOKE) {
// Safe: ensure we don't keep stale metadata.
if (entry.op_type == OpType::REMOVE) {
ApplyRemove(entry);
} else {
ApplyPutRevoke(entry);
}
return true;
}
// PUT_END (or others): discard to avoid resurrecting stale state.
if (entry.op_type == OpType::PUT_END) {
HAMetricManager::instance().inc_oplog_dropped_put_end();
}
VLOG(1) << "OpLogApplier: discard late skipped entry, op_type="
<< static_cast<int>(entry.op_type)
<< ", sequence_id=" << entry.sequence_id
<< ", key=" << entry.object_key;
return true;
}
VLOG(2) << "OpLogApplier: skip already-applied entry, sequence_id="
<< entry.sequence_id << ", expected=" << expected
<< ", key=" << entry.object_key;
return true; // consumed (no-op)
}
if (IsSequenceNewer(entry.sequence_id, expected)) {
// Future entry - store into pending, wait for the gap to be filled.
std::lock_guard<std::mutex> lock(pending_mutex_);
if (pending_entries_.size() >=
static_cast<size_t>(kMaxPendingEntries)) {
LOG(ERROR) << "OpLogApplier: too many pending entries ("
<< pending_entries_.size()
<< "), discarding entry sequence_id="
<< entry.sequence_id << ", key=" << entry.object_key;
return false;
}
pending_entries_[entry.sequence_id] = entry;
VLOG(1) << "OpLogApplier: future entry buffered, sequence_id="
<< entry.sequence_id << ", expected=" << expected
<< ", key=" << entry.object_key
<< ", pending_entries=" << pending_entries_.size();
return false;
}
// Apply the operation based on type
switch (entry.op_type) {
case OpType::PUT_END:
ApplyPutEnd(entry);
break;
case OpType::PUT_REVOKE:
ApplyPutRevoke(entry);
break;
case OpType::REMOVE:
ApplyRemove(entry);
break;
default:
LOG(ERROR) << "OpLogApplier: unsupported op_type="
<< static_cast<int>(entry.op_type)
<< ", sequence_id=" << entry.sequence_id
<< ", key=" << entry.object_key;
return false;
}
// Update expected sequence ID
expected_sequence_id_.store(entry.sequence_id + 1);
// Update metrics
HAMetricManager::instance().inc_oplog_applied_entries();
HAMetricManager::instance().set_oplog_applied_sequence_id(
static_cast<int64_t>(entry.sequence_id));
// Try to process pending entries
ProcessPendingEntries();
return true;
}
size_t OpLogApplier::ApplyOpLogEntries(const std::vector<OpLogEntry>& entries) {
size_t applied_count = 0;
for (const auto& entry : entries) {
if (ApplyOpLogEntry(entry)) {
applied_count++;
}
}
return applied_count;
}
uint64_t OpLogApplier::GetKeySequenceId(const std::string& key) const {
// Global sequence_id is used for ordering.
(void)key; // Suppress unused parameter warning
return 0;
}
uint64_t OpLogApplier::GetExpectedSequenceId() const {
return expected_sequence_id_.load();
}
void OpLogApplier::Recover(uint64_t last_applied_sequence_id) {
expected_sequence_id_.store(last_applied_sequence_id + 1);
LOG(INFO) << "OpLogApplier: recovered from sequence_id="
<< last_applied_sequence_id << ", expected_sequence_id set to="
<< expected_sequence_id_.load();
}
size_t OpLogApplier::ProcessPendingEntries() {
// Check for missing sequence IDs, possibly skip after timeout, and/or
// request them.
uint64_t missing_seq_to_request = 0;
uint64_t skipped_count = 0;
{
std::lock_guard<std::mutex> lock(pending_mutex_);
auto now = std::chrono::steady_clock::now();
for (;;) {
if (pending_entries_.empty()) {
break;
}
const uint64_t first_pending_seq = pending_entries_.begin()->first;
const uint64_t expected = expected_sequence_id_.load();
if (IsSequenceOlderOrEqual(first_pending_seq, expected)) {
break;
}
// There's a gap: expected is missing.
const uint64_t missing_seq = expected;
auto it = missing_sequence_ids_.find(missing_seq);
if (it == missing_sequence_ids_.end()) {
missing_sequence_ids_[missing_seq] = now;
ScheduleWaitForMissingEntries(missing_seq);
break;
}
const auto waited =
std::chrono::duration_cast<std::chrono::seconds>(now -
it->second);
// Skip after timeout to avoid global stall (user requested
// behavior).
if (waited.count() >= kMissingEntrySkipSeconds) {
skipped_sequence_ids_[missing_seq] = now;
missing_sequence_ids_.erase(missing_seq);
expected_sequence_id_.store(missing_seq + 1);
skipped_count++;
HAMetricManager::instance().inc_oplog_skipped_entries();
LOG(WARNING)
<< "OpLogApplier: skipped missing entry seq=" << missing_seq
<< " after " << waited.count() << "s timeout";
continue; // may skip multiple consecutive gaps
}
// Best-effort request from etcd (before skip triggers).
if (waited.count() >= kMissingEntryRequestSeconds) {
missing_seq_to_request = missing_seq;
break;
}
break;
}
}
// Request missing OpLog if needed (outside the lock to avoid deadlock)
bool retrieved_missing = false;
if (missing_seq_to_request > 0) {
retrieved_missing = RequestMissingOpLog(missing_seq_to_request);
if (retrieved_missing) {
std::lock_guard<std::mutex> lock(pending_mutex_);
missing_sequence_ids_.erase(missing_seq_to_request);
}
}
size_t processed_count = 0;
for (;;) {
OpLogEntry entry_copy;
bool has_entry = false;
{
std::lock_guard<std::mutex> lock(pending_mutex_);
if (pending_entries_.empty()) {
break;
}
auto it = pending_entries_.begin();
const uint64_t expected = expected_sequence_id_.load();
if (!IsSequenceEqual(it->first, expected)) {
break; // still waiting for earlier sequence_id
}
entry_copy = it->second;
pending_entries_.erase(it);
has_entry = true;
}
if (!has_entry) {
break;
}
// Apply outside lock.
switch (entry_copy.op_type) {
case OpType::PUT_END:
ApplyPutEnd(entry_copy);
break;
case OpType::PUT_REVOKE:
ApplyPutRevoke(entry_copy);
break;
case OpType::REMOVE:
ApplyRemove(entry_copy);
break;
default:
LOG(ERROR)
<< "OpLogApplier: unsupported op_type in pending entry";
break;
}
expected_sequence_id_.store(entry_copy.sequence_id + 1);
{
std::lock_guard<std::mutex> lock(pending_mutex_);
missing_sequence_ids_.erase(entry_copy.sequence_id);
}
processed_count++;
}
// Clean up old missing sequence IDs (older than 1 minute)
{
std::lock_guard<std::mutex> lock(pending_mutex_);
auto now = std::chrono::steady_clock::now();
for (auto it = missing_sequence_ids_.begin();
it != missing_sequence_ids_.end();) {
auto age = std::chrono::duration_cast<std::chrono::seconds>(
now - it->second);
if (age.count() > 60) {
LOG(WARNING)
<< "OpLogApplier: giving up on missing sequence_id="
<< it->first << " after " << age.count() << " seconds";
it = missing_sequence_ids_.erase(it);
} else {
++it;
}
}
// Clean up old skipped sequence IDs too (avoid unbounded growth).
for (auto it = skipped_sequence_ids_.begin();
it != skipped_sequence_ids_.end();) {
auto age = std::chrono::duration_cast<std::chrono::seconds>(
now - it->second);
if (age.count() > 60) {
it = skipped_sequence_ids_.erase(it);
} else {
++it;
}
}
}
if (skipped_count > 0) {
LOG(WARNING) << "OpLogApplier: skipped " << skipped_count
<< " missing sequence_id(s) after timeout, "
"expected_sequence_id now="
<< expected_sequence_id_.load();
}
if (processed_count > 0) {
LOG(INFO) << "OpLogApplier: processed " << processed_count
<< " pending entries, expected_sequence_id now="
<< expected_sequence_id_.load();
}
// Update pending entries metric
{
std::lock_guard<std::mutex> lock(pending_mutex_);
HAMetricManager::instance().set_oplog_pending_entries(
static_cast<int64_t>(pending_entries_.size()));
}
return processed_count;
}
OpLogApplier::GapResolveResult OpLogApplier::TryResolveGapsOnceForPromotion(
size_t max_ids) {
GapResolveResult r;
#ifdef STORE_USE_ETCD
EtcdOpLogStore* store = GetEtcdOpLogStore();
if (store == nullptr) {
return r;
}
std::vector<uint64_t> gap_ids;
gap_ids.reserve(max_ids);
{
std::lock_guard<std::mutex> lock(pending_mutex_);
for (const auto& kv : missing_sequence_ids_) {
if (gap_ids.size() >= max_ids) break;
gap_ids.push_back(kv.first);
}
for (const auto& kv : skipped_sequence_ids_) {
if (gap_ids.size() >= max_ids) break;
gap_ids.push_back(kv.first);
}
}
if (gap_ids.empty()) {
return r;
}
std::sort(gap_ids.begin(), gap_ids.end());
gap_ids.erase(std::unique(gap_ids.begin(), gap_ids.end()), gap_ids.end());
r.attempted = gap_ids.size();
std::vector<uint64_t> successfully_processed;
for (uint64_t seq : gap_ids) {
OpLogEntry e;
ErrorCode err = store->ReadOpLog(seq, e);
if (err != ErrorCode::OK) {
// Log failed gap for monitoring, but don't clear it so it can be
// retried later.
LOG(WARNING) << "Promotion gap resolve: failed to fetch seq=" << seq
<< ", err=" << static_cast<int>(err);
continue;
}
r.fetched++;
// Apply policy: only delete/revoke; drop PUT_END.
if (e.op_type == OpType::REMOVE) {
ApplyRemove(e);
r.applied_deletes++;
successfully_processed.push_back(seq);
} else if (e.op_type == OpType::PUT_REVOKE) {
ApplyPutRevoke(e);
r.applied_deletes++;
successfully_processed.push_back(seq);
} else {
// PUT_END or others: mark as processed (dropped) so we don't retry.
successfully_processed.push_back(seq);
}
}
// Only clear gaps we successfully fetched and processed.
// Failed gaps remain in missing_sequence_ids_/skipped_sequence_ids_ for
// potential retry or monitoring.
if (!successfully_processed.empty()) {
std::lock_guard<std::mutex> lock(pending_mutex_);
for (uint64_t seq : successfully_processed) {
missing_sequence_ids_.erase(seq);
skipped_sequence_ids_.erase(seq);
}
}
return r;
#else
(void)max_ids;
return r;
#endif
}
bool OpLogApplier::CheckSequenceOrder(const OpLogEntry& entry) {
// Only check global sequence order.
// Use IsSequenceEqual for wrap-around safety (though equality check doesn't
// need special handling, we use it for consistency).
return IsSequenceEqual(entry.sequence_id, expected_sequence_id_.load());
}
void OpLogApplier::ApplyPutEnd(const OpLogEntry& entry) {
// Payload contains serialized metadata (replicas, size, etc.) in JSON
// format. Deserialize the payload immediately and store structured
// metadata. This allows Standby to serve requests immediately after
// promotion.
if (entry.payload.empty()) {
// No payload - create empty metadata (legacy compatibility)
LOG(WARNING) << "OpLogApplier: PUT_END without payload, key="
<< entry.object_key
<< ", sequence_id=" << entry.sequence_id;
StandbyObjectMetadata empty_metadata;
empty_metadata.last_sequence_id = entry.sequence_id;
if (!metadata_store_->PutMetadata(entry.object_key, empty_metadata)) {
LOG(ERROR) << "OpLogApplier: failed to PutMetadata key="
<< entry.object_key
<< ", sequence_id=" << entry.sequence_id;
}
return;
}
// Deserialize payload using struct_pack (msgpack binary format)
MetadataPayload payload;
auto result = struct_pack::deserialize_to(payload, entry.payload);
if (result != struct_pack::errc::ok) {
LOG(ERROR) << "OpLogApplier: failed to deserialize payload for key="
<< entry.object_key << ", sequence_id=" << entry.sequence_id
<< ", payload_size=" << entry.payload.size()
<< ", error_code=" << static_cast<int>(result);
// Fallback to empty metadata if parsing fails
StandbyObjectMetadata empty_metadata;
empty_metadata.last_sequence_id = entry.sequence_id;
metadata_store_->PutMetadata(entry.object_key, empty_metadata);
return;
}
// Convert to StandbyObjectMetadata and store
StandbyObjectMetadata metadata =
payload.ToStandbyMetadata(entry.sequence_id);
if (!metadata_store_->PutMetadata(entry.object_key, metadata)) {
LOG(ERROR) << "OpLogApplier: failed to PutMetadata key="
<< entry.object_key << ", sequence_id=" << entry.sequence_id;
} else {
VLOG(1) << "OpLogApplier: applied PUT_END, key=" << entry.object_key
<< ", sequence_id=" << entry.sequence_id
<< ", replicas=" << metadata.replicas.size()
<< ", size=" << metadata.size;
}
}
void OpLogApplier::ApplyPutRevoke(const OpLogEntry& entry) {
// PUT_REVOKE means the object should be removed from metadata store
// (but the key itself may still exist if there are other replicas).
// Current implementation removes the entire key; if we later support
// partial replica revocation this logic will need to be refined.
if (!metadata_store_->Remove(entry.object_key)) {
LOG(WARNING) << "OpLogApplier: failed to Remove key="
<< entry.object_key
<< " in PUT_REVOKE, sequence_id=" << entry.sequence_id
<< " (key may not exist)";
} else {
VLOG(1) << "OpLogApplier: applied PUT_REVOKE, key=" << entry.object_key
<< ", sequence_id=" << entry.sequence_id;
}
}
void OpLogApplier::ApplyRemove(const OpLogEntry& entry) {
if (!metadata_store_->Remove(entry.object_key)) {
LOG(WARNING) << "OpLogApplier: failed to Remove key="
<< entry.object_key
<< ", sequence_id=" << entry.sequence_id
<< " (key may not exist)";
} else {
VLOG(1) << "OpLogApplier: applied REMOVE, key=" << entry.object_key
<< ", sequence_id=" << entry.sequence_id;
}
}
bool OpLogApplier::RequestMissingOpLog(uint64_t missing_seq_id) {
#ifdef STORE_USE_ETCD
HAMetricManager::instance().inc_oplog_gap_resolve_attempts();
EtcdOpLogStore* oplog_store = GetEtcdOpLogStore();
if (oplog_store == nullptr) {
LOG(WARNING)
<< "OpLogApplier: cannot request missing OpLog, cluster_id not set";
return false;
}
OpLogEntry entry;
ErrorCode err = oplog_store->ReadOpLog(missing_seq_id, entry);
if (err == ErrorCode::ETCD_KEY_NOT_EXIST) {
LOG(INFO) << "OpLogApplier: missing OpLog entry not found in etcd, "
"sequence_id="
<< missing_seq_id;
return false;
}
if (err != ErrorCode::OK) {
LOG(ERROR) << "OpLogApplier: failed to read missing OpLog from etcd, "
"sequence_id="
<< missing_seq_id << ", error=" << static_cast<int>(err);
return false;
}
std::string size_reason;
if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) {
LOG(ERROR) << "OpLogApplier: missing entry size rejected, sequence_id="
<< missing_seq_id << ", key=" << entry.object_key
<< ", reason=" << size_reason;
return false;
}
// Verify checksum before adding to pending entries.
if (!OpLogManager::VerifyChecksum(entry)) {
LOG(ERROR) << "OpLogApplier: checksum mismatch for retrieved missing "
"entry, sequence_id="
<< missing_seq_id << ", key=" << entry.object_key
<< ". Possible data corruption. Discarding entry.";
HAMetricManager::instance().inc_oplog_checksum_failures();
return false;
}
// Successfully retrieved the missing OpLog entry
LOG(INFO) << "OpLogApplier: retrieved missing OpLog entry, sequence_id="
<< missing_seq_id
<< ", op_type=" << static_cast<int>(entry.op_type)
<< ", key=" << entry.object_key;
HAMetricManager::instance().inc_oplog_gap_resolve_success();
// Add to pending entries
// Note: We don't call ProcessPendingEntries() here to avoid potential
// recursion. The caller (ProcessPendingEntries itself) will process the
// entry in the next loop.
{
std::lock_guard<std::mutex> lock(pending_mutex_);
pending_entries_[entry.sequence_id] = entry;
}
return true;
#else
LOG(WARNING) << "OpLogApplier: STORE_USE_ETCD not enabled, cannot request "
"missing OpLog";
return false;
#endif
}
void OpLogApplier::ScheduleWaitForMissingEntries(uint64_t missing_seq_id) {
// This method is called when we first detect a missing sequence_id.
// The actual waiting and requesting is handled in ProcessPendingEntries().
// We just log it here for tracking.
VLOG(1) << "OpLogApplier: scheduling wait for missing sequence_id="
<< missing_seq_id << ", will request after "
<< kMissingEntryRequestSeconds << " seconds";
}
} // namespace mooncake

183
mooncake-store/src/oplog_manager.cpp Normal file
View File

@ -0,0 +1,183 @@
#include "oplog_manager.h"
#include <algorithm>
#include <chrono>
#include <xxhash.h>
#include <glog/logging.h>
#include "etcd_oplog_store.h"
namespace mooncake {
OpLogManager::OpLogManager() = default;
void OpLogManager::SetEtcdOpLogStore(
std::shared_ptr<EtcdOpLogStore> etcd_oplog_store) {
std::unique_lock<std::shared_mutex> lock(mutex_);
etcd_oplog_store_ = etcd_oplog_store;
}
uint64_t OpLogManager::Append(OpType type, const std::string& key,
const std::string& payload) {
OpLogEntry entry;
entry.op_type = type;
entry.object_key = key;
entry.payload = payload;
entry.timestamp_ms = NowMs();
entry.checksum = ComputeChecksum(entry.payload);
entry.prefix_hash = ComputePrefixHash(entry.object_key);
std::unique_lock<std::shared_mutex> lock(mutex_);
entry.sequence_id = ++last_seq_id_;
const uint64_t seq = entry.sequence_id; // save before potential unlock
if (buffer_.size() >= kMaxBufferEntries_) {
buffer_.pop_front();
++first_seq_id_;
}
buffer_.emplace_back(entry); // Copy entry to buffer
// Write to etcd if EtcdOpLogStore is set.
// Strategy: PUT_END is async (sync=false) — only pushes to batch queue
// (microsecond-level), safe to hold mutex_.
// REMOVE / PUT_REVOKE are sync (sync=true) — blocks until etcd confirms
// persistence; must release mutex_ to avoid blocking other Append calls
// during the wait. The caller relies on sync semantics to know the
// entry is durable before freeing/reusing associated memory.
if (etcd_oplog_store_) {
bool sync = (type != OpType::PUT_END);
if (sync) {
// Release lock before the blocking wait to avoid holding
// mutex_ for the entire etcd round-trip.
lock.unlock();
}
ErrorCode err = etcd_oplog_store_->WriteOpLog(entry, sync);
if (err != ErrorCode::OK) {
LOG(WARNING) << "Failed to write OpLog to etcd, sequence_id=" << seq
<< ", but entry is in memory buffer";
}
}
return seq;
}
OpLogEntry OpLogManager::AllocateEntry(OpType type, const std::string& key,
const std::string& payload) {
OpLogEntry entry;
entry.op_type = type;
entry.object_key = key;
entry.payload = payload;
entry.timestamp_ms = NowMs();
entry.checksum = ComputeChecksum(entry.payload);
entry.prefix_hash = ComputePrefixHash(entry.object_key);
std::unique_lock<std::shared_mutex> lock(mutex_);
entry.sequence_id = ++last_seq_id_;
if (buffer_.size() >= kMaxBufferEntries_) {
buffer_.pop_front();
++first_seq_id_;
}
buffer_.emplace_back(entry);
return entry;
}
ErrorCode OpLogManager::PersistEntryToEtcd(const OpLogEntry& entry) const {
std::shared_lock<std::shared_mutex> lock(mutex_);
auto store = etcd_oplog_store_;
lock.unlock();
if (!store) {
return ErrorCode::ETCD_OPERATION_ERROR;
}
// Strategy 2+: PUT_END is Async, REMOVE (and others) are Sync
bool sync = (entry.op_type != OpType::PUT_END);
return store->WriteOpLog(entry, sync);
}
tl::expected<uint64_t, ErrorCode> OpLogManager::AppendAndPersist(
OpType type, const std::string& key, const std::string& payload) {
// Seq pre-allocation semantics: allocate first, then persist.
OpLogEntry entry = AllocateEntry(type, key, payload);
ErrorCode err = PersistEntryToEtcd(entry);
if (err != ErrorCode::OK) {
return tl::make_unexpected(err);
}
return entry.sequence_id;
}
uint64_t OpLogManager::GetLastSequenceId() const {
std::shared_lock<std::shared_mutex> lock(mutex_);
return last_seq_id_;
}
void OpLogManager::SetInitialSequenceId(uint64_t sequence_id) {
std::unique_lock<std::shared_mutex> lock(mutex_);
if (last_seq_id_ == 0 && buffer_.empty()) {
// Only allow setting initial sequence_id if OpLogManager is empty
last_seq_id_ = sequence_id;
first_seq_id_ = sequence_id +
1; // first_seq_id_ should be > last_seq_id_ when empty
LOG(INFO) << "OpLogManager initial sequence_id set to " << sequence_id;
} else {
LOG(WARNING)
<< "Cannot set initial sequence_id: OpLogManager is not empty "
<< "(last_seq_id_=" << last_seq_id_
<< ", buffer_size=" << buffer_.size() << ")";
}
}
size_t OpLogManager::GetEntryCount() const {
std::shared_lock<std::shared_mutex> lock(mutex_);
return buffer_.size();
}
uint64_t OpLogManager::NowMs() {
using namespace std::chrono;
return duration_cast<milliseconds>(steady_clock::now().time_since_epoch())
.count();
}
uint32_t OpLogManager::ComputeChecksum(const std::string& data) {
// Use xxHash XXH32 for a fast, deterministic 32-bit checksum.
// Requires linking against xxHash (e.g., libxxhash) and including
// <xxhash.h>.
return static_cast<uint32_t>(XXH32(data.data(), data.size(), 0));
}
uint32_t OpLogManager::ComputePrefixHash(const std::string& key) {
if (key.empty()) {
return 0;
}
// Use XXH32 for consistency with ComputeChecksum and better performance.
// XXH32 provides faster hashing and lower collision rate than std::hash.
// Computing hash for the entire key ensures better distribution and fewer
// collisions.
return static_cast<uint32_t>(XXH32(key.data(), key.size(), 0));
}
bool OpLogManager::VerifyChecksum(const OpLogEntry& entry) {
uint32_t computed = ComputeChecksum(entry.payload);
return computed == entry.checksum;
}
bool OpLogManager::ValidateEntrySize(const OpLogEntry& entry,
std::string* reason) {
if (entry.object_key.size() > kMaxObjectKeySize) {
if (reason) {
*reason = "object_key too large: size=" +
std::to_string(entry.object_key.size());
}
return false;
}
if (entry.payload.size() > kMaxPayloadSize) {
if (reason) {
*reason = "payload too large: size=" +
std::to_string(entry.payload.size());
}
return false;
}
return true;
}
} // namespace mooncake

608
mooncake-store/src/oplog_watcher.cpp Normal file
View File

@ -0,0 +1,608 @@
#include "oplog_watcher.h"
#include <algorithm>
#include <chrono>
#include <glog/logging.h>
#include <sstream>
#include <thread>
#ifdef STORE_USE_ETCD
#include "etcd_helper.h"
#include "etcd_oplog_store.h"
#include "ha_metric_manager.h"
#include "oplog_applier.h"
#include "oplog_manager.h"
#include "utils/base64.h"
#if __has_include(<jsoncpp/json/json.h>)
#include <jsoncpp/json/json.h> // Ubuntu
#else
#include <json/json.h> // CentOS
#endif
namespace mooncake {
OpLogWatcher::OpLogWatcher(const std::string& etcd_endpoints,
const std::string& cluster_id, OpLogApplier* applier)
: etcd_endpoints_(etcd_endpoints),
cluster_id_(cluster_id),
applier_(applier) {
if (applier_ == nullptr) {
LOG(FATAL) << "OpLogApplier cannot be null";
}
// Normalize cluster_id to avoid double slashes in watch prefix.
while (!cluster_id_.empty() && cluster_id_.back() == '/') {
cluster_id_.pop_back();
}
if (!cluster_id_.empty() && !IsValidClusterIdComponent(cluster_id_)) {
LOG(FATAL)
<< "Invalid cluster_id for OpLogWatcher: '" << cluster_id_
<< "'. Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes.";
}
#ifdef STORE_USE_ETCD
op_log_store_ = std::make_unique<EtcdOpLogStore>(
cluster_id_, /*enable_latest_seq_batch_update=*/false);
if (op_log_store_->Init() != ErrorCode::OK) {
LOG(ERROR) << "Failed to initialize EtcdOpLogStore";
}
#endif
// Allocate the shared callback context so that C-style callbacks
// can safely check whether the watcher is still alive.
watch_callback_ctx_ = new WatchCallbackContext();
watch_callback_ctx_->watcher = this;
}
OpLogWatcher::~OpLogWatcher() {
Stop();
// If Stop() returned early (watcher was never started), watch_callback_ctx_
// was never freed. It is safe to delete here because no goroutine / watch
// thread was ever launched, so no callbacks can be in-flight.
// In all other paths, Stop() already sets watch_callback_ctx_ to nullptr,
// so `delete nullptr` is a harmless no-op.
delete watch_callback_ctx_;
watch_callback_ctx_ = nullptr;
}
void OpLogWatcher::Start() {
// Backward-compatible: start from the last processed sequence id.
(void)StartFromSequenceId(last_processed_sequence_id_.load());
}
bool OpLogWatcher::StartFromSequenceId(uint64_t start_seq_id) {
if (running_.load()) {
LOG(WARNING) << "OpLogWatcher is already running";
return true;
}
#ifdef STORE_USE_ETCD
uint64_t read_seq_id = start_seq_id;
EtcdRevisionId last_read_rev = 0;
size_t total_applied = 0;
for (;;) {
std::vector<OpLogEntry> batch;
EtcdRevisionId rev = 0;
if (!ReadOpLogSince(read_seq_id, batch, rev)) {
LOG(ERROR) << "ReadOpLogSince failed during initial sync"
<< ", read_seq_id=" << read_seq_id;
return false;
}
last_read_rev = rev;
if (!batch.empty()) {
for (const auto& e : batch) {
if (applier_->ApplyOpLogEntry(e)) {
last_processed_sequence_id_.store(e.sequence_id);
read_seq_id = e.sequence_id;
total_applied++;
}
}
}
if (batch.size() < kSyncBatchSize) {
break;
}
}
if (last_read_rev > 0) {
next_watch_revision_.store(static_cast<int64_t>(last_read_rev + 1));
} else {
next_watch_revision_.store(0);
}
LOG(INFO) << "OpLogWatcher initial sync done: applied=" << total_applied
<< ", last_seq=" << last_processed_sequence_id_.load()
<< ", next_watch_revision=" << next_watch_revision_.load();
#endif
running_.store(true);
watch_thread_ = std::thread(&OpLogWatcher::WatchOpLog, this);
LOG(INFO) << "OpLogWatcher started for cluster_id=" << cluster_id_;
return true;
}
void OpLogWatcher::Stop() {
if (!running_.load()) {
return;
}
running_.store(false);
#ifdef STORE_USE_ETCD
// 1. Invalidate the callback context under the mutex so that any
// in-flight or future callbacks from the Go goroutine will see
// watcher == nullptr and return immediately.
if (watch_callback_ctx_) {
std::lock_guard<std::mutex> lock(watch_callback_ctx_->mutex);
watch_callback_ctx_->watcher = nullptr;
}
// 2. Wait for the C++ watch thread to finish.
if (watch_thread_.joinable()) {
watch_thread_.join();
}
// 3. Cancel the Go goroutine and wait for it to fully exit.
std::string watch_prefix = "/oplog/" + cluster_id_ + "/";
ErrorCode err = EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(),
watch_prefix.size());
if (err != ErrorCode::OK) {
LOG(WARNING) << "Failed to cancel watch for prefix " << watch_prefix
<< ", error=" << static_cast<int>(err);
}
ErrorCode wait_err = EtcdHelper::WaitWatchWithPrefixStopped(
watch_prefix.c_str(), watch_prefix.size(), /*timeout_ms=*/5000);
// 4. Free the callback context only if the goroutine confirmed stopped.
// Otherwise, intentionally leak to prevent use-after-free from late
// callbacks.
if (wait_err == ErrorCode::OK) {
delete watch_callback_ctx_;
} else {
LOG(WARNING)
<< "Watch goroutine did not stop in time for prefix "
<< watch_prefix
<< "; leaking WatchCallbackContext to avoid use-after-free";
}
watch_callback_ctx_ = nullptr;
#else
if (watch_thread_.joinable()) {
watch_thread_.join();
}
delete watch_callback_ctx_;
watch_callback_ctx_ = nullptr;
#endif
LOG(INFO) << "OpLogWatcher stopped";
}
bool OpLogWatcher::ReadOpLogSince(uint64_t start_seq_id,
std::vector<OpLogEntry>& entries,
EtcdRevisionId& revision_id) {
#ifdef STORE_USE_ETCD
if (!op_log_store_) {
return false;
}
ErrorCode err = op_log_store_->ReadOpLogSinceWithRevision(
start_seq_id, kSyncBatchSize, entries, revision_id);
if (err != ErrorCode::OK) {
LOG(ERROR) << "Failed to read OpLog since sequence_id=" << start_seq_id
<< ", error=" << static_cast<int>(err);
return false;
}
return true;
#else
(void)start_seq_id;
(void)entries;
(void)revision_id;
return false;
#endif
}
uint64_t OpLogWatcher::GetLastProcessedSequenceId() const {
return last_processed_sequence_id_.load();
}
void OpLogWatcher::WatchCallback(void* context, const char* key,
size_t key_size, const char* value,
size_t value_size, int event_type,
int64_t mod_revision) {
auto* ctx = static_cast<WatchCallbackContext*>(context);
if (ctx == nullptr) {
return;
}
// Lock the control block to ensure the watcher is still alive for
// the entire duration of this callback invocation.
std::lock_guard<std::mutex> lock(ctx->mutex);
OpLogWatcher* watcher = ctx->watcher;
if (watcher == nullptr) {
// Watcher has been stopped / destroyed; discard the event.
return;
}
if (!watcher->running_.load(std::memory_order_acquire)) {
return;
}
std::string key_str;
if (key != nullptr && key_size > 0) {
key_str.assign(key, key_size);
}
std::string value_str;
if (value != nullptr && value_size > 0) {
value_str = std::string(value, value_size);
}
watcher->HandleWatchEvent(key_str, value_str, event_type, mod_revision);
}
void OpLogWatcher::WatchOpLog() {
#ifdef STORE_USE_ETCD
LOG(INFO) << "OpLog watch thread started for cluster_id=" << cluster_id_;
std::string watch_prefix = "/oplog/" + cluster_id_ + "/";
while (running_.load()) {
// Cancel any existing watch before starting a new one
// This prevents "prefix already being watched" errors
(void)EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(),
watch_prefix.size());
(void)EtcdHelper::WaitWatchWithPrefixStopped(watch_prefix.c_str(),
watch_prefix.size(),
/*timeout_ms=*/5000);
// Start watching - pass the shared callback context so that the
// Go goroutine can safely check watcher liveness via mutex.
EtcdRevisionId start_rev =
static_cast<EtcdRevisionId>(next_watch_revision_.load());
// Use watcher with mod_revision so we can update next_watch_revision_
// precisely.
ErrorCode err = EtcdHelper::WatchWithPrefixFromRevision(
watch_prefix.c_str(), watch_prefix.size(), start_rev,
watch_callback_ctx_, WatchCallback);
if (err != ErrorCode::OK) {
LOG(ERROR) << "Failed to start watch for prefix " << watch_prefix
<< ", error=" << static_cast<int>(err);
watch_healthy_.store(false);
NotifyStateEvent(StandbyEvent::WATCH_BROKEN);
// Wait a bit longer before retrying, to ensure old goroutines have
// time to exit
std::this_thread::sleep_for(std::chrono::milliseconds(500));
// Try to reconnect
TryReconnect();
continue;
}
LOG(INFO) << "Watch started for prefix " << watch_prefix;
watch_healthy_.store(true);
consecutive_errors_.store(0);
NotifyStateEvent(StandbyEvent::WATCH_HEALTHY);
// The watch is now running in the background (via Go goroutine)
// We just need to keep the thread alive until Stop() is called or watch
// fails
while (running_.load() && watch_healthy_.load()) {
// Drive pending/missing handling even when no new watch events
// arrive. Without this, a single out-of-order arrival could park
// entries in pending_entries_ forever if the missing entry isn't
// delivered via watch (but exists in etcd and could be fetched).
(void)applier_->ProcessPendingEntries();
std::this_thread::sleep_for(std::chrono::milliseconds(100));
// Periodically check watch health
if (consecutive_errors_.load() >= kMaxConsecutiveErrors) {
LOG(WARNING)
<< "Too many consecutive errors ("
<< consecutive_errors_.load() << "), reconnecting watch...";
watch_healthy_.store(false);
NotifyStateEvent(StandbyEvent::MAX_ERRORS_REACHED);
break;
}
}
if (running_.load() && !watch_healthy_.load()) {
// Cancel current watch before reconnecting
(void)EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(),
watch_prefix.size());
(void)EtcdHelper::WaitWatchWithPrefixStopped(watch_prefix.c_str(),
watch_prefix.size(),
/*timeout_ms=*/5000);
NotifyStateEvent(StandbyEvent::WATCH_BROKEN);
TryReconnect();
}
}
LOG(INFO) << "OpLog watch thread stopped";
#else
LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot watch OpLog from etcd";
running_.store(false);
#endif
}
void OpLogWatcher::TryReconnect() {
if (!running_.load()) {
return;
}
int reconnect_attempt = reconnect_count_.fetch_add(1) + 1;
// Calculate delay with exponential backoff
int delay_ms =
std::min(kReconnectDelayMs * reconnect_attempt, kMaxReconnectDelayMs);
LOG(INFO) << "Attempting to reconnect watch (attempt #" << reconnect_attempt
<< "), waiting " << delay_ms << "ms...";
std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms));
// Sync any missed entries before resuming watch
if (SyncMissedEntries()) {
LOG(INFO) << "Successfully synced missed OpLog entries";
NotifyStateEvent(StandbyEvent::RECOVERY_SUCCESS);
} else {
LOG(WARNING)
<< "Failed to sync missed OpLog entries, continuing anyway";
NotifyStateEvent(StandbyEvent::RECOVERY_FAILED);
}
}
bool OpLogWatcher::SyncMissedEntries() {
#ifdef STORE_USE_ETCD
uint64_t last_seq = last_processed_sequence_id_.load();
if (last_seq == 0) {
// No entries processed yet, nothing to sync
return true;
}
LOG(INFO) << "Syncing missed OpLog entries since sequence_id=" << last_seq;
uint64_t read_seq_id = last_seq;
EtcdRevisionId last_read_rev = 0;
size_t total_applied = 0;
for (;;) {
std::vector<OpLogEntry> batch;
EtcdRevisionId rev = 0;
if (!ReadOpLogSince(read_seq_id, batch, rev)) {
LOG(ERROR) << "Failed to read missed OpLog entries";
return false;
}
if (rev > 0) {
last_read_rev = rev;
}
if (!batch.empty()) {
for (const auto& entry : batch) {
if (applier_->ApplyOpLogEntry(entry)) {
last_processed_sequence_id_.store(entry.sequence_id);
read_seq_id = entry.sequence_id;
total_applied++;
} else {
LOG(WARNING)
<< "Failed to apply missed OpLog entry, sequence_id="
<< entry.sequence_id;
}
}
}
if (batch.size() < kSyncBatchSize) {
break;
}
}
if (last_read_rev > 0) {
next_watch_revision_.store(static_cast<int64_t>(last_read_rev + 1));
}
LOG(INFO) << "Synced " << total_applied << " missed OpLog entries";
return true;
#else
return false;
#endif
}
void OpLogWatcher::HandleWatchEvent(const std::string& key,
const std::string& value, int event_type) {
HandleWatchEvent(key, value, event_type, /*mod_revision=*/0);
}
void OpLogWatcher::HandleWatchEvent(const std::string& key,
const std::string& value, int event_type,
int64_t mod_revision) {
// event_type:
// 0 = PUT, 1 = DELETE, 2 = WATCH_BROKEN (Go watcher terminated; should
// reconnect)
if (event_type == 2) {
LOG(WARNING) << "OpLog watch broken, will reconnect. cluster_id="
<< cluster_id_
<< ", next_watch_revision=" << next_watch_revision_.load()
<< ", last_seq=" << last_processed_sequence_id_.load();
watch_healthy_.store(false);
consecutive_errors_.fetch_add(1);
return;
}
if (mod_revision > 0) {
// Keep next_watch_revision_ monotonic: next = max(next, modRev+1)
int64_t candidate = mod_revision + 1;
int64_t cur = next_watch_revision_.load();
while (candidate > cur &&
!next_watch_revision_.compare_exchange_weak(cur, candidate)) {
// retry
}
}
// event_type: 0 = PUT, 1 = DELETE
if (event_type == 1) {
// DELETE event - OpLog entry was cleaned up
VLOG(1) << "OpLog entry deleted: " << key;
consecutive_errors_.store(0); // Watch is working
return;
}
if (event_type != 0) {
LOG(WARNING) << "Unknown event type: " << event_type
<< " for key: " << key;
consecutive_errors_.fetch_add(1);
return;
}
// Skip the "latest" key and snapshot keys
if (key.find("/latest") != std::string::npos ||
key.find("/snapshot/") != std::string::npos) {
return;
}
// Parse the OpLog entry from JSON
OpLogEntry entry;
if (!DeserializeOpLogEntry(value, entry)) {
LOG(ERROR) << "Failed to deserialize OpLog entry from key: " << key;
consecutive_errors_.fetch_add(1);
return;
}
// Basic DoS protection: validate key/payload sizes before further
// processing.
std::string size_reason;
if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) {
LOG(ERROR) << "OpLog entry size rejected: sequence_id="
<< entry.sequence_id << ", key=" << entry.object_key
<< ", reason=" << size_reason;
consecutive_errors_.fetch_add(1);
return;
}
// Verify checksum to detect data corruption or tampering.
if (!OpLogManager::VerifyChecksum(entry)) {
LOG(ERROR)
<< "OpLog entry checksum mismatch: sequence_id="
<< entry.sequence_id << ", key=" << entry.object_key
<< ". Possible data corruption or tampering. Discarding entry.";
consecutive_errors_.fetch_add(1);
HAMetricManager::instance().inc_oplog_checksum_failures();
return;
}
// Apply the OpLog entry
if (applier_->ApplyOpLogEntry(entry)) {
// last_processed_sequence_id_ must be monotonic. We may "consume"
// duplicate / already-applied entries (entry.sequence_id < expected) as
// no-ops, so never regress this counter.
uint64_t cur = last_processed_sequence_id_.load();
while (IsSequenceNewer(entry.sequence_id, cur) &&
!last_processed_sequence_id_.compare_exchange_weak(
cur, entry.sequence_id)) {
// retry
}
consecutive_errors_.store(0); // Reset error counter on success
reconnect_count_.store(0); // Reset reconnect counter on success
VLOG(2) << "Applied OpLog entry: sequence_id=" << entry.sequence_id
<< ", op_type=" << static_cast<int>(entry.op_type)
<< ", key=" << entry.object_key;
} else {
// ApplyOpLogEntry returns false for out-of-order entries,
// which is expected behavior, not an error
VLOG(1) << "OpLog entry not applied (may be out of order): sequence_id="
<< entry.sequence_id;
}
}
bool OpLogWatcher::DeserializeOpLogEntry(const std::string& json_str,
OpLogEntry& entry) {
Json::Value root;
Json::CharReaderBuilder reader;
std::string errs;
std::istringstream s(json_str);
if (!Json::parseFromStream(reader, s, &root, &errs)) {
LOG(ERROR) << "Failed to parse OpLogEntry JSON: " << errs;
return false;
}
entry.sequence_id = root.get("sequence_id", 0).asUInt64();
entry.timestamp_ms = root.get("timestamp_ms", 0).asUInt64();
entry.op_type = static_cast<OpType>(root.get("op_type", 0).asInt());
entry.object_key = root.get("object_key", "").asString();
// CRITICAL: Base64 decode payload to restore binary data
std::string encoded_payload = root.get("payload", "").asString();
entry.payload = base64::Decode(encoded_payload);
entry.checksum = root.get("checksum", 0).asUInt();
entry.prefix_hash = root.get("prefix_hash", 0).asUInt();
return true;
}
} // namespace mooncake
#else // STORE_USE_ETCD not defined
namespace mooncake {
OpLogWatcher::OpLogWatcher(const std::string& etcd_endpoints,
const std::string& cluster_id, OpLogApplier* applier)
: etcd_endpoints_(etcd_endpoints),
cluster_id_(cluster_id),
applier_(applier) {
LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled";
}
OpLogWatcher::~OpLogWatcher() { Stop(); }
void OpLogWatcher::Start() {
LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled";
}
bool OpLogWatcher::StartFromSequenceId(uint64_t /*start_seq_id*/) {
LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled";
return false;
}
void OpLogWatcher::Stop() {
// No-op when STORE_USE_ETCD is not enabled
}
bool OpLogWatcher::ReadOpLogSince(uint64_t /*start_seq_id*/,
std::vector<OpLogEntry>& /*entries*/,
EtcdRevisionId& /*revision_id*/) {
LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled";
return false;
}
uint64_t OpLogWatcher::GetLastProcessedSequenceId() const {
return last_processed_sequence_id_.load();
}
void OpLogWatcher::WatchOpLog() {
LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled";
}
void OpLogWatcher::HandleWatchEvent(const std::string& key,
const std::string& value, int event_type) {
LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled";
}
void OpLogWatcher::HandleWatchEvent(const std::string& key,
const std::string& value, int event_type,
int64_t mod_revision) {
(void)key;
(void)value;
(void)event_type;
(void)mod_revision;
LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled";
}
void OpLogWatcher::TryReconnect() {
LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled";
}
bool OpLogWatcher::SyncMissedEntries() {
LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled";
return false;
}
} // namespace mooncake
#endif // STORE_USE_ETCD

301
mooncake-store/src/standby_state_machine.cpp Normal file
View File

@ -0,0 +1,301 @@
#include "standby_state_machine.h"
#include <glog/logging.h>
namespace mooncake {
StandbyStateMachine::StandbyStateMachine()
: state_enter_time_(std::chrono::steady_clock::now()) {}
StateTransitionResult StandbyStateMachine::ValidateTransition(
StandbyState from, StandbyEvent event) const {
StateTransitionResult result;
result.allowed = false;
result.old_state = from;
result.new_state = from;
// State transition table
switch (from) {
case StandbyState::STOPPED:
if (event == StandbyEvent::START) {
result.allowed = true;
result.new_state = StandbyState::CONNECTING;
}
break;
case StandbyState::CONNECTING:
switch (event) {
case StandbyEvent::CONNECTED:
result.allowed = true;
result.new_state = StandbyState::SYNCING;
break;
case StandbyEvent::CONNECTION_FAILED:
case StandbyEvent::FATAL_ERROR:
result.allowed = true;
result.new_state = StandbyState::FAILED;
break;
case StandbyEvent::STOP:
result.allowed = true;
result.new_state = StandbyState::STOPPED;
break;
default:
break;
}
break;
case StandbyState::SYNCING:
switch (event) {
case StandbyEvent::SYNC_COMPLETE:
result.allowed = true;
result.new_state = StandbyState::WATCHING;
break;
case StandbyEvent::SYNC_FAILED:
case StandbyEvent::DISCONNECTED:
result.allowed = true;
result.new_state = StandbyState::RECONNECTING;
break;
case StandbyEvent::STOP:
result.allowed = true;
result.new_state = StandbyState::STOPPED;
break;
case StandbyEvent::FATAL_ERROR:
result.allowed = true;
result.new_state = StandbyState::FAILED;
break;
default:
break;
}
break;
case StandbyState::WATCHING:
switch (event) {
case StandbyEvent::WATCH_BROKEN:
case StandbyEvent::DISCONNECTED:
result.allowed = true;
result.new_state = StandbyState::RECONNECTING;
break;
case StandbyEvent::MAX_ERRORS_REACHED:
result.allowed = true;
result.new_state = StandbyState::RECOVERING;
break;
case StandbyEvent::PROMOTE:
result.allowed = true;
result.new_state = StandbyState::PROMOTING;
break;
case StandbyEvent::STOP:
result.allowed = true;
result.new_state = StandbyState::STOPPED;
break;
case StandbyEvent::FATAL_ERROR:
result.allowed = true;
result.new_state = StandbyState::FAILED;
break;
// WATCH_HEALTHY in WATCHING state is a no-op (stay in WATCHING)
case StandbyEvent::WATCH_HEALTHY:
result.allowed = true;
result.new_state = StandbyState::WATCHING;
break;
default:
break;
}
break;
case StandbyState::RECOVERING:
switch (event) {
case StandbyEvent::RECOVERY_SUCCESS:
result.allowed = true;
result.new_state = StandbyState::WATCHING;
break;
case StandbyEvent::RECOVERY_FAILED:
case StandbyEvent::DISCONNECTED:
result.allowed = true;
result.new_state = StandbyState::RECONNECTING;
break;
case StandbyEvent::STOP:
result.allowed = true;
result.new_state = StandbyState::STOPPED;
break;
case StandbyEvent::FATAL_ERROR:
result.allowed = true;
result.new_state = StandbyState::FAILED;
break;
default:
break;
}
break;
case StandbyState::RECONNECTING:
switch (event) {
case StandbyEvent::CONNECTED:
result.allowed = true;
result.new_state = StandbyState::SYNCING;
break;
case StandbyEvent::WATCH_HEALTHY:
// Watch successfully re-established after reconnect
result.allowed = true;
result.new_state = StandbyState::WATCHING;
break;
case StandbyEvent::RECOVERY_SUCCESS:
// Missed entries synced — ready to watch again
result.allowed = true;
result.new_state = StandbyState::WATCHING;
break;
case StandbyEvent::RECOVERY_FAILED:
// Sync failed — stay in RECONNECTING and retry
result.allowed = true;
result.new_state = StandbyState::RECONNECTING;
break;
case StandbyEvent::MAX_ERRORS_REACHED:
case StandbyEvent::FATAL_ERROR:
result.allowed = true;
result.new_state = StandbyState::FAILED;
break;
case StandbyEvent::STOP:
result.allowed = true;
result.new_state = StandbyState::STOPPED;
break;
default:
break;
}
break;
case StandbyState::PROMOTING:
switch (event) {
case StandbyEvent::PROMOTION_SUCCESS:
result.allowed = true;
result.new_state = StandbyState::PROMOTED;
break;
case StandbyEvent::PROMOTION_FAILED:
result.allowed = true;
result.new_state = StandbyState::FAILED;
break;
case StandbyEvent::STOP:
result.allowed = true;
result.new_state = StandbyState::STOPPED;
break;
default:
break;
}
break;
case StandbyState::PROMOTED:
if (event == StandbyEvent::STOP) {
result.allowed = true;
result.new_state = StandbyState::STOPPED;
}
break;
case StandbyState::FAILED:
if (event == StandbyEvent::STOP) {
result.allowed = true;
result.new_state = StandbyState::STOPPED;
} else if (event == StandbyEvent::START) {
// Allow restart from FAILED state
result.allowed = true;
result.new_state = StandbyState::CONNECTING;
}
break;
}
if (!result.allowed) {
result.reason = std::string("Invalid transition from ") +
StandbyStateToString(from) + " on event " +
StandbyEventToString(event);
}
return result;
}
StateTransitionResult StandbyStateMachine::ProcessEvent(StandbyEvent event) {
StandbyState old_state = current_state_.load(std::memory_order_acquire);
StateTransitionResult result = ValidateTransition(old_state, event);
std::vector<StateChangeCallback> callbacks_copy;
if (result.allowed && result.new_state != old_state) {
std::lock_guard<std::mutex> lock(mutex_);
// Double-check state hasn't changed (compare-and-swap pattern)
StandbyState current = current_state_.load(std::memory_order_acquire);
if (current != old_state) {
// State changed by another thread, re-validate
result = ValidateTransition(current, event);
old_state = current;
result.old_state = current;
if (!result.allowed || result.new_state == old_state) {
return result;
}
}
// Record transition
TransitionRecord record;
record.timestamp = std::chrono::steady_clock::now();
record.from_state = old_state;
record.to_state = result.new_state;
record.event = event;
transition_history_.push_back(record);
if (transition_history_.size() > kMaxHistorySize) {
transition_history_.erase(transition_history_.begin());
}
// Update state
current_state_.store(result.new_state, std::memory_order_release);
state_enter_time_ = record.timestamp;
LOG(INFO) << "Standby state transition: "
<< StandbyStateToString(old_state) << " -> "
<< StandbyStateToString(result.new_state)
<< " (event: " << StandbyEventToString(event) << ")";
// Copy callbacks while holding the lock; invoke them after releasing
// the lock to avoid deadlock (callbacks may re-enter ProcessEvent).
callbacks_copy = callbacks_;
} else if (!result.allowed) {
VLOG(1) << "Standby state transition rejected: " << result.reason;
}
// Notify callbacks outside the lock to avoid deadlock when callbacks
// re-enter ProcessEvent (e.g. IncrementErrors -> MAX_ERRORS_REACHED).
for (const auto& callback : callbacks_copy) {
if (callback) {
callback(old_state, result.new_state, event);
}
}
return result;
}
void StandbyStateMachine::RegisterCallback(StateChangeCallback callback) {
std::lock_guard<std::mutex> lock(mutex_);
callbacks_.push_back(std::move(callback));
}
std::vector<StandbyStateMachine::TransitionRecord>
StandbyStateMachine::GetTransitionHistory(size_t max_records) const {
std::lock_guard<std::mutex> lock(mutex_);
if (transition_history_.size() <= max_records) {
return transition_history_;
}
return std::vector<TransitionRecord>(
transition_history_.end() - max_records, transition_history_.end());
}
std::chrono::milliseconds StandbyStateMachine::GetTimeInCurrentState() const {
std::lock_guard<std::mutex> lock(mutex_);
auto now = std::chrono::steady_clock::now();
return std::chrono::duration_cast<std::chrono::milliseconds>(
now - state_enter_time_);
}
int StandbyStateMachine::IncrementErrors() {
int new_count = consecutive_errors_.fetch_add(1) + 1;
if (new_count >= kMaxConsecutiveErrors) {
// Trigger MAX_ERRORS_REACHED event
ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED);
}
return new_count;
}
} // namespace mooncake

104
mooncake-store/tests/CMakeLists.txt
View File

@ -1,17 +1,34 @@
function(add_store_test name)
add_executable(${name} ${ARGN})
target_link_libraries(${name} PUBLIC
mooncake_store
transfer_engine
cachelib_memory_allocator
${ETCD_WRAPPER_LIB}
glog
ibverbs
gtest
gtest_main
pthread
)
add_test(NAME ${name} COMMAND ${name})
add_executable(${name} ${ARGN})
target_link_libraries(
${name}
PUBLIC mooncake_store
transfer_engine
cachelib_memory_allocator
${ETCD_WRAPPER_LIB}
glog
ibverbs
gtest
gtest_main
pthread)
add_test(NAME ${name} COMMAND ${name})
endfunction()
function(add_hot_standby_ut_test name)
add_executable(${name} ${ARGN})
target_link_libraries(
${name}
PUBLIC mooncake_store
transfer_engine
cachelib_memory_allocator
${ETCD_WRAPPER_LIB}
glog
gflags
ibverbs
gtest
gtest_main
pthread)
add_test(NAME ${name} COMMAND ${name})
endfunction()
add_store_test(buffer_allocator_test buffer_allocator_test.cpp)
@ -19,9 +36,12 @@ add_store_test(allocation_strategy_test allocation_strategy_test.cpp)
add_store_test(eviction_strategy_test eviction_strategy_test.cpp)
add_store_test(master_service_test master_service_test.cpp)
add_store_test(master_service_ssd_test master_service_ssd_test.cpp)
add_store_test(master_service_ssd_test_for_snapshot master_service_ssd_test_for_snapshot.cpp)
add_store_test(master_service_ssd_test_for_snapshot
master_service_ssd_test_for_snapshot.cpp)
add_store_test(client_integration_test client_integration_test.cpp)
add_store_test(cxl_client_integration_test cxl_client_integration_test.cpp)
if(USE_CXL)
add_store_test(cxl_client_integration_test cxl_client_integration_test.cpp)
endif()
add_store_test(master_metrics_test master_metrics_test.cpp)
add_store_test(posix_file_test posix_file_test.cpp)
add_store_test(thread_pool_test thread_pool_test.cpp)
@ -39,7 +59,8 @@ add_store_test(zstd_util_test zstd_util_test.cpp)
add_store_test(local_file_backend_test local_file_backend_test.cpp)
add_store_test(file_util_test file_util_test.cpp)
add_store_test(snapshot_child_process_test snapshot_child_process_test.cpp)
add_store_test(master_service_test_for_snapshot master_service_test_for_snapshot.cpp)
add_store_test(master_service_test_for_snapshot
master_service_test_for_snapshot.cpp)
add_store_test(non_ha_reconnect_test non_ha_reconnect_test.cpp)
add_store_test(storage_backend_test storage_backend_test.cpp)
add_store_test(mutex_test mutex_test.cpp)
@ -52,19 +73,44 @@ add_store_test(health_check_test health_check_test.cpp)
add_subdirectory(e2e)
add_executable(high_availability_test high_availability_test.cpp)
target_link_libraries(high_availability_test PUBLIC mooncake_store transfer_engine cachelib_memory_allocator ${ETCD_WRAPPER_LIB} glog gtest gtest_main pthread)
if (STORE_USE_ETCD)
add_test(NAME high_availability_test COMMAND high_availability_test)
target_link_libraries(
high_availability_test
PUBLIC mooncake_store
transfer_engine
cachelib_memory_allocator
${ETCD_WRAPPER_LIB}
glog
gtest
gtest_main
pthread)
if(STORE_USE_ETCD)
add_test(NAME high_availability_test COMMAND high_availability_test)
endif()
add_executable(stress_workload_test stress_workload_test.cpp)
target_link_libraries(stress_workload_test PUBLIC
mooncake_store
transfer_engine
cachelib_memory_allocator
${ETCD_WRAPPER_LIB}
mooncake_common
glog
gflags
pthread
)
target_link_libraries(
stress_workload_test
PUBLIC mooncake_store
transfer_engine
cachelib_memory_allocator
${ETCD_WRAPPER_LIB}
mooncake_common
glog
gflags
pthread)
# Hot standby UTs (tests/hot_standby_ut)
add_hot_standby_ut_test(standby_state_machine_test
hot_standby_ut/standby_state_machine_test.cpp)
add_hot_standby_ut_test(oplog_applier_test
hot_standby_ut/oplog_applier_test.cpp)
add_hot_standby_ut_test(oplog_manager_test
hot_standby_ut/oplog_manager_test.cpp)
add_hot_standby_ut_test(oplog_watcher_test
hot_standby_ut/oplog_watcher_test.cpp)
add_hot_standby_ut_test(ha_metric_manager_test
hot_standby_ut/ha_metric_manager_test.cpp)
add_hot_standby_ut_test(hot_standby_service_test
hot_standby_ut/hot_standby_service_test.cpp)
add_hot_standby_ut_test(etcd_oplog_store_test
hot_standby_ut/etcd_oplog_store_test.cpp)

406
mooncake-store/tests/hot_standby_ut/etcd_oplog_store_test.cpp Normal file
View File

@ -0,0 +1,406 @@
#include "etcd_oplog_store.h"
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <chrono>
#include <string>
#include <thread>
#include <vector>
#include "etcd_helper.h"
DEFINE_string(etcd_endpoints, "0.0.0.0:2379",
"Etcd endpoints for EtcdOpLogStoreTest");
namespace mooncake::test {
class EtcdOpLogStoreTest : public ::testing::Test {
protected:
static void SetUpTestSuite() {
#ifdef STORE_USE_ETCD
google::InitGoogleLogging("EtcdOpLogStoreTest");
FLAGS_logtostderr = 1;
ASSERT_EQ(ErrorCode::OK,
EtcdHelper::ConnectToEtcdStoreClient(FLAGS_etcd_endpoints))
<< "Failed to connect to etcd at " << FLAGS_etcd_endpoints;
#endif
}
static void TearDownTestSuite() {
#ifdef STORE_USE_ETCD
google::ShutdownGoogleLogging();
#endif
}
void SetUp() override {
#ifndef STORE_USE_ETCD
GTEST_SKIP()
<< "STORE_USE_ETCD is disabled, skipping EtcdOpLogStore tests.";
#else
cluster_id_ = "test_cluster_etcd_oplog_store";
store_ = std::make_unique<EtcdOpLogStore>(
cluster_id_,
/*enable_latest_seq_batch_update=*/false,
/*enable_batch_write=*/true);
ASSERT_EQ(ErrorCode::OK, store_->Init());
CleanupTestData();
#endif
}
void TearDown() override {
#ifdef STORE_USE_ETCD
CleanupTestData();
store_.reset();
#endif
}
std::string cluster_id_;
std::unique_ptr<EtcdOpLogStore> store_;
void CleanupTestData() {
#ifdef STORE_USE_ETCD
// Delete all keys under /oplog/{cluster_id_}/ prefix
std::string prefix = std::string("/oplog/") + cluster_id_ + "/";
auto prefix_end = [](std::string p) -> std::string {
for (int i = static_cast<int>(p.size()) - 1; i >= 0; --i) {
unsigned char c = static_cast<unsigned char>(p[i]);
if (c < 0xFF) {
p[i] = static_cast<char>(c + 1);
p.resize(i + 1);
return p;
}
}
return std::string(1, '\0');
};
std::string end_key = prefix_end(prefix);
(void)EtcdHelper::DeleteRange(prefix.c_str(), prefix.size(),
end_key.c_str(), end_key.size());
#endif
}
static OpLogEntry MakeEntry(uint64_t seq, OpType type,
const std::string& key,
const std::string& payload) {
OpLogEntry e;
e.sequence_id = seq;
e.timestamp_ms = 123456;
e.op_type = type;
e.object_key = key;
e.payload = payload;
e.checksum = 0;
e.prefix_hash = 0;
return e;
}
};
// ========== 3.1.1 Basic CRUD tests ==========
TEST_F(EtcdOpLogStoreTest, TestWriteOpLog) {
OpLogEntry e = MakeEntry(1, OpType::PUT_END, "key1", "value1");
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e));
// Latest sequence ID should be updated to 1
uint64_t latest = 0;
ASSERT_EQ(ErrorCode::OK, store_->GetLatestSequenceId(latest));
EXPECT_EQ(1u, latest);
}
TEST_F(EtcdOpLogStoreTest, TestReadOpLog) {
OpLogEntry e = MakeEntry(2, OpType::PUT_END, "key2", "value2");
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e));
OpLogEntry out;
ASSERT_EQ(ErrorCode::OK, store_->ReadOpLog(2, out));
EXPECT_EQ(2u, out.sequence_id);
EXPECT_EQ(OpType::PUT_END, out.op_type);
EXPECT_EQ("key2", out.object_key);
EXPECT_EQ("value2", out.payload);
}
TEST_F(EtcdOpLogStoreTest, TestReadOpLogSince) {
// Write multiple entries
for (uint64_t i = 10; i < 15; ++i) {
OpLogEntry e = MakeEntry(i, OpType::PUT_END, "key_" + std::to_string(i),
"value_" + std::to_string(i));
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e));
}
std::vector<OpLogEntry> entries;
ASSERT_EQ(ErrorCode::OK, store_->ReadOpLogSince(11, 10, entries));
// Expect entries with seq > 11: 12,13,14
ASSERT_EQ(3u, entries.size());
EXPECT_EQ(12u, entries[0].sequence_id);
EXPECT_EQ(13u, entries[1].sequence_id);
EXPECT_EQ(14u, entries[2].sequence_id);
}
TEST_F(EtcdOpLogStoreTest, TestReadOpLogSince_Empty) {
std::vector<OpLogEntry> entries;
ASSERT_EQ(ErrorCode::OK, store_->ReadOpLogSince(1000, 10, entries));
EXPECT_TRUE(entries.empty());
}
TEST_F(EtcdOpLogStoreTest, TestReadOpLogSince_Limit) {
for (uint64_t i = 1; i <= 5; ++i) {
OpLogEntry e = MakeEntry(i, OpType::PUT_END, "key_" + std::to_string(i),
"value_" + std::to_string(i));
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e));
}
std::vector<OpLogEntry> entries;
ASSERT_EQ(ErrorCode::OK, store_->ReadOpLogSince(0, 3, entries));
ASSERT_EQ(3u, entries.size());
EXPECT_EQ(1u, entries[0].sequence_id);
EXPECT_EQ(2u, entries[1].sequence_id);
EXPECT_EQ(3u, entries[2].sequence_id);
}
// ========== 3.1.2 Serialization tests ==========
TEST_F(EtcdOpLogStoreTest, TestSerializeDeserializeRoundTrip) {
OpLogEntry in =
MakeEntry(42, OpType::PUT_END, "roundtrip-key", "roundtrip-value");
// Indirectly verify serialization / deserialization via WriteOpLog +
// ReadOpLog
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(in));
OpLogEntry out;
ASSERT_EQ(ErrorCode::OK, store_->ReadOpLog(42, out));
EXPECT_EQ(in.sequence_id, out.sequence_id);
EXPECT_EQ(in.op_type, out.op_type);
EXPECT_EQ(in.object_key, out.object_key);
EXPECT_EQ(in.payload, out.payload);
}
TEST_F(EtcdOpLogStoreTest, TestDeserializeInvalidJson) {
// Write invalid JSON directly into etcd; subsequent ReadOpLog should return
// INTERNAL_ERROR
std::string key = "/oplog/" + cluster_id_ + "/00000000000000000077";
std::string bad_json = "{ this is not valid json }";
ASSERT_EQ(ErrorCode::OK,
EtcdHelper::Put(key.c_str(), key.size(), bad_json.c_str(),
bad_json.size()));
OpLogEntry out;
ASSERT_EQ(ErrorCode::INTERNAL_ERROR, store_->ReadOpLog(77, out));
}
// ========== 3.1.3 Fencing tests ==========
TEST_F(EtcdOpLogStoreTest, TestWriteOpLog_Fencing) {
OpLogEntry e1 = MakeEntry(100, OpType::PUT_END, "key_fence", "value1");
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e1));
// Same seq, same content => idempotent (OK)
OpLogEntry e2 = e1;
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e2));
// NOTE: Duplicate sequence_id with different content is not a supported
// production scenario (sequence_id is monotonic). The batching write path
// does not guarantee conflict detection for that case, so we don't assert
// on it here.
}
TEST_F(EtcdOpLogStoreTest, TestWriteOpLog_Idempotent) {
OpLogEntry e = MakeEntry(200, OpType::PUT_END, "key_idem", "v");
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e));
// Repeatedly writing the exact same entry should return OK (idempotent)
// even if the underlying Create operation reports a transaction failure.
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e));
}
// ========== 3.1.4 Sequence ID management tests ==========
TEST_F(EtcdOpLogStoreTest, TestGetLatestSequenceId) {
OpLogEntry e1 = MakeEntry(1, OpType::PUT_END, "k1", "v1");
OpLogEntry e2 = MakeEntry(2, OpType::PUT_END, "k2", "v2");
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e1));
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e2));
uint64_t latest = 0;
ASSERT_EQ(ErrorCode::OK, store_->GetLatestSequenceId(latest));
EXPECT_EQ(2u, latest);
}
TEST_F(EtcdOpLogStoreTest, TestGetMaxSequenceIdAndEmpty) {
uint64_t max_seq = 0;
// Empty cluster: after cleanup, GetMaxSequenceId should return
// ETCD_KEY_NOT_EXIST
CleanupTestData();
EXPECT_EQ(ErrorCode::ETCD_KEY_NOT_EXIST, store_->GetMaxSequenceId(max_seq));
// After writing several entries, MaxSequenceId should equal the last
// entry's seq
for (uint64_t i = 10; i <= 15; ++i) {
OpLogEntry e = MakeEntry(i, OpType::PUT_END, "key_" + std::to_string(i),
"value_" + std::to_string(i));
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e));
}
ASSERT_EQ(ErrorCode::OK, store_->GetMaxSequenceId(max_seq));
EXPECT_EQ(15u, max_seq);
}
TEST_F(EtcdOpLogStoreTest, TestUpdateLatestSequenceId) {
// Directly call UpdateLatestSequenceId, then GetLatestSequenceId should
// match
ASSERT_EQ(ErrorCode::OK, store_->UpdateLatestSequenceId(12345));
uint64_t latest = 0;
ASSERT_EQ(ErrorCode::OK, store_->GetLatestSequenceId(latest));
EXPECT_EQ(12345u, latest);
}
// ========== 3.1.5 Batch update tests ==========
TEST_F(EtcdOpLogStoreTest, TestBatchUpdate_EnabledAndThreshold) {
// Use a store with batch enabled, then verify /latest is updated to the max
// seq
EtcdOpLogStore writer(cluster_id_,
/*enable_latest_seq_batch_update=*/true,
/*enable_batch_write=*/true);
ASSERT_EQ(ErrorCode::OK, writer.Init());
const uint64_t base_seq = 1000;
const int kEntries = 5;
for (int i = 0; i < kEntries; ++i) {
OpLogEntry e = MakeEntry(base_seq + i, OpType::PUT_END,
"batch_key_" + std::to_string(i),
"batch_val_" + std::to_string(i));
ASSERT_EQ(ErrorCode::OK, writer.WriteOpLog(e));
}
// Wait a short period to give the batch thread a chance to flush `/latest`
std::this_thread::sleep_for(std::chrono::milliseconds(2 * 1000));
uint64_t latest = 0;
ASSERT_EQ(ErrorCode::OK, store_->GetLatestSequenceId(latest));
EXPECT_EQ(base_seq + kEntries - 1, latest);
}
TEST_F(EtcdOpLogStoreTest, TestBatchUpdate_FailurePlaceholder) {
GTEST_SKIP() << "Batch failure scenarios are better tested with a "
"fault-injection etcd wrapper.";
}
// ========== 3.1.6 Cleanup tests ==========
TEST_F(EtcdOpLogStoreTest, TestCleanupOpLogBeforeAndBoundary) {
// Write seq 1..5
for (uint64_t i = 1; i <= 5; ++i) {
OpLogEntry e =
MakeEntry(i, OpType::PUT_END, "cleanup_key_" + std::to_string(i),
"cleanup_val_" + std::to_string(i));
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e));
}
// Cleanup seq < 3 => 1,2 should be deleted; 3,4,5 should remain
ASSERT_EQ(ErrorCode::OK, store_->CleanupOpLogBefore(3));
OpLogEntry out;
EXPECT_EQ(ErrorCode::ETCD_KEY_NOT_EXIST, store_->ReadOpLog(1, out));
EXPECT_EQ(ErrorCode::ETCD_KEY_NOT_EXIST, store_->ReadOpLog(2, out));
ASSERT_EQ(ErrorCode::OK, store_->ReadOpLog(3, out));
EXPECT_EQ(3u, out.sequence_id);
ASSERT_EQ(ErrorCode::OK, store_->ReadOpLog(5, out));
EXPECT_EQ(5u, out.sequence_id);
}
TEST_F(EtcdOpLogStoreTest, TestCleanupOpLogBefore_Empty) {
// Cleanup on an empty cluster should return OK
CleanupTestData();
EXPECT_EQ(ErrorCode::OK, store_->CleanupOpLogBefore(100));
}
// ========== 3.1.7 Cluster ID validation tests ==========
TEST_F(EtcdOpLogStoreTest, TestInvalidClusterId_Rejected) {
// Invalid cluster_id (containing slashes) should trigger LOG(FATAL) and
// terminate
EXPECT_DEATH(
{
EtcdOpLogStore bad_store("invalid/cluster", false);
(void)bad_store;
},
"Invalid cluster_id");
}
TEST_F(EtcdOpLogStoreTest, TestClusterIdNormalization) {
// Trailing slashes should be normalized away from the cluster_id
std::string raw_cluster = cluster_id_ + "///";
EtcdOpLogStore normalized_store(raw_cluster,
/*enable_latest_seq_batch_update=*/false,
/*enable_batch_write=*/true);
ASSERT_EQ(ErrorCode::OK, normalized_store.Init());
OpLogEntry e = MakeEntry(999, OpType::PUT_END, "norm-key", "norm-val");
ASSERT_EQ(ErrorCode::OK, normalized_store.WriteOpLog(e));
// Read the same seq via the current store_ to confirm the normalized
// cluster_id is used
OpLogEntry out;
ASSERT_EQ(ErrorCode::OK, store_->ReadOpLog(999, out));
EXPECT_EQ("norm-key", out.object_key);
EXPECT_EQ("norm-val", out.payload);
}
// ========== 3.1.8 Pagination tests ==========
TEST_F(EtcdOpLogStoreTest, TestReadOpLogSince_Pagination) {
// Write 20 entries and verify pagination via limit
for (uint64_t i = 1; i <= 20; ++i) {
OpLogEntry e =
MakeEntry(i, OpType::PUT_END, "page_key_" + std::to_string(i),
"page_val_" + std::to_string(i));
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e));
}
std::vector<OpLogEntry> entries;
ASSERT_EQ(ErrorCode::OK, store_->ReadOpLogSince(0, 20, entries));
ASSERT_EQ(20u, entries.size());
for (uint64_t i = 0; i < entries.size(); ++i) {
EXPECT_EQ(i + 1, entries[i].sequence_id);
}
}
TEST_F(EtcdOpLogStoreTest, TestReadOpLogSince_LargeDataset) {
// Write a larger number of entries to verify ReadOpLogSince returns the
// first N correctly
const uint64_t total = 200;
const uint64_t limit = 150;
CleanupTestData();
for (uint64_t i = 1; i <= total; ++i) {
OpLogEntry e =
MakeEntry(i, OpType::PUT_END, "large_key_" + std::to_string(i),
"large_val_" + std::to_string(i));
ASSERT_EQ(ErrorCode::OK, store_->WriteOpLog(e));
}
std::vector<OpLogEntry> entries;
ASSERT_EQ(ErrorCode::OK, store_->ReadOpLogSince(0, limit, entries));
ASSERT_EQ(limit, entries.size());
for (uint64_t i = 0; i < limit; ++i) {
EXPECT_EQ(i + 1, entries[i].sequence_id);
}
}
} // namespace mooncake::test
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

181
mooncake-store/tests/hot_standby_ut/ha_metric_manager_test.cpp Normal file
View File

@ -0,0 +1,181 @@
#include "ha_metric_manager.h"
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <atomic>
#include <chrono>
#include <string>
#include <thread>
namespace mooncake::test {
class HAMetricManagerTest : public ::testing::Test {
protected:
void SetUp() override {
google::InitGoogleLogging("HAMetricManagerTest");
FLAGS_logtostderr = 1;
}
void TearDown() override { google::ShutdownGoogleLogging(); }
HAMetricManager& M() { return HAMetricManager::instance(); }
};
// ========== 7.1.1 Metric update tests ==========
TEST_F(HAMetricManagerTest, TestSetOpLogLastSequenceId) {
M().set_oplog_last_sequence_id(123);
EXPECT_EQ(123, M().get_oplog_last_sequence_id());
}
TEST_F(HAMetricManagerTest, TestSetOpLogAppliedSequenceId) {
M().set_oplog_applied_sequence_id(456);
EXPECT_EQ(456, M().get_oplog_applied_sequence_id());
}
TEST_F(HAMetricManagerTest, TestSetOpLogStandbyLag) {
M().set_oplog_standby_lag(10);
EXPECT_EQ(10, M().get_oplog_standby_lag());
}
TEST_F(HAMetricManagerTest, TestSetOpLogPendingEntries) {
M().set_oplog_pending_entries(7);
EXPECT_EQ(7, M().get_oplog_pending_entries());
}
TEST_F(HAMetricManagerTest, TestSetPendingMutationQueueSize) {
M().set_pending_mutation_queue_size(5);
EXPECT_EQ(5, M().get_pending_mutation_queue_size());
}
TEST_F(HAMetricManagerTest, TestIncOpLogSkippedEntries) {
auto before = M().get_oplog_skipped_entries_total();
M().inc_oplog_skipped_entries();
EXPECT_EQ(before + 1, M().get_oplog_skipped_entries_total());
}
TEST_F(HAMetricManagerTest, TestIncOpLogChecksumFailures) {
auto before = M().get_oplog_checksum_failures_total();
M().inc_oplog_checksum_failures(2);
EXPECT_EQ(before + 2, M().get_oplog_checksum_failures_total());
}
TEST_F(HAMetricManagerTest, TestIncGapResolveCounters) {
auto before_attempts = M().get_oplog_gap_resolve_attempts_total();
auto before_success = M().get_oplog_gap_resolve_success_total();
M().inc_oplog_gap_resolve_attempts(3);
M().inc_oplog_gap_resolve_success(1);
EXPECT_EQ(before_attempts + 3, M().get_oplog_gap_resolve_attempts_total());
EXPECT_EQ(before_success + 1, M().get_oplog_gap_resolve_success_total());
}
TEST_F(HAMetricManagerTest, TestIncOpLogEtcdWriteFailuresAndRetries) {
auto before_failures = M().get_oplog_etcd_write_failures_total();
auto before_retries = M().get_oplog_etcd_write_retries_total();
M().inc_oplog_etcd_write_failures(4);
M().inc_oplog_etcd_write_retries(5);
EXPECT_EQ(before_failures + 4, M().get_oplog_etcd_write_failures_total());
EXPECT_EQ(before_retries + 5, M().get_oplog_etcd_write_retries_total());
}
TEST_F(HAMetricManagerTest, TestIncWatchDisconnectionsAndAppliedEntries) {
auto before_disc = M().get_oplog_watch_disconnections_total();
auto before_applied = M().get_oplog_applied_entries_total();
M().inc_oplog_watch_disconnections(2);
M().inc_oplog_applied_entries(10);
EXPECT_EQ(before_disc + 2, M().get_oplog_watch_disconnections_total());
EXPECT_EQ(before_applied + 10, M().get_oplog_applied_entries_total());
}
TEST_F(HAMetricManagerTest, TestRecordOpLogEtcdWriteLatency) {
// Call histogram observe functions, mainly to ensure they do not crash
M().observe_oplog_etcd_write_latency_us(100);
M().observe_oplog_etcd_write_latency_us(5000);
SUCCEED();
}
TEST_F(HAMetricManagerTest, TestRecordOpLogApplyLatency) {
M().observe_oplog_apply_latency_us(50);
M().observe_oplog_apply_latency_us(1000);
SUCCEED();
}
// ========== 7.1.2 Metric serialization tests ==========
TEST_F(HAMetricManagerTest, TestSerializeMetrics) {
M().set_oplog_last_sequence_id(1);
M().set_oplog_applied_sequence_id(1);
M().set_oplog_standby_lag(0);
std::string text = M().serialize_metrics();
EXPECT_FALSE(text.empty());
// Basic fields should appear in the Prometheus text output
EXPECT_NE(std::string::npos, text.find("ha_oplog_last_sequence_id"));
EXPECT_NE(std::string::npos, text.find("ha_oplog_applied_sequence_id"));
EXPECT_NE(std::string::npos, text.find("ha_oplog_standby_lag"));
}
TEST_F(HAMetricManagerTest, TestGetSummaryString) {
M().set_oplog_last_sequence_id(100);
M().set_oplog_applied_sequence_id(95);
M().set_oplog_standby_lag(5);
std::string summary = M().get_summary_string();
EXPECT_FALSE(summary.empty());
// Summary string should contain key fields
EXPECT_NE(std::string::npos, summary.find("last_seq"));
EXPECT_NE(std::string::npos, summary.find("applied_seq"));
}
// ========== 7.1.3 Singleton tests ==========
TEST_F(HAMetricManagerTest, TestSingletonInstance) {
HAMetricManager& a = HAMetricManager::instance();
HAMetricManager& b = HAMetricManager::instance();
EXPECT_EQ(&a, &b);
a.set_oplog_last_sequence_id(1234);
EXPECT_EQ(1234, b.get_oplog_last_sequence_id());
}
TEST_F(HAMetricManagerTest, TestConcurrentAccess) {
constexpr int kThreads = 8;
constexpr int kIncrementsPerThread = 1000;
auto& mgr = HAMetricManager::instance();
auto before = mgr.get_oplog_applied_entries_total();
std::vector<std::thread> threads;
threads.reserve(kThreads);
for (int i = 0; i < kThreads; ++i) {
threads.emplace_back([&mgr]() {
for (int j = 0; j < kIncrementsPerThread; ++j) {
mgr.inc_oplog_applied_entries();
}
});
}
for (auto& t : threads) {
t.join();
}
auto after = mgr.get_oplog_applied_entries_total();
EXPECT_EQ(before + kThreads * kIncrementsPerThread, after);
}
} // namespace mooncake::test
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

343
mooncake-store/tests/hot_standby_ut/hot_standby_service_test.cpp Normal file
View File

@ -0,0 +1,343 @@
#include "hot_standby_service.h"
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <chrono>
#include <memory>
#include <string>
#include <thread>
#include "master_service.h"
namespace mooncake::test {
class HotStandbyServiceTest : public ::testing::Test {
protected:
void SetUp() override {
google::InitGoogleLogging("HotStandbyServiceTest");
FLAGS_logtostderr = 1;
config_.enable_verification = false;
config_.max_replication_lag_entries = 1000;
service_ = std::make_unique<HotStandbyService>(config_);
etcd_endpoints_ = "http://localhost:2379";
cluster_id_ = "test_cluster_001";
}
void TearDown() override {
if (service_) {
service_->Stop();
}
google::ShutdownGoogleLogging();
}
HotStandbyConfig config_;
std::unique_ptr<HotStandbyService> service_;
std::string etcd_endpoints_;
std::string cluster_id_;
};
// ========== 6.1.1 Start/Stop tests ==========
TEST_F(HotStandbyServiceTest, TestStart) {
#ifdef STORE_USE_ETCD
// Requires a real etcd cluster and valid cluster configuration; acts as an
// integration placeholder
GTEST_SKIP()
<< "Requires real etcd connection, run in integration environment.";
#else
ErrorCode err =
service_->Start("primary_unused", etcd_endpoints_, cluster_id_);
EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err);
EXPECT_EQ(StandbyState::FAILED, service_->GetState());
#endif
}
TEST_F(HotStandbyServiceTest, TestStart_AlreadyRunning) {
#ifdef STORE_USE_ETCD
GTEST_SKIP()
<< "Requires real etcd connection to verify double start semantics.";
#else
// After the first Start fails and state becomes FAILED, the second Start
// should still return INTERNAL_ERROR
ErrorCode err1 =
service_->Start("primary_unused", etcd_endpoints_, cluster_id_);
EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err1);
ErrorCode err2 =
service_->Start("primary_unused", etcd_endpoints_, cluster_id_);
EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err2);
#endif
}
TEST_F(HotStandbyServiceTest, TestStart_InvalidEtcdEndpoints) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real etcd to simulate invalid endpoints.";
#else
std::string invalid_endpoints = "invalid_endpoint";
ErrorCode err =
service_->Start("primary_unused", invalid_endpoints, cluster_id_);
EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err);
#endif
}
TEST_F(HotStandbyServiceTest, TestStop) {
// Stop should be safe and idempotent even if Start was never called
service_->Stop();
SUCCEED();
}
TEST_F(HotStandbyServiceTest, TestStop_WhenNotRunning) {
// Multiple Stop calls should be idempotent
service_->Stop();
service_->Stop();
SUCCEED();
}
// ========== 6.1.2 State transition tests ==========
TEST_F(HotStandbyServiceTest, TestStateTransition_StartToWatching) {
#ifdef STORE_USE_ETCD
GTEST_SKIP()
<< "Requires real etcd to drive full state transition to WATCHING.";
#else
// In non-STORE_USE_ETCD builds, Start will set the state machine directly
// to FAILED
EXPECT_EQ(StandbyState::STOPPED, service_->GetState());
ErrorCode err =
service_->Start("primary_unused", etcd_endpoints_, cluster_id_);
EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err);
EXPECT_EQ(StandbyState::FAILED, service_->GetState());
#endif
}
TEST_F(HotStandbyServiceTest, TestStateTransition_ConnectionFailed) {
#ifdef STORE_USE_ETCD
GTEST_SKIP()
<< "Connection failure requires real etcd and invalid endpoints.";
#else
// In non-etcd mode we cannot distinguish detailed connection errors; only
// verify it doesn't crash
ErrorCode err =
service_->Start("primary_unused", "bad_endpoint", cluster_id_);
EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err);
#endif
}
TEST_F(HotStandbyServiceTest, TestStateTransition_SyncFailed) {
#ifdef STORE_USE_ETCD
GTEST_SKIP()
<< "Sync failure requires real etcd and OpLog watcher behavior.";
#else
// In non-etcd mode, the sync phase is not actually executed; just ensure
// the call is safe
ErrorCode err =
service_->Start("primary_unused", etcd_endpoints_, cluster_id_);
EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err);
#endif
}
// ========== 6.1.3 Sync status tests ==========
TEST_F(HotStandbyServiceTest, TestGetSyncStatus_InitialState) {
StandbySyncStatus status = service_->GetSyncStatus();
EXPECT_EQ(0u, status.applied_seq_id);
EXPECT_EQ(0u, status.primary_seq_id);
EXPECT_EQ(0u, status.lag_entries);
EXPECT_FALSE(status.is_syncing);
EXPECT_FALSE(status.is_connected);
EXPECT_EQ(StandbyState::STOPPED, status.state);
}
TEST_F(HotStandbyServiceTest, TestGetSyncStatus_AfterSync) {
#ifdef STORE_USE_ETCD
GTEST_SKIP()
<< "Requires real etcd and OpLog activity to change sync status.";
#else
// In non-etcd mode, calling Start will not change applied/primary, but the
// state machine enters FAILED
(void)service_->Start("primary_unused", etcd_endpoints_, cluster_id_);
StandbySyncStatus status = service_->GetSyncStatus();
EXPECT_EQ(StandbyState::FAILED, status.state);
#endif
}
TEST_F(HotStandbyServiceTest, TestGetSyncStatus) {
// Basic coverage: multiple calls should return consistent values and not
// crash
StandbySyncStatus s1 = service_->GetSyncStatus();
StandbySyncStatus s2 = service_->GetSyncStatus();
EXPECT_EQ(s1.state, s2.state);
}
// ========== 6.1.4 Promotion tests ==========
TEST_F(HotStandbyServiceTest, TestPromote_WhenNotReady) {
// In the initial state promotion preconditions are not met, so it should
// return an error code (not OK).
ErrorCode err = service_->Promote();
EXPECT_NE(ErrorCode::OK, err);
}
TEST_F(HotStandbyServiceTest, TestPromote_WhenReady) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real etcd and full replication pipeline to reach "
"ready state.";
#else
// Even in non-etcd mode, Promote should safely return OK (simulated
// success)
ErrorCode err = service_->Promote();
EXPECT_EQ(ErrorCode::OK, err);
#endif
}
TEST_F(HotStandbyServiceTest, TestPromote_FinalCatchUp) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real etcd and OpLog data to exercise final "
"catch-up logic.";
#else
ErrorCode err = service_->Promote();
EXPECT_EQ(ErrorCode::OK, err);
#endif
}
TEST_F(HotStandbyServiceTest, TestPromote_WithGaps) {
#ifdef STORE_USE_ETCD
GTEST_SKIP()
<< "Requires real etcd and gaps in OpLog to validate gap resolution.";
#else
ErrorCode err = service_->Promote();
EXPECT_EQ(ErrorCode::OK, err);
#endif
}
TEST_F(HotStandbyServiceTest, TestPromote_Timeout) {
#ifdef STORE_USE_ETCD
GTEST_SKIP()
<< "Requires real etcd and slow reads to trigger catch-up timeout.";
#else
ErrorCode err = service_->Promote();
EXPECT_EQ(ErrorCode::OK, err);
#endif
}
TEST_F(HotStandbyServiceTest, TestPromote_BatchLimit) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real etcd and large OpLog to hit batch limit.";
#else
ErrorCode err = service_->Promote();
EXPECT_NE(ErrorCode::OK, err);
#endif
}
// ========== 6.1.5 Warm start tests ==========
TEST_F(HotStandbyServiceTest, TestWarmStart_WithLocalState) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real etcd and pre-populated local metadata to "
"test warm start.";
#else
// In non-etcd mode, only verify that Start is safe to call
(void)service_->Start("primary_unused", etcd_endpoints_, cluster_id_);
SUCCEED();
#endif
}
TEST_F(HotStandbyServiceTest, TestWarmStart_WithoutLocalState) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real etcd and snapshot provider configuration.";
#else
(void)service_->Start("primary_unused", etcd_endpoints_, cluster_id_);
SUCCEED();
#endif
}
TEST_F(HotStandbyServiceTest, TestWarmStart_WithSnapshot) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires snapshot provider and real etcd to exercise "
"snapshot bootstrap.";
#else
config_.enable_snapshot_bootstrap = true;
// Recreate service to apply the new configuration
service_.reset(new HotStandbyService(config_));
(void)service_->Start("primary_unused", etcd_endpoints_, cluster_id_);
SUCCEED();
#endif
}
// ========== 6.1.6 Metadata operation tests ==========
TEST_F(HotStandbyServiceTest, TestGetMetadataCount) {
EXPECT_EQ(0u, service_->GetMetadataCount());
}
TEST_F(HotStandbyServiceTest, TestExportMetadataSnapshot) {
std::vector<std::pair<std::string, StandbyObjectMetadata>> snapshot;
EXPECT_TRUE(service_->ExportMetadataSnapshot(snapshot));
EXPECT_TRUE(snapshot.empty());
}
TEST_F(HotStandbyServiceTest, TestGetLatestAppliedSequenceId) {
uint64_t seq = service_->GetLatestAppliedSequenceId();
EXPECT_EQ(0u, seq);
}
// ========== 6.1.7 Replication loop tests ==========
TEST_F(HotStandbyServiceTest, TestReplicationLoop_UpdatesMetrics) {
#ifdef STORE_USE_ETCD
GTEST_SKIP()
<< "Requires real etcd and running replication loop to update metrics.";
#else
// In non-etcd mode, ReplicationLoop is never started, but calling Stop
// should be safe
service_->Stop();
SUCCEED();
#endif
}
TEST_F(HotStandbyServiceTest, TestReplicationLoop_HandlesDisconnect) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real etcd and watcher disconnect to exercise "
"disconnect path.";
#else
// DisconnectFromPrimary is private; verify Stop() is safe instead
service_->Stop();
SUCCEED();
#endif
}
// ========== 6.1.8 Verification loop tests ==========
TEST_F(HotStandbyServiceTest, TestVerificationLoop_WhenEnabled) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real etcd and running verification loop to "
"observe behavior.";
#else
config_.enable_verification = true;
service_.reset(new HotStandbyService(config_));
(void)service_->Start("primary_unused", etcd_endpoints_, cluster_id_);
service_->Stop();
SUCCEED();
#endif
}
TEST_F(HotStandbyServiceTest, TestVerificationLoop_WhenDisabled) {
// By default config_.enable_verification = false, so Start should not spawn
// a verification thread
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real etcd connection to start service.";
#else
(void)service_->Start("primary_unused", etcd_endpoints_, cluster_id_);
service_->Stop();
SUCCEED();
#endif
}
} // namespace mooncake::test
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

628
mooncake-store/tests/hot_standby_ut/oplog_applier_test.cpp Normal file
View File

@ -0,0 +1,628 @@
#include "oplog_applier.h"
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <chrono>
#include <map>
#include <memory>
#include <string>
#include <thread>
#include <vector>
#include <xxhash.h>
#include "etcd_oplog_store.h"
#include "metadata_store.h"
#include "oplog_manager.h"
#include "types.h"
namespace mooncake::test {
// Mock MetadataStore for testing OpLogApplier
class MockMetadataStore : public MetadataStore {
public:
MockMetadataStore() = default;
~MockMetadataStore() override = default;
bool PutMetadata(const std::string& key,
const StandbyObjectMetadata& metadata) override {
metadata_map_[key] = metadata;
return true;
}
bool Put(const std::string& key, const std::string& payload) override {
// For testing, we can use PutMetadata with empty metadata
StandbyObjectMetadata meta;
metadata_map_[key] = meta;
return true;
}
std::optional<StandbyObjectMetadata> GetMetadata(
const std::string& key) const override {
auto it = metadata_map_.find(key);
if (it != metadata_map_.end()) {
return it->second;
}
return std::nullopt;
}
bool Remove(const std::string& key) override {
auto it = metadata_map_.find(key);
if (it != metadata_map_.end()) {
metadata_map_.erase(it);
return true;
}
return false;
}
bool Exists(const std::string& key) const override {
return metadata_map_.find(key) != metadata_map_.end();
}
size_t GetKeyCount() const override { return metadata_map_.size(); }
// Test helper methods
void Clear() { metadata_map_.clear(); }
size_t Size() const { return metadata_map_.size(); }
private:
std::map<std::string, StandbyObjectMetadata> metadata_map_;
};
// Helper function to create a valid OpLogEntry with checksum
// Uses the same checksum algorithm as OpLogManager (XXH32)
OpLogEntry MakeEntry(uint64_t seq, OpType type, const std::string& key,
const std::string& payload) {
OpLogEntry e;
e.sequence_id = seq;
e.timestamp_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count();
e.op_type = type;
e.object_key = key;
e.payload = payload;
// Compute checksum and prefix_hash using the same algorithm as OpLogManager
e.checksum =
static_cast<uint32_t>(XXH32(payload.data(), payload.size(), 0));
e.prefix_hash =
key.empty() ? 0
: static_cast<uint32_t>(XXH32(key.data(), key.size(), 0));
return e;
}
// Helper function to create a valid JSON payload for PUT_END
std::string MakeValidJsonPayload(uint64_t client_id_first = 1,
uint64_t client_id_second = 2,
uint64_t size = 1024) {
// NOTE: OpLogApplier's current implementation expects PUT_END payload to be
// struct_pack-serialized MetadataPayload (msgpack binary), not JSON.
MetadataPayload payload;
payload.client_id = {client_id_first, client_id_second};
payload.size = size;
payload.replicas = {};
auto buf = struct_pack::serialize(payload);
return std::string(buf.data(), buf.size());
}
class OpLogApplierTest : public ::testing::Test {
protected:
void SetUp() override {
google::InitGoogleLogging("OpLogApplierTest");
FLAGS_logtostderr = 1;
mock_metadata_store_ = std::make_unique<MockMetadataStore>();
cluster_id_ = "test_cluster_001";
applier_ = std::make_unique<OpLogApplier>(mock_metadata_store_.get(),
cluster_id_);
}
void TearDown() override { google::ShutdownGoogleLogging(); }
std::unique_ptr<MockMetadataStore> mock_metadata_store_;
std::unique_ptr<OpLogApplier> applier_;
std::string cluster_id_;
};
// ========== 4.1.1 Basic apply tests ==========
TEST_F(OpLogApplierTest, TestApplyPutEnd) {
std::string payload = MakeValidJsonPayload();
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload);
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry));
// After applying seq=1, expected_sequence_id becomes 2
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
EXPECT_TRUE(mock_metadata_store_->Exists("key1"));
EXPECT_EQ(1u, mock_metadata_store_->GetKeyCount());
}
TEST_F(OpLogApplierTest, TestApplyPutRevoke) {
// First add a key
std::string payload = MakeValidJsonPayload();
OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload);
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1));
EXPECT_TRUE(mock_metadata_store_->Exists("key1"));
// Then revoke it
OpLogEntry entry2 = MakeEntry(2, OpType::PUT_REVOKE, "key1", "");
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry2));
EXPECT_EQ(3u, applier_->GetExpectedSequenceId());
EXPECT_FALSE(mock_metadata_store_->Exists("key1"));
}
TEST_F(OpLogApplierTest, TestApplyRemove) {
// First add a key
std::string payload = MakeValidJsonPayload();
OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload);
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1));
EXPECT_TRUE(mock_metadata_store_->Exists("key1"));
// Then remove it
OpLogEntry entry2 = MakeEntry(2, OpType::REMOVE, "key1", "");
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry2));
EXPECT_EQ(3u, applier_->GetExpectedSequenceId());
EXPECT_FALSE(mock_metadata_store_->Exists("key1"));
}
TEST_F(OpLogApplierTest, TestApplyOpLogEntry_InvalidOpType) {
OpLogEntry entry =
MakeEntry(1, OpType::PUT_END, "key1", MakeValidJsonPayload());
// Manually set an invalid op_type (assuming OpType is an enum)
// Since we can't directly set invalid enum, we test with valid types
// and verify that unsupported types in ProcessPendingEntries are handled
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry));
}
// ========== 4.1.2 Sequence ordering tests ==========
TEST_F(OpLogApplierTest, TestApplyInOrder) {
std::string payload = MakeValidJsonPayload();
OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload);
OpLogEntry entry2 = MakeEntry(2, OpType::PUT_END, "key2", payload);
OpLogEntry entry3 = MakeEntry(3, OpType::PUT_END, "key3", payload);
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry2));
EXPECT_EQ(3u, applier_->GetExpectedSequenceId());
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry3));
EXPECT_EQ(4u, applier_->GetExpectedSequenceId());
EXPECT_EQ(3u, mock_metadata_store_->GetKeyCount());
EXPECT_TRUE(mock_metadata_store_->Exists("key1"));
EXPECT_TRUE(mock_metadata_store_->Exists("key2"));
EXPECT_TRUE(mock_metadata_store_->Exists("key3"));
}
TEST_F(OpLogApplierTest, TestApplyOutOfOrder) {
std::string payload = MakeValidJsonPayload();
OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload);
OpLogEntry entry3 = MakeEntry(3, OpType::PUT_END, "key3", payload);
OpLogEntry entry2 = MakeEntry(2, OpType::PUT_END, "key2", payload);
// Apply entry1 (seq=1) - should succeed
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
// Apply entry3 (seq=3) - should be cached (out of order)
EXPECT_FALSE(applier_->ApplyOpLogEntry(entry3));
EXPECT_EQ(2u,
applier_->GetExpectedSequenceId()); // Still waiting for seq=2
EXPECT_FALSE(mock_metadata_store_->Exists("key3"));
// Apply entry2 (seq=2) - should succeed and trigger processing of entry3
// ApplyOpLogEntry internally calls ProcessPendingEntries(), so entry3
// should be processed
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry2));
EXPECT_EQ(4u, applier_->GetExpectedSequenceId()); // Now at seq=4
// entry3 should already be processed by ApplyOpLogEntry
EXPECT_TRUE(mock_metadata_store_->Exists("key1"));
EXPECT_TRUE(mock_metadata_store_->Exists("key2"));
EXPECT_TRUE(mock_metadata_store_->Exists("key3"));
// ProcessPendingEntries may return 0 if entry3 was already processed
(void)applier_->ProcessPendingEntries();
EXPECT_EQ(4u, applier_->GetExpectedSequenceId());
}
TEST_F(OpLogApplierTest, TestApplyWithGap) {
std::string payload = MakeValidJsonPayload();
OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload);
OpLogEntry entry4 = MakeEntry(4, OpType::PUT_END, "key4", payload);
// Apply entry1 (seq=1)
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
// Apply entry4 (seq=4) - gap at seq=2,3
EXPECT_FALSE(applier_->ApplyOpLogEntry(entry4));
EXPECT_EQ(2u,
applier_->GetExpectedSequenceId()); // Still waiting for seq=2
// Process pending entries - should detect gap and schedule wait
(void)applier_->ProcessPendingEntries();
// May process 0 entries if gap resolution is still waiting
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
}
TEST_F(OpLogApplierTest, TestApplyDuplicateSequenceId) {
std::string payload = MakeValidJsonPayload();
OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload);
OpLogEntry entry1_dup = MakeEntry(1, OpType::PUT_END, "key1_dup", payload);
// Apply entry1
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
// Try to apply duplicate sequence_id (older than expected)
// Should be treated as no-op (already applied) and return true
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1_dup));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
// key1_dup should not be added (treated as no-op)
EXPECT_FALSE(mock_metadata_store_->Exists("key1_dup"));
}
// ========== 4.1.3 Gap resolution tests ==========
TEST_F(OpLogApplierTest, TestRequestMissingOpLog_Success) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real EtcdOpLogStore, skipping integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogApplierTest, TestRequestMissingOpLog_Failure) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real EtcdOpLogStore, skipping integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogApplierTest, TestRequestMissingOpLog_Timeout) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real EtcdOpLogStore, skipping integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogApplierTest, TestTryResolveGapsOnceForPromotion) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real EtcdOpLogStore, skipping integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogApplierTest, TestGapResolution_Retry) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real EtcdOpLogStore, skipping integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
// ========== 4.1.4 Checksum tests ==========
TEST_F(OpLogApplierTest, TestApplyOpLogEntry_ValidChecksum) {
std::string payload = MakeValidJsonPayload();
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload);
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
EXPECT_TRUE(mock_metadata_store_->Exists("key1"));
}
TEST_F(OpLogApplierTest, TestApplyOpLogEntry_InvalidChecksum) {
std::string payload = MakeValidJsonPayload();
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload);
// Tamper with the checksum
entry.checksum = entry.checksum + 1;
EXPECT_FALSE(applier_->ApplyOpLogEntry(entry));
EXPECT_EQ(1u, applier_->GetExpectedSequenceId()); // Should not advance
EXPECT_FALSE(mock_metadata_store_->Exists("key1"));
}
TEST_F(OpLogApplierTest, TestChecksumFailureMetric) {
std::string payload = MakeValidJsonPayload();
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload);
// Tamper with the checksum
entry.checksum = entry.checksum + 1;
EXPECT_FALSE(applier_->ApplyOpLogEntry(entry));
// Metric increment is tested implicitly by the failure
}
// ========== 4.1.5 Size validation tests ==========
TEST_F(OpLogApplierTest, TestApplyOpLogEntry_ValidSize) {
std::string payload = MakeValidJsonPayload();
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload);
EXPECT_TRUE(OpLogManager::ValidateEntrySize(entry));
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry));
}
TEST_F(OpLogApplierTest, TestApplyOpLogEntry_InvalidSize) {
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "");
// Make key too large
entry.object_key.assign(OpLogManager::kMaxObjectKeySize + 1, 'k');
EXPECT_FALSE(OpLogManager::ValidateEntrySize(entry));
EXPECT_FALSE(applier_->ApplyOpLogEntry(entry));
EXPECT_EQ(1u, applier_->GetExpectedSequenceId()); // Should not advance
EXPECT_FALSE(mock_metadata_store_->Exists("key1"));
}
TEST_F(OpLogApplierTest, TestApplyOpLogEntry_PayloadTooLarge) {
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "");
// Make payload too large
entry.payload.assign(OpLogManager::kMaxPayloadSize + 1, 'p');
EXPECT_FALSE(OpLogManager::ValidateEntrySize(entry));
EXPECT_FALSE(applier_->ApplyOpLogEntry(entry));
EXPECT_EQ(1u, applier_->GetExpectedSequenceId()); // Should not advance
}
// ========== 4.1.6 Recovery tests ==========
TEST_F(OpLogApplierTest, TestRecover) {
// Set initial state: last applied sequence_id = 10
applier_->Recover(10);
EXPECT_EQ(11u, applier_->GetExpectedSequenceId());
// Apply entry with seq=11 should succeed
std::string payload = MakeValidJsonPayload();
OpLogEntry entry = MakeEntry(11, OpType::PUT_END, "key1", payload);
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry));
EXPECT_EQ(12u, applier_->GetExpectedSequenceId());
}
TEST_F(OpLogApplierTest, TestRecover_ZeroSequenceId) {
// Recover from sequence_id 0
applier_->Recover(0);
EXPECT_EQ(1u, applier_->GetExpectedSequenceId());
// Apply entry with seq=1 should succeed
std::string payload = MakeValidJsonPayload();
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload);
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
}
TEST_F(OpLogApplierTest, TestRecover_AfterGap) {
std::string payload = MakeValidJsonPayload();
OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload);
OpLogEntry entry3 = MakeEntry(3, OpType::PUT_END, "key3", payload);
// Apply entry1
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
// Apply entry3 (creates gap)
EXPECT_FALSE(applier_->ApplyOpLogEntry(entry3));
// Recover from seq=3 (skip the gap)
applier_->Recover(3);
EXPECT_EQ(4u, applier_->GetExpectedSequenceId());
// Now entry3 should be processable
(void)applier_->ProcessPendingEntries();
// entry3 should be in pending, but expected_seq is now 4, so it won't be
// processed This tests that recovery resets the expected sequence
}
// ========== 4.1.7 Pending entries tests ==========
TEST_F(OpLogApplierTest, TestProcessPendingEntries) {
std::string payload = MakeValidJsonPayload();
OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload);
OpLogEntry entry3 = MakeEntry(3, OpType::PUT_END, "key3", payload);
OpLogEntry entry2 = MakeEntry(2, OpType::PUT_END, "key2", payload);
// Apply entry1
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
// Apply entry3 (out of order)
EXPECT_FALSE(applier_->ApplyOpLogEntry(entry3));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
// Process pending - should not process entry3 yet (waiting for seq=2)
size_t processed1 = applier_->ProcessPendingEntries();
EXPECT_EQ(0u, processed1);
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
// Apply entry2 - this will internally call ProcessPendingEntries() and
// process entry3
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry2));
EXPECT_EQ(4u, applier_->GetExpectedSequenceId());
// entry3 should already be processed by ApplyOpLogEntry
EXPECT_TRUE(mock_metadata_store_->Exists("key1"));
EXPECT_TRUE(mock_metadata_store_->Exists("key2"));
EXPECT_TRUE(mock_metadata_store_->Exists("key3"));
// ProcessPendingEntries may return 0 if entry3 was already processed
(void)applier_->ProcessPendingEntries();
EXPECT_EQ(4u, applier_->GetExpectedSequenceId());
}
TEST_F(OpLogApplierTest, TestPendingEntriesTimeout) {
std::string payload = MakeValidJsonPayload();
OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload);
OpLogEntry entry3 = MakeEntry(3, OpType::PUT_END, "key3", payload);
// Apply entry1
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
// Apply entry3 (creates gap at seq=2)
EXPECT_FALSE(applier_->ApplyOpLogEntry(entry3));
// Process pending entries multiple times to trigger timeout
// After kMissingEntrySkipSeconds (3s), the gap should be skipped
size_t processed = 0;
for (int i = 0; i < 10; ++i) {
processed = applier_->ProcessPendingEntries();
if (processed > 0 || applier_->GetExpectedSequenceId() > 2) {
break;
}
std::this_thread::sleep_for(std::chrono::milliseconds(500));
}
// After timeout, gap should be skipped and entry3 should be processed
// Note: This test may be flaky due to timing, but it tests the timeout
// logic
EXPECT_GE(applier_->GetExpectedSequenceId(), 2u);
}
TEST_F(OpLogApplierTest, TestPendingEntriesSkip) {
std::string payload = MakeValidJsonPayload();
OpLogEntry entry1 = MakeEntry(1, OpType::PUT_END, "key1", payload);
OpLogEntry entry4 = MakeEntry(4, OpType::PUT_END, "key4", payload);
// Apply entry1
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry1));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
// Apply entry4 (creates gap at seq=2,3)
EXPECT_FALSE(applier_->ApplyOpLogEntry(entry4));
// Process pending entries to trigger skip logic
// After timeout (3 seconds), gaps should be skipped
for (int i = 0; i < 10; ++i) {
applier_->ProcessPendingEntries();
uint64_t expected = applier_->GetExpectedSequenceId();
if (expected >= 3) { // Gap at seq=2 is skipped, expected becomes 3
break;
}
std::this_thread::sleep_for(std::chrono::milliseconds(500));
}
// After skip, expected_seq should advance to 3 (gap at seq=2 is skipped)
// entry4 is still pending, waiting for seq=3
EXPECT_GE(applier_->GetExpectedSequenceId(), 3u);
// entry4 should still be pending (not applied yet)
EXPECT_FALSE(mock_metadata_store_->Exists("key4"));
}
// ========== 4.1.8 JSON parsing tests ==========
TEST_F(OpLogApplierTest, TestApplyPutEnd_ValidJson) {
std::string payload = MakeValidJsonPayload(1, 2, 2048);
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload);
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
EXPECT_TRUE(mock_metadata_store_->Exists("key1"));
auto meta = mock_metadata_store_->GetMetadata("key1");
ASSERT_TRUE(meta.has_value());
EXPECT_EQ(1u, meta->client_id.first);
EXPECT_EQ(2u, meta->client_id.second);
EXPECT_EQ(2048u, meta->size);
}
TEST_F(OpLogApplierTest, TestApplyPutEnd_InvalidJson) {
std::string invalid_json = "{invalid json}";
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", invalid_json);
// Should still succeed (fallback to empty metadata)
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
EXPECT_TRUE(mock_metadata_store_->Exists("key1"));
// Metadata should exist but with default values
auto meta = mock_metadata_store_->GetMetadata("key1");
ASSERT_TRUE(meta.has_value());
EXPECT_EQ(0u, meta->client_id.first);
EXPECT_EQ(0u, meta->client_id.second);
EXPECT_EQ(0u, meta->size);
}
TEST_F(OpLogApplierTest, TestApplyPutEnd_EmptyPayload) {
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "");
// Should succeed with empty payload (creates empty metadata)
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
EXPECT_TRUE(mock_metadata_store_->Exists("key1"));
auto meta = mock_metadata_store_->GetMetadata("key1");
ASSERT_TRUE(meta.has_value());
EXPECT_EQ(0u, meta->client_id.first);
EXPECT_EQ(0u, meta->client_id.second);
EXPECT_EQ(0u, meta->size);
}
// ========== Additional Edge Case Tests ==========
TEST_F(OpLogApplierTest, TestApplyOpLogEntries_Batch) {
std::string payload = MakeValidJsonPayload();
std::vector<OpLogEntry> entries;
entries.push_back(MakeEntry(1, OpType::PUT_END, "key1", payload));
entries.push_back(MakeEntry(2, OpType::PUT_END, "key2", payload));
entries.push_back(MakeEntry(3, OpType::PUT_END, "key3", payload));
size_t applied = applier_->ApplyOpLogEntries(entries);
EXPECT_EQ(3u, applied);
EXPECT_EQ(4u, applier_->GetExpectedSequenceId());
EXPECT_EQ(3u, mock_metadata_store_->GetKeyCount());
}
TEST_F(OpLogApplierTest, TestApplyOpLogEntries_WithGaps) {
std::string payload = MakeValidJsonPayload();
std::vector<OpLogEntry> entries;
entries.push_back(MakeEntry(1, OpType::PUT_END, "key1", payload));
entries.push_back(
MakeEntry(3, OpType::PUT_END, "key3", payload)); // Gap at seq=2
entries.push_back(MakeEntry(2, OpType::PUT_END, "key2", payload));
size_t applied = applier_->ApplyOpLogEntries(entries);
// entry1 should be applied, entry3 should be pending, entry2 should be
// applied and trigger processing of entry3
EXPECT_GE(applied, 2u); // entry1 and entry2 are applied
EXPECT_LE(applied, 3u);
// entry2's ApplyOpLogEntry internally calls ProcessPendingEntries(), so
// entry3 should be processed
EXPECT_GE(applier_->GetExpectedSequenceId(), 4u);
EXPECT_TRUE(mock_metadata_store_->Exists("key1"));
EXPECT_TRUE(mock_metadata_store_->Exists("key2"));
EXPECT_TRUE(mock_metadata_store_->Exists("key3"));
// ProcessPendingEntries may return 0 if entry3 was already processed
(void)applier_->ProcessPendingEntries();
EXPECT_GE(applier_->GetExpectedSequenceId(), 4u);
}
TEST_F(OpLogApplierTest, TestGetExpectedSequenceId) {
EXPECT_EQ(1u, applier_->GetExpectedSequenceId());
std::string payload = MakeValidJsonPayload();
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", payload);
EXPECT_TRUE(applier_->ApplyOpLogEntry(entry));
EXPECT_EQ(2u, applier_->GetExpectedSequenceId());
}
TEST_F(OpLogApplierTest, TestGetKeySequenceId_Deprecated) {
// This method is deprecated and always returns 0
EXPECT_EQ(0u, applier_->GetKeySequenceId("any_key"));
}
} // namespace mooncake::test
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

291
mooncake-store/tests/hot_standby_ut/oplog_manager_test.cpp Normal file
View File

@ -0,0 +1,291 @@
#include "oplog_manager.h"
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <algorithm>
#include <limits>
#include <mutex>
#include <string>
#include <thread>
#include <vector>
namespace mooncake::test {
class OpLogManagerTest : public ::testing::Test {
protected:
void SetUp() override {
google::InitGoogleLogging("OpLogManagerTest");
FLAGS_logtostderr = 1;
manager_ = std::make_unique<OpLogManager>();
}
void TearDown() override { google::ShutdownGoogleLogging(); }
OpLogManager& M() { return *manager_; }
std::unique_ptr<OpLogManager> manager_;
};
// ========== 2.1.1 Basic functionality tests ==========
TEST_F(OpLogManagerTest, TestAppendEntry) {
uint64_t id = M().Append(OpType::PUT_END, "key1", "value1");
EXPECT_LT(0u, id);
EXPECT_EQ(id, M().GetLastSequenceId());
EXPECT_EQ(1u, M().GetEntryCount());
}
TEST_F(OpLogManagerTest, TestSequenceIdIncrement) {
uint64_t id1 = M().Append(OpType::PUT_END, "key1", "value1");
uint64_t id2 = M().Append(OpType::PUT_END, "key2", "value2");
uint64_t id3 = M().Append(OpType::REMOVE, "key3", "");
EXPECT_LT(0u, id1);
EXPECT_EQ(id1 + 1, id2);
EXPECT_EQ(id2 + 1, id3);
EXPECT_EQ(id3, M().GetLastSequenceId());
EXPECT_EQ(3u, M().GetEntryCount());
}
TEST_F(OpLogManagerTest, TestAllocateEntry) {
OpLogEntry e1 = M().AllocateEntry(OpType::PUT_END, "key1", "value1");
OpLogEntry e2 = M().AllocateEntry(OpType::PUT_END, "key2", "value2");
EXPECT_LT(0u, e1.sequence_id);
EXPECT_EQ(e1.sequence_id + 1, e2.sequence_id);
EXPECT_EQ(e2.sequence_id, M().GetLastSequenceId());
EXPECT_EQ(2u, M().GetEntryCount());
// Basic field validation
EXPECT_EQ(OpType::PUT_END, e1.op_type);
EXPECT_EQ("key1", e1.object_key);
EXPECT_EQ("value1", e1.payload);
EXPECT_NE(0u, e1.timestamp_ms);
EXPECT_NE(0u, e1.checksum);
EXPECT_NE(0u, e1.prefix_hash);
}
TEST_F(OpLogManagerTest, TestPersistEntryToEtcd) {
// Without an EtcdOpLogStore configured, PersistEntryToEtcd should return an
// error
OpLogEntry entry =
M().AllocateEntry(OpType::PUT_END, "key", "payload-data");
ErrorCode err = M().PersistEntryToEtcd(entry);
EXPECT_EQ(ErrorCode::ETCD_OPERATION_ERROR, err);
}
TEST_F(OpLogManagerTest, TestAppendAndPersist) {
// Without an EtcdOpLogStore configured, AppendAndPersist should return an
// error
auto res = M().AppendAndPersist(OpType::REMOVE, "key", "");
ASSERT_FALSE(res.has_value());
EXPECT_EQ(ErrorCode::ETCD_OPERATION_ERROR, res.error());
}
// ========== Initial Sequence Id Tests ==========
TEST_F(OpLogManagerTest, SetInitialSequenceIdOnEmptyManager) {
EXPECT_EQ(0u, M().GetLastSequenceId());
EXPECT_EQ(0u, M().GetEntryCount());
M().SetInitialSequenceId(100);
EXPECT_EQ(100u, M().GetLastSequenceId());
// The first appended entry should have sequence_id 101
uint64_t id = M().Append(OpType::PUT_END, "key", "value");
EXPECT_EQ(101u, id);
}
TEST_F(OpLogManagerTest, SetInitialSequenceIdIgnoredWhenNotEmpty) {
uint64_t id1 = M().Append(OpType::PUT_END, "key1", "value1");
EXPECT_EQ(id1, M().GetLastSequenceId());
// Setting initial sequence id on a non-empty manager should be ignored
M().SetInitialSequenceId(500);
EXPECT_EQ(id1, M().GetLastSequenceId());
}
// ========== 2.1.2 Checksum tests ==========
TEST_F(OpLogManagerTest, TestChecksumComputation) {
// Same payload => same checksum; different payload => different checksum
OpLogEntry e1 = M().AllocateEntry(OpType::PUT_END, "k1", "payload-X");
OpLogEntry e2 = M().AllocateEntry(OpType::PUT_END, "k2", "payload-X");
OpLogEntry e3 = M().AllocateEntry(OpType::PUT_END, "k3", "payload-Y");
EXPECT_EQ(e1.checksum, e2.checksum);
EXPECT_NE(e1.checksum, e3.checksum);
}
TEST_F(OpLogManagerTest, TestPrefixHashComputation) {
// Same key => same prefix_hash; different key => (with high probability)
// different prefix_hash
OpLogEntry e1 = M().AllocateEntry(OpType::PUT_END, "same-key", "v1");
OpLogEntry e2 = M().AllocateEntry(OpType::PUT_END, "same-key", "v2");
OpLogEntry e3 = M().AllocateEntry(OpType::PUT_END, "other-key", "v3");
EXPECT_EQ(e1.prefix_hash, e2.prefix_hash);
EXPECT_NE(e1.prefix_hash, e3.prefix_hash);
}
TEST_F(OpLogManagerTest, TestVerifyChecksum) {
OpLogEntry entry =
M().AllocateEntry(OpType::PUT_END, "key", "payload-data");
EXPECT_TRUE(OpLogManager::VerifyChecksum(entry));
// Verification should fail after tampering with the payload
entry.payload = "tampered";
EXPECT_FALSE(OpLogManager::VerifyChecksum(entry));
}
// ========== 2.1.3 Size validation tests ==========
TEST_F(OpLogManagerTest, TestValidateEntrySize_Valid) {
OpLogEntry entry;
entry.object_key = "normal-key";
entry.payload = "small-payload";
std::string reason;
EXPECT_TRUE(OpLogManager::ValidateEntrySize(entry, &reason));
EXPECT_TRUE(reason.empty());
}
TEST_F(OpLogManagerTest, TestValidateEntrySize_KeyTooLarge) {
OpLogEntry entry;
entry.object_key.assign(OpLogManager::kMaxObjectKeySize + 1, 'k');
entry.payload = "payload";
std::string reason;
EXPECT_FALSE(OpLogManager::ValidateEntrySize(entry, &reason));
EXPECT_FALSE(reason.empty());
}
TEST_F(OpLogManagerTest, TestValidateEntrySize_PayloadTooLarge) {
OpLogEntry entry;
entry.object_key = "key";
entry.payload.assign(OpLogManager::kMaxPayloadSize + 1, 'p');
std::string reason;
EXPECT_FALSE(OpLogManager::ValidateEntrySize(entry, &reason));
EXPECT_FALSE(reason.empty());
}
TEST_F(OpLogManagerTest, TestValidateEntrySize_EmptyKey) {
// Current implementation only enforces upper bounds; empty keys are
// accepted
OpLogEntry entry;
entry.object_key = "";
entry.payload = "payload";
std::string reason;
EXPECT_TRUE(OpLogManager::ValidateEntrySize(entry, &reason));
}
// ========== 2.1.4 Etcd integration tests (placeholder) ==========
TEST_F(OpLogManagerTest, TestWriteToEtcd_Success) {
#if defined(STORE_USE_ETCD)
GTEST_SKIP()
<< "TODO: requires real EtcdOpLogStore and running etcd cluster.";
#else
GTEST_SKIP() << "STORE_USE_ETCD is disabled.";
#endif
}
TEST_F(OpLogManagerTest, TestWriteToEtcd_Failure) {
OpLogEntry entry =
M().AllocateEntry(OpType::PUT_END, "key", "payload-data");
ErrorCode err = M().PersistEntryToEtcd(entry);
EXPECT_EQ(ErrorCode::ETCD_OPERATION_ERROR, err);
}
TEST_F(OpLogManagerTest, TestWriteToEtcd_Retry) {
GTEST_SKIP() << "TODO: retry / idempotent semantics are tested at "
"EtcdOpLogStore level.";
}
TEST_F(OpLogManagerTest, TestIdempotentWrite) {
GTEST_SKIP() << "TODO: idempotent write belongs to "
"EtcdOpLogStore::WriteOpLog tests.";
}
// ========== 2.1.5 Boundary condition tests ==========
TEST_F(OpLogManagerTest, TestSequenceIdWrapAround) {
// Theoretical wrap-around test: set initial value near UINT64_MAX and
// verify wrap-around semantics
uint64_t near_max = std::numeric_limits<uint64_t>::max() - 2;
M().SetInitialSequenceId(near_max);
std::vector<uint64_t> ids;
ids.push_back(M().Append(OpType::PUT_END, "k1", "v1")); // max-1
ids.push_back(M().Append(OpType::PUT_END, "k2", "v2")); // max
ids.push_back(M().Append(OpType::PUT_END, "k3", "v3")); // 0 (wrap)
ASSERT_EQ(3u, ids.size());
// Use wrap-around-safe comparison helpers to verify monotonic increase
EXPECT_TRUE(IsSequenceNewer(ids[1], ids[0]));
EXPECT_TRUE(IsSequenceNewer(
ids[2], ids[1])); // 0 is considered newer than UINT64_MAX
}
TEST_F(OpLogManagerTest, TestConcurrentAppend) {
constexpr int kThreads = 8;
constexpr int kPerThread = 1000;
std::vector<uint64_t> ids;
ids.reserve(kThreads * kPerThread);
std::mutex m;
auto worker = [&]() {
for (int i = 0; i < kPerThread; ++i) {
uint64_t id = M().Append(OpType::PUT_END, "key", "value");
std::lock_guard<std::mutex> lock(m);
ids.push_back(id);
}
};
std::vector<std::thread> threads;
for (int i = 0; i < kThreads; ++i) {
threads.emplace_back(worker);
}
for (auto& t : threads) {
t.join();
}
EXPECT_EQ(static_cast<size_t>(kThreads * kPerThread), ids.size());
std::sort(ids.begin(), ids.end());
// Ensure there are no duplicates and sequence IDs are strictly increasing
for (size_t i = 1; i < ids.size(); ++i) {
EXPECT_GT(ids[i], ids[i - 1]);
}
}
TEST_F(OpLogManagerTest, TestLargePayload) {
// Construct a payload close to the upper limit and verify it passes
// validation and appends successfully
std::string key = "large-payload-key";
std::string payload(OpLogManager::kMaxPayloadSize - 1, 'x');
OpLogEntry entry;
entry.object_key = key;
entry.payload = payload;
std::string reason;
EXPECT_TRUE(OpLogManager::ValidateEntrySize(entry, &reason));
uint64_t id = M().Append(OpType::PUT_END, key, payload);
EXPECT_LT(0u, id);
EXPECT_EQ(1u, M().GetEntryCount());
}
} // namespace mooncake::test
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

426
mooncake-store/tests/hot_standby_ut/oplog_watcher_test.cpp Normal file
View File

@ -0,0 +1,426 @@
#include "oplog_watcher.h"
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <atomic>
#include <chrono>
#include <memory>
#include <string>
#include <thread>
#include <vector>
#include <xxhash.h>
#if __has_include(<jsoncpp/json/json.h>)
#include <jsoncpp/json/json.h>
#else
#include <json/json.h>
#endif
#include "metadata_store.h"
#include "oplog_applier.h"
#include "oplog_manager.h"
#include "standby_state_machine.h"
#include "types.h"
#include "etcd_oplog_store.h"
namespace mooncake::test {
// Minimal MetadataStore implementation for OpLogApplier
class MinimalMockMetadataStore : public MetadataStore {
public:
bool PutMetadata(const std::string&,
const StandbyObjectMetadata&) override {
return true;
}
bool Put(const std::string&, const std::string&) override { return true; }
std::optional<StandbyObjectMetadata> GetMetadata(
const std::string&) const override {
return std::nullopt;
}
bool Remove(const std::string&) override { return true; }
bool Exists(const std::string&) const override { return false; }
size_t GetKeyCount() const override { return 0; }
};
// Simple wrapper around OpLogApplier for testing OpLogWatcher.
// We don't override any methods since OpLogApplier's methods are not virtual;
// we just provide a valid instance to OpLogWatcher.
class MockOpLogApplier : public OpLogApplier {
public:
MockOpLogApplier() : OpLogApplier(&metadata_store_, "test_cluster") {}
private:
MinimalMockMetadataStore metadata_store_;
};
// Helper function to create a valid OpLogEntry with checksum
OpLogEntry MakeEntry(uint64_t seq, OpType type, const std::string& key,
const std::string& payload) {
OpLogEntry e;
e.sequence_id = seq;
e.timestamp_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count();
e.op_type = type;
e.object_key = key;
e.payload = payload;
e.checksum =
static_cast<uint32_t>(XXH32(payload.data(), payload.size(), 0));
e.prefix_hash =
key.empty() ? 0
: static_cast<uint32_t>(XXH32(key.data(), key.size(), 0));
return e;
}
// Helper function to serialize OpLogEntry to JSON (same as EtcdOpLogStore)
std::string SerializeOpLogEntry(const OpLogEntry& entry) {
Json::Value root;
root["sequence_id"] = static_cast<Json::UInt64>(entry.sequence_id);
root["timestamp_ms"] = static_cast<Json::UInt64>(entry.timestamp_ms);
root["op_type"] = static_cast<int>(entry.op_type);
root["object_key"] = entry.object_key;
root["payload"] = entry.payload;
root["checksum"] = static_cast<Json::UInt>(entry.checksum);
root["prefix_hash"] = static_cast<Json::UInt>(entry.prefix_hash);
Json::StreamWriterBuilder builder;
builder["indentation"] = ""; // Compact format
std::unique_ptr<Json::StreamWriter> writer(builder.newStreamWriter());
std::ostringstream oss;
writer->write(root, &oss);
return oss.str();
}
class OpLogWatcherTest : public ::testing::Test {
protected:
void SetUp() override {
google::InitGoogleLogging("OpLogWatcherTest");
FLAGS_logtostderr = 1;
etcd_endpoints_ = "http://localhost:2379";
cluster_id_ = "test_cluster_001";
mock_applier_ = std::make_unique<MockOpLogApplier>();
watcher_ = std::make_unique<OpLogWatcher>(etcd_endpoints_, cluster_id_,
mock_applier_.get());
}
void TearDown() override {
if (watcher_) {
watcher_->Stop();
}
google::ShutdownGoogleLogging();
}
std::string etcd_endpoints_;
std::string cluster_id_;
std::unique_ptr<MockOpLogApplier> mock_applier_;
std::unique_ptr<OpLogWatcher> watcher_;
};
// ========== 5.1.1 Start/Stop tests ==========
TEST_F(OpLogWatcherTest, TestStart) {
#ifdef STORE_USE_ETCD
// This test requires a real etcd connection
GTEST_SKIP() << "Requires real etcd connection, skipping integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogWatcherTest, TestStartFromSequenceId) {
#ifdef STORE_USE_ETCD
// This test requires a real etcd connection
GTEST_SKIP() << "Requires real etcd connection, skipping integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogWatcherTest, TestStop) {
#ifdef STORE_USE_ETCD
// Stop without starting should be safe
watcher_->Stop();
EXPECT_FALSE(watcher_->IsWatchHealthy());
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogWatcherTest, TestStartWhenAlreadyRunning) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real etcd connection, skipping integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
// ========== 5.1.2 Watch event handling tests ==========
TEST_F(OpLogWatcherTest, TestHandleWatchEvent_Put) {
#ifdef STORE_USE_ETCD
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "payload1");
std::string json_value = SerializeOpLogEntry(entry);
std::string key = "/oplog/" + cluster_id_ + "/00000000000000000001";
// Use reflection to call HandleWatchEvent (it's private, so we test via
// public interface) For unit testing, we can't directly call
// HandleWatchEvent, so we skip this test and rely on integration tests
GTEST_SKIP() << "HandleWatchEvent is private, requires integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogWatcherTest, TestHandleWatchEvent_Delete) {
#ifdef STORE_USE_ETCD
// DELETE events are handled but don't apply entries
GTEST_SKIP() << "HandleWatchEvent is private, requires integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogWatcherTest, TestHandleWatchEvent_InvalidEntry) {
#ifdef STORE_USE_ETCD
// Invalid JSON should be rejected
std::string invalid_json = "{invalid json}";
std::string key = "/oplog/" + cluster_id_ + "/00000000000000000001";
// Can't directly test HandleWatchEvent, requires integration test
GTEST_SKIP() << "HandleWatchEvent is private, requires integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogWatcherTest, TestHandleWatchEvent_OutOfOrder) {
#ifdef STORE_USE_ETCD
// Out-of-order entries should be handled by OpLogApplier
GTEST_SKIP() << "HandleWatchEvent is private, requires integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
// ========== 5.1.3 Reconnection tests ==========
TEST_F(OpLogWatcherTest, TestReconnectAfterDisconnect) {
#ifdef STORE_USE_ETCD
GTEST_SKIP()
<< "Requires real etcd connection and watch failure simulation";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogWatcherTest, TestReconnectWithStateCallback) {
#ifdef STORE_USE_ETCD
std::atomic<bool> callback_called{false};
StandbyEvent received_event = StandbyEvent::START;
watcher_->SetStateCallback([&](StandbyEvent event) {
callback_called.store(true);
received_event = event;
});
// State callbacks are triggered during watch operations
// Requires integration test with real etcd
GTEST_SKIP() << "Requires real etcd connection for state callback testing";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogWatcherTest, TestReconnectResumeFromLastSequence) {
#ifdef STORE_USE_ETCD
GTEST_SKIP() << "Requires real etcd connection and reconnection simulation";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
// ========== 5.1.4 Checksum verification tests ==========
TEST_F(OpLogWatcherTest, TestHandleWatchEvent_ValidChecksum) {
#ifdef STORE_USE_ETCD
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "payload1");
std::string json_value = SerializeOpLogEntry(entry);
// Valid checksum should pass validation
// Can't directly test HandleWatchEvent, requires integration test
GTEST_SKIP() << "HandleWatchEvent is private, requires integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogWatcherTest, TestHandleWatchEvent_InvalidChecksum) {
#ifdef STORE_USE_ETCD
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "payload1");
std::string json_value = SerializeOpLogEntry(entry);
// Tamper with checksum in JSON
// Replace checksum value in JSON string
size_t pos = json_value.find("\"checksum\":");
if (pos != std::string::npos) {
size_t start = pos + 10; // length of "checksum":
size_t end = json_value.find_first_of(",}", start);
if (end != std::string::npos) {
json_value.replace(start, end - start, "999999");
}
}
// Invalid checksum should be rejected
// Can't directly test HandleWatchEvent, requires integration test
GTEST_SKIP() << "HandleWatchEvent is private, requires integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
// ========== 5.1.5 Size validation tests ==========
TEST_F(OpLogWatcherTest, TestHandleWatchEvent_ValidSize) {
#ifdef STORE_USE_ETCD
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "payload1");
std::string json_value = SerializeOpLogEntry(entry);
// Valid size should pass validation
EXPECT_TRUE(OpLogManager::ValidateEntrySize(entry));
GTEST_SKIP() << "HandleWatchEvent is private, requires integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogWatcherTest, TestHandleWatchEvent_InvalidSize) {
#ifdef STORE_USE_ETCD
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "payload1");
// Make key too large
entry.object_key.assign(OpLogManager::kMaxObjectKeySize + 1, 'k');
std::string json_value = SerializeOpLogEntry(entry);
// Invalid size should be rejected
EXPECT_FALSE(OpLogManager::ValidateEntrySize(entry));
GTEST_SKIP() << "HandleWatchEvent is private, requires integration test";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
// ========== 5.1.6 State callback tests ==========
TEST_F(OpLogWatcherTest, TestStateCallback_WatchHealthy) {
#ifdef STORE_USE_ETCD
std::atomic<bool> callback_called{false};
StandbyEvent received_event = StandbyEvent::START;
watcher_->SetStateCallback([&](StandbyEvent event) {
callback_called.store(true);
received_event = event;
});
// Watch healthy events are triggered during normal watch operations
// Requires integration test with real etcd
GTEST_SKIP() << "Requires real etcd connection for state callback testing";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogWatcherTest, TestStateCallback_WatchBroken) {
#ifdef STORE_USE_ETCD
std::atomic<bool> callback_called{false};
StandbyEvent received_event = StandbyEvent::START;
watcher_->SetStateCallback([&](StandbyEvent event) {
callback_called.store(true);
received_event = event;
});
// Watch broken events are triggered when watch fails
// Requires integration test with real etcd and watch failure
GTEST_SKIP()
<< "Requires real etcd connection and watch failure simulation";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
// ========== 5.1.7 Cluster ID validation tests ==========
TEST_F(OpLogWatcherTest, TestInvalidClusterId_Rejected) {
// Invalid cluster_id should cause LOG(FATAL) in constructor
// We can't test this directly as it would terminate the process
// But we can verify that valid cluster_id works
std::string valid_cluster_id = "test_cluster_001";
std::unique_ptr<MockOpLogApplier> mock_applier =
std::make_unique<MockOpLogApplier>();
std::unique_ptr<OpLogWatcher> watcher = std::make_unique<OpLogWatcher>(
etcd_endpoints_, valid_cluster_id, mock_applier.get());
EXPECT_NE(nullptr, watcher);
watcher->Stop();
}
TEST_F(OpLogWatcherTest, TestGetLastProcessedSequenceId) {
#ifdef STORE_USE_ETCD
// Initially should be 0
EXPECT_EQ(0u, watcher_->GetLastProcessedSequenceId());
// After processing entries, should be updated
// This requires integration test with real etcd
GTEST_SKIP() << "Requires real etcd connection for sequence ID tracking";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
TEST_F(OpLogWatcherTest, TestIsWatchHealthy) {
#ifdef STORE_USE_ETCD
// Initially should be false (not started)
EXPECT_FALSE(watcher_->IsWatchHealthy());
// After starting and successful watch, should be true
// This requires integration test with real etcd
GTEST_SKIP() << "Requires real etcd connection for watch health testing";
#else
GTEST_SKIP() << "STORE_USE_ETCD not enabled";
#endif
}
// ========== Additional Helper Tests ==========
TEST_F(OpLogWatcherTest, TestSerializeOpLogEntry) {
OpLogEntry entry = MakeEntry(1, OpType::PUT_END, "key1", "payload1");
std::string json = SerializeOpLogEntry(entry);
// Verify JSON contains expected fields
EXPECT_NE(std::string::npos, json.find("sequence_id"));
EXPECT_NE(std::string::npos, json.find("op_type"));
EXPECT_NE(std::string::npos, json.find("object_key"));
EXPECT_NE(std::string::npos, json.find("payload"));
EXPECT_NE(std::string::npos, json.find("checksum"));
EXPECT_NE(std::string::npos, json.find("prefix_hash"));
}
TEST_F(OpLogWatcherTest, TestMakeEntry) {
OpLogEntry entry =
MakeEntry(100, OpType::REMOVE, "test_key", "test_payload");
EXPECT_EQ(100u, entry.sequence_id);
EXPECT_EQ(OpType::REMOVE, entry.op_type);
EXPECT_EQ("test_key", entry.object_key);
EXPECT_EQ("test_payload", entry.payload);
EXPECT_NE(0u, entry.checksum);
EXPECT_NE(0u, entry.prefix_hash);
EXPECT_NE(0u, entry.timestamp_ms);
}
} // namespace mooncake::test
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

731
mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp Normal file
View File

@ -0,0 +1,731 @@
#include "standby_state_machine.h"
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <atomic>
#include <chrono>
#include <thread>
#include <vector>
namespace mooncake::test {
class StandbyStateMachineTest : public ::testing::Test {
protected:
void SetUp() override {
google::InitGoogleLogging("StandbyStateMachineTest");
FLAGS_logtostderr = true;
machine_ = std::make_unique<StandbyStateMachine>();
}
void TearDown() override { google::ShutdownGoogleLogging(); }
std::unique_ptr<StandbyStateMachine> machine_;
// Helper function to reach WATCHING state
void ReachWatchingState() {
machine_->ProcessEvent(StandbyEvent::START);
machine_->ProcessEvent(StandbyEvent::CONNECTED);
machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE);
EXPECT_EQ(StandbyState::WATCHING, machine_->GetState());
}
// Helper function to reach SYNCING state
void ReachSyncingState() {
machine_->ProcessEvent(StandbyEvent::START);
machine_->ProcessEvent(StandbyEvent::CONNECTED);
EXPECT_EQ(StandbyState::SYNCING, machine_->GetState());
}
};
// ========== Initial State Tests ==========
TEST_F(StandbyStateMachineTest, TestInitialState) {
EXPECT_EQ(StandbyState::STOPPED, machine_->GetState());
EXPECT_FALSE(machine_->IsRunning());
EXPECT_FALSE(machine_->IsConnected());
EXPECT_FALSE(machine_->IsWatchHealthy());
EXPECT_FALSE(machine_->IsReadyForPromotion());
EXPECT_EQ(0, machine_->GetConsecutiveErrors());
EXPECT_EQ(0, machine_->GetReconnectCount());
}
// ========== Basic State Transition Tests ==========
TEST_F(StandbyStateMachineTest, TestStartTransition) {
auto result = machine_->ProcessEvent(StandbyEvent::START);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::STOPPED, result.old_state);
EXPECT_EQ(StandbyState::CONNECTING, result.new_state);
EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState());
// In CONNECTING state, syncing has not actually started yet, so
// IsRunning/IsConnected should both be false
EXPECT_FALSE(machine_->IsRunning());
EXPECT_FALSE(machine_->IsConnected());
}
TEST_F(StandbyStateMachineTest, TestConnectedTransition) {
machine_->ProcessEvent(StandbyEvent::START);
EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState());
auto result = machine_->ProcessEvent(StandbyEvent::CONNECTED);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::CONNECTING, result.old_state);
EXPECT_EQ(StandbyState::SYNCING, result.new_state);
EXPECT_EQ(StandbyState::SYNCING, machine_->GetState());
EXPECT_TRUE(machine_->IsRunning());
EXPECT_TRUE(machine_->IsConnected());
}
TEST_F(StandbyStateMachineTest, TestSyncCompleteTransition) {
ReachSyncingState();
auto result = machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::SYNCING, result.old_state);
EXPECT_EQ(StandbyState::WATCHING, result.new_state);
EXPECT_EQ(StandbyState::WATCHING, machine_->GetState());
EXPECT_TRUE(machine_->IsRunning());
EXPECT_TRUE(machine_->IsConnected());
EXPECT_TRUE(machine_->IsWatchHealthy());
EXPECT_TRUE(machine_->IsReadyForPromotion());
}
TEST_F(StandbyStateMachineTest, TestWatchHealthyNoOp) {
ReachWatchingState();
// WATCH_HEALTHY in WATCHING state is a no-op (stays in WATCHING)
auto result = machine_->ProcessEvent(StandbyEvent::WATCH_HEALTHY);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::WATCHING, result.old_state);
EXPECT_EQ(StandbyState::WATCHING, result.new_state);
EXPECT_EQ(StandbyState::WATCHING, machine_->GetState());
}
TEST_F(StandbyStateMachineTest, TestWatchBrokenTransition) {
ReachWatchingState();
auto result = machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::WATCHING, result.old_state);
EXPECT_EQ(StandbyState::RECONNECTING, result.new_state);
EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState());
EXPECT_TRUE(machine_->IsRunning());
EXPECT_FALSE(machine_->IsWatchHealthy());
EXPECT_FALSE(machine_->IsReadyForPromotion());
}
TEST_F(StandbyStateMachineTest, TestDisconnectedFromWatching) {
ReachWatchingState();
auto result = machine_->ProcessEvent(StandbyEvent::DISCONNECTED);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::WATCHING, result.old_state);
EXPECT_EQ(StandbyState::RECONNECTING, result.new_state);
EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState());
}
TEST_F(StandbyStateMachineTest, TestPromoteTransition) {
ReachWatchingState();
auto result = machine_->ProcessEvent(StandbyEvent::PROMOTE);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::WATCHING, result.old_state);
EXPECT_EQ(StandbyState::PROMOTING, result.new_state);
EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState());
EXPECT_TRUE(machine_->IsRunning());
EXPECT_TRUE(machine_->IsConnected());
EXPECT_FALSE(machine_->IsWatchHealthy());
EXPECT_FALSE(machine_->IsReadyForPromotion());
}
TEST_F(StandbyStateMachineTest, TestPromotionSuccessTransition) {
ReachWatchingState();
machine_->ProcessEvent(StandbyEvent::PROMOTE);
EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState());
auto result = machine_->ProcessEvent(StandbyEvent::PROMOTION_SUCCESS);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::PROMOTING, result.old_state);
EXPECT_EQ(StandbyState::PROMOTED, result.new_state);
EXPECT_EQ(StandbyState::PROMOTED, machine_->GetState());
EXPECT_FALSE(machine_->IsRunning());
EXPECT_FALSE(machine_->IsConnected());
}
TEST_F(StandbyStateMachineTest, TestPromotionFailedTransition) {
ReachWatchingState();
machine_->ProcessEvent(StandbyEvent::PROMOTE);
EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState());
auto result = machine_->ProcessEvent(StandbyEvent::PROMOTION_FAILED);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::PROMOTING, result.old_state);
EXPECT_EQ(StandbyState::FAILED, result.new_state);
EXPECT_EQ(StandbyState::FAILED, machine_->GetState());
EXPECT_FALSE(machine_->IsRunning());
}
TEST_F(StandbyStateMachineTest, TestStopTransition) {
ReachWatchingState();
auto result = machine_->ProcessEvent(StandbyEvent::STOP);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::WATCHING, result.old_state);
EXPECT_EQ(StandbyState::STOPPED, result.new_state);
EXPECT_EQ(StandbyState::STOPPED, machine_->GetState());
EXPECT_FALSE(machine_->IsRunning());
EXPECT_FALSE(machine_->IsConnected());
}
// ========== Error and Failure State Tests ==========
TEST_F(StandbyStateMachineTest, TestConnectionFailedFromConnecting) {
machine_->ProcessEvent(StandbyEvent::START);
EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState());
auto result = machine_->ProcessEvent(StandbyEvent::CONNECTION_FAILED);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::CONNECTING, result.old_state);
EXPECT_EQ(StandbyState::FAILED, result.new_state);
EXPECT_EQ(StandbyState::FAILED, machine_->GetState());
EXPECT_FALSE(machine_->IsRunning());
}
TEST_F(StandbyStateMachineTest, TestFatalErrorFromConnecting) {
machine_->ProcessEvent(StandbyEvent::START);
EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState());
auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::CONNECTING, result.old_state);
EXPECT_EQ(StandbyState::FAILED, result.new_state);
EXPECT_EQ(StandbyState::FAILED, machine_->GetState());
}
TEST_F(StandbyStateMachineTest, TestSyncFailedFromSyncing) {
ReachSyncingState();
auto result = machine_->ProcessEvent(StandbyEvent::SYNC_FAILED);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::SYNCING, result.old_state);
EXPECT_EQ(StandbyState::RECONNECTING, result.new_state);
EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState());
EXPECT_TRUE(machine_->IsRunning());
}
TEST_F(StandbyStateMachineTest, TestDisconnectedFromSyncing) {
ReachSyncingState();
auto result = machine_->ProcessEvent(StandbyEvent::DISCONNECTED);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::SYNCING, result.old_state);
EXPECT_EQ(StandbyState::RECONNECTING, result.new_state);
EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState());
}
TEST_F(StandbyStateMachineTest, TestFatalErrorFromSyncing) {
ReachSyncingState();
auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::SYNCING, result.old_state);
EXPECT_EQ(StandbyState::FAILED, result.new_state);
EXPECT_EQ(StandbyState::FAILED, machine_->GetState());
}
TEST_F(StandbyStateMachineTest, TestFatalErrorFromWatching) {
ReachWatchingState();
auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::WATCHING, result.old_state);
EXPECT_EQ(StandbyState::FAILED, result.new_state);
EXPECT_EQ(StandbyState::FAILED, machine_->GetState());
}
// ========== Reconnecting State Tests ==========
TEST_F(StandbyStateMachineTest, TestReconnectingToSyncing) {
ReachWatchingState();
machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN);
EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState());
auto result = machine_->ProcessEvent(StandbyEvent::CONNECTED);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::RECONNECTING, result.old_state);
EXPECT_EQ(StandbyState::SYNCING, result.new_state);
EXPECT_EQ(StandbyState::SYNCING, machine_->GetState());
}
TEST_F(StandbyStateMachineTest, TestReconnectingToFailed) {
ReachWatchingState();
machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN);
EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState());
auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::RECONNECTING, result.old_state);
EXPECT_EQ(StandbyState::FAILED, result.new_state);
EXPECT_EQ(StandbyState::FAILED, machine_->GetState());
}
TEST_F(StandbyStateMachineTest, TestReconnectingMaxErrors) {
ReachWatchingState();
machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN);
EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState());
auto result = machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::RECONNECTING, result.old_state);
EXPECT_EQ(StandbyState::FAILED, result.new_state);
EXPECT_EQ(StandbyState::FAILED, machine_->GetState());
}
// ========== Recovering State Tests ==========
TEST_F(StandbyStateMachineTest, TestRecoveringToWatching) {
ReachWatchingState();
machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED);
EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState());
auto result = machine_->ProcessEvent(StandbyEvent::RECOVERY_SUCCESS);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::RECOVERING, result.old_state);
EXPECT_EQ(StandbyState::WATCHING, result.new_state);
EXPECT_EQ(StandbyState::WATCHING, machine_->GetState());
}
TEST_F(StandbyStateMachineTest, TestRecoveringToReconnecting) {
ReachWatchingState();
machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED);
EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState());
auto result = machine_->ProcessEvent(StandbyEvent::RECOVERY_FAILED);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::RECOVERING, result.old_state);
EXPECT_EQ(StandbyState::RECONNECTING, result.new_state);
EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState());
}
TEST_F(StandbyStateMachineTest, TestRecoveringDisconnected) {
ReachWatchingState();
machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED);
EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState());
auto result = machine_->ProcessEvent(StandbyEvent::DISCONNECTED);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::RECOVERING, result.old_state);
EXPECT_EQ(StandbyState::RECONNECTING, result.new_state);
EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState());
}
TEST_F(StandbyStateMachineTest, TestRecoveringFatalError) {
ReachWatchingState();
machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED);
EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState());
auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::RECOVERING, result.old_state);
EXPECT_EQ(StandbyState::FAILED, result.new_state);
EXPECT_EQ(StandbyState::FAILED, machine_->GetState());
}
// ========== Failed State Tests ==========
TEST_F(StandbyStateMachineTest, TestFailedToStopped) {
machine_->ProcessEvent(StandbyEvent::START);
machine_->ProcessEvent(StandbyEvent::FATAL_ERROR);
EXPECT_EQ(StandbyState::FAILED, machine_->GetState());
auto result = machine_->ProcessEvent(StandbyEvent::STOP);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::FAILED, result.old_state);
EXPECT_EQ(StandbyState::STOPPED, result.new_state);
EXPECT_EQ(StandbyState::STOPPED, machine_->GetState());
}
TEST_F(StandbyStateMachineTest, TestFailedToConnecting) {
machine_->ProcessEvent(StandbyEvent::START);
machine_->ProcessEvent(StandbyEvent::FATAL_ERROR);
EXPECT_EQ(StandbyState::FAILED, machine_->GetState());
// Allow restart from FAILED state
auto result = machine_->ProcessEvent(StandbyEvent::START);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::FAILED, result.old_state);
EXPECT_EQ(StandbyState::CONNECTING, result.new_state);
EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState());
}
// ========== Promoted State Tests ==========
TEST_F(StandbyStateMachineTest, TestPromotedToStopped) {
ReachWatchingState();
machine_->ProcessEvent(StandbyEvent::PROMOTE);
machine_->ProcessEvent(StandbyEvent::PROMOTION_SUCCESS);
EXPECT_EQ(StandbyState::PROMOTED, machine_->GetState());
auto result = machine_->ProcessEvent(StandbyEvent::STOP);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::PROMOTED, result.old_state);
EXPECT_EQ(StandbyState::STOPPED, result.new_state);
EXPECT_EQ(StandbyState::STOPPED, machine_->GetState());
}
// ========== Invalid Transition Tests ==========
TEST_F(StandbyStateMachineTest, TestInvalidTransitions) {
// Cannot transition from STOPPED directly to WATCHING
auto result1 = machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE);
EXPECT_FALSE(result1.allowed);
EXPECT_EQ(StandbyState::STOPPED, machine_->GetState());
// Cannot promote when not in WATCHING state
ReachSyncingState();
auto result2 = machine_->ProcessEvent(StandbyEvent::PROMOTE);
EXPECT_FALSE(result2.allowed);
EXPECT_EQ(StandbyState::SYNCING, machine_->GetState());
// Cannot transition from STOPPED to CONNECTED
machine_->ProcessEvent(StandbyEvent::STOP);
auto result3 = machine_->ProcessEvent(StandbyEvent::CONNECTED);
EXPECT_FALSE(result3.allowed);
EXPECT_EQ(StandbyState::STOPPED, machine_->GetState());
}
// ========== Error Handling Tests ==========
TEST_F(StandbyStateMachineTest, TestConsecutiveErrors) {
ReachWatchingState();
// Simulate multiple errors
for (int i = 0; i < 5; ++i) {
machine_->IncrementErrors();
}
EXPECT_EQ(5, machine_->GetConsecutiveErrors());
// Reset errors
machine_->ResetErrors();
EXPECT_EQ(0, machine_->GetConsecutiveErrors());
}
TEST_F(StandbyStateMachineTest, TestMaxErrorsReachedAutoTransition) {
ReachWatchingState();
// IncrementErrors() automatically triggers MAX_ERRORS_REACHED when
// threshold is reached
for (int i = 0; i < StandbyStateMachine::kMaxConsecutiveErrors; ++i) {
machine_->IncrementErrors();
}
// Should have transitioned to RECOVERING (from WATCHING on
// MAX_ERRORS_REACHED)
EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState());
EXPECT_EQ(StandbyStateMachine::kMaxConsecutiveErrors,
machine_->GetConsecutiveErrors());
}
TEST_F(StandbyStateMachineTest, TestMaxErrorsReachedManual) {
ReachWatchingState();
// Manually trigger MAX_ERRORS_REACHED
auto result = machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::WATCHING, result.old_state);
EXPECT_EQ(StandbyState::RECOVERING, result.new_state);
EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState());
}
TEST_F(StandbyStateMachineTest, TestReconnectCount) {
EXPECT_EQ(0, machine_->GetReconnectCount());
machine_->IncrementReconnectCount();
EXPECT_EQ(1, machine_->GetReconnectCount());
machine_->IncrementReconnectCount();
EXPECT_EQ(2, machine_->GetReconnectCount());
machine_->ResetReconnectCount();
EXPECT_EQ(0, machine_->GetReconnectCount());
}
// ========== Callback Tests ==========
TEST_F(StandbyStateMachineTest, TestStateChangeCallback) {
std::vector<StandbyState> state_history;
std::vector<StandbyEvent> event_history;
machine_->RegisterCallback([&](StandbyState old_state,
StandbyState new_state, StandbyEvent event) {
state_history.push_back(new_state);
event_history.push_back(event);
});
// Trigger state transitions
machine_->ProcessEvent(StandbyEvent::START);
machine_->ProcessEvent(StandbyEvent::CONNECTED);
machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE);
// Verify callbacks were called
EXPECT_EQ(3, state_history.size());
EXPECT_EQ(StandbyState::CONNECTING, state_history[0]);
EXPECT_EQ(StandbyState::SYNCING, state_history[1]);
EXPECT_EQ(StandbyState::WATCHING, state_history[2]);
EXPECT_EQ(StandbyEvent::START, event_history[0]);
EXPECT_EQ(StandbyEvent::CONNECTED, event_history[1]);
EXPECT_EQ(StandbyEvent::SYNC_COMPLETE, event_history[2]);
}
TEST_F(StandbyStateMachineTest, TestMultipleCallbacks) {
int callback1_count = 0;
int callback2_count = 0;
machine_->RegisterCallback(
[&](StandbyState, StandbyState, StandbyEvent) { callback1_count++; });
machine_->RegisterCallback(
[&](StandbyState, StandbyState, StandbyEvent) { callback2_count++; });
// Trigger state transitions
machine_->ProcessEvent(StandbyEvent::START);
machine_->ProcessEvent(StandbyEvent::CONNECTED);
// Both callbacks should be called
EXPECT_EQ(2, callback1_count);
EXPECT_EQ(2, callback2_count);
}
TEST_F(StandbyStateMachineTest, TestCallbackExceptionHandling) {
bool callback_called = false;
machine_->RegisterCallback([&](StandbyState, StandbyState, StandbyEvent) {
callback_called = true;
});
// Callback should be invoked and must not interfere with state transitions
auto result = machine_->ProcessEvent(StandbyEvent::START);
EXPECT_TRUE(result.allowed);
EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState());
EXPECT_TRUE(callback_called);
}
// ========== History Tests ==========
TEST_F(StandbyStateMachineTest, TestTransitionHistory) {
// Perform several transitions
machine_->ProcessEvent(StandbyEvent::START);
machine_->ProcessEvent(StandbyEvent::CONNECTED);
machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE);
auto history = machine_->GetTransitionHistory(10);
EXPECT_EQ(3, history.size());
EXPECT_EQ(StandbyState::STOPPED, history[0].from_state);
EXPECT_EQ(StandbyState::CONNECTING, history[0].to_state);
EXPECT_EQ(StandbyEvent::START, history[0].event);
EXPECT_EQ(StandbyState::CONNECTING, history[1].from_state);
EXPECT_EQ(StandbyState::SYNCING, history[1].to_state);
EXPECT_EQ(StandbyEvent::CONNECTED, history[1].event);
EXPECT_EQ(StandbyState::SYNCING, history[2].from_state);
EXPECT_EQ(StandbyState::WATCHING, history[2].to_state);
EXPECT_EQ(StandbyEvent::SYNC_COMPLETE, history[2].event);
}
TEST_F(StandbyStateMachineTest, TestTransitionHistoryLimit) {
// Perform many transitions to test history limit
for (int i = 0; i < 20; ++i) {
machine_->ProcessEvent(StandbyEvent::START);
machine_->ProcessEvent(StandbyEvent::STOP);
}
// Request limited history
auto history = machine_->GetTransitionHistory(5);
EXPECT_LE(history.size(), 5);
}
TEST_F(StandbyStateMachineTest, TestTimeInState) {
machine_->ProcessEvent(StandbyEvent::START);
// Wait a bit
std::this_thread::sleep_for(std::chrono::milliseconds(100));
auto time_in_state = machine_->GetTimeInCurrentState();
EXPECT_GE(time_in_state.count(), 100);
EXPECT_LE(time_in_state.count(),
200); // Allow some margin for test execution
}
// ========== Concurrent Tests ==========
TEST_F(StandbyStateMachineTest, TestConcurrentStateQueries) {
ReachWatchingState();
// Multiple threads querying state concurrently
std::vector<std::thread> threads;
std::atomic<int> success_count{0};
for (int i = 0; i < 10; ++i) {
threads.emplace_back([&]() {
for (int j = 0; j < 100; ++j) {
StandbyState state = machine_->GetState();
if (state == StandbyState::WATCHING) {
success_count++;
}
}
});
}
for (auto& t : threads) {
t.join();
}
EXPECT_EQ(1000, success_count.load());
}
TEST_F(StandbyStateMachineTest, TestConcurrentEventProcessing) {
ReachWatchingState();
// Multiple threads trying to process events concurrently
// Only one should succeed (state machine should serialize)
std::vector<std::thread> threads;
std::atomic<int> success_count{0};
std::atomic<int> failure_count{0};
for (int i = 0; i < 10; ++i) {
threads.emplace_back([&]() {
auto result = machine_->ProcessEvent(StandbyEvent::STOP);
if (result.allowed) {
success_count++;
} else {
failure_count++;
}
});
}
for (auto& t : threads) {
t.join();
}
// Only one STOP should succeed (transition to STOPPED)
EXPECT_EQ(1, success_count.load());
EXPECT_EQ(9, failure_count.load());
EXPECT_EQ(StandbyState::STOPPED, machine_->GetState());
}
// ========== State Query Tests ==========
TEST_F(StandbyStateMachineTest, TestIsRunning) {
EXPECT_FALSE(machine_->IsRunning()); // STOPPED
machine_->ProcessEvent(StandbyEvent::START);
// CONNECTING only means establishing connection; sync has not started, so
// it is not considered "running"
EXPECT_FALSE(machine_->IsRunning()); // CONNECTING
machine_->ProcessEvent(StandbyEvent::CONNECTED);
EXPECT_TRUE(machine_->IsRunning()); // SYNCING
machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE);
EXPECT_TRUE(machine_->IsRunning()); // WATCHING
machine_->ProcessEvent(StandbyEvent::STOP);
EXPECT_FALSE(machine_->IsRunning()); // STOPPED
}
TEST_F(StandbyStateMachineTest, TestIsConnected) {
EXPECT_FALSE(machine_->IsConnected()); // STOPPED
machine_->ProcessEvent(StandbyEvent::START);
EXPECT_FALSE(machine_->IsConnected()); // CONNECTING
machine_->ProcessEvent(StandbyEvent::CONNECTED);
EXPECT_TRUE(machine_->IsConnected()); // SYNCING
machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE);
EXPECT_TRUE(machine_->IsConnected()); // WATCHING
machine_->ProcessEvent(StandbyEvent::STOP);
EXPECT_FALSE(machine_->IsConnected()); // STOPPED
}
TEST_F(StandbyStateMachineTest, TestIsWatchHealthy) {
EXPECT_FALSE(machine_->IsWatchHealthy()); // STOPPED
ReachWatchingState();
EXPECT_TRUE(machine_->IsWatchHealthy()); // WATCHING
machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN);
EXPECT_FALSE(machine_->IsWatchHealthy()); // RECONNECTING
}
TEST_F(StandbyStateMachineTest, TestIsReadyForPromotion) {
EXPECT_FALSE(machine_->IsReadyForPromotion()); // STOPPED
ReachWatchingState();
EXPECT_TRUE(machine_->IsReadyForPromotion()); // WATCHING
machine_->ProcessEvent(StandbyEvent::PROMOTE);
EXPECT_FALSE(machine_->IsReadyForPromotion()); // PROMOTING
}
// ========== Complete State Machine Flow Tests ==========
TEST_F(StandbyStateMachineTest, TestCompleteNormalFlow) {
// Complete flow: STOPPED -> CONNECTING -> SYNCING -> WATCHING
EXPECT_EQ(StandbyState::STOPPED, machine_->GetState());
machine_->ProcessEvent(StandbyEvent::START);
EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState());
machine_->ProcessEvent(StandbyEvent::CONNECTED);
EXPECT_EQ(StandbyState::SYNCING, machine_->GetState());
machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE);
EXPECT_EQ(StandbyState::WATCHING, machine_->GetState());
EXPECT_TRUE(machine_->IsReadyForPromotion());
}
TEST_F(StandbyStateMachineTest, TestCompletePromotionFlow) {
// Complete promotion flow
ReachWatchingState();
machine_->ProcessEvent(StandbyEvent::PROMOTE);
EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState());
machine_->ProcessEvent(StandbyEvent::PROMOTION_SUCCESS);
EXPECT_EQ(StandbyState::PROMOTED, machine_->GetState());
}
TEST_F(StandbyStateMachineTest, TestCompleteReconnectFlow) {
// Complete reconnect flow: WATCHING -> RECONNECTING -> SYNCING -> WATCHING
ReachWatchingState();
machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN);
EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState());
machine_->ProcessEvent(StandbyEvent::CONNECTED);
EXPECT_EQ(StandbyState::SYNCING, machine_->GetState());
machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE);
EXPECT_EQ(StandbyState::WATCHING, machine_->GetState());
}
TEST_F(StandbyStateMachineTest, TestCompleteRecoveryFlow) {
// Complete recovery flow: WATCHING -> RECOVERING -> WATCHING
ReachWatchingState();
machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED);
EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState());
machine_->ProcessEvent(StandbyEvent::RECOVERY_SUCCESS);
EXPECT_EQ(StandbyState::WATCHING, machine_->GetState());
}
} // namespace mooncake::test
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}