From aecbeaa898f35d61fdb0985ac2990312ecfb7528 Mon Sep 17 00:00:00 2001 From: ykwd Date: Thu, 21 Aug 2025 11:35:54 +0800 Subject: [PATCH] [Store] Serialize/Deserialize Offset Allocator (#760) Also add generic interfaces to serialize/deserialize other objects. --- .../offset_allocator/offset_allocator.hpp | 127 +++- mooncake-store/include/serializer.h | 377 +++++++++++ mooncake-store/include/types.h | 4 + mooncake-store/tests/CMakeLists.txt | 11 + .../tests/offset_allocator_test.cpp | 598 +++++++++++++++--- mooncake-store/tests/serializer_test.cpp | 173 +++++ 6 files changed, 1196 insertions(+), 94 deletions(-) create mode 100644 mooncake-store/include/serializer.h create mode 100644 mooncake-store/tests/serializer_test.cpp diff --git a/mooncake-store/include/offset_allocator/offset_allocator.hpp b/mooncake-store/include/offset_allocator/offset_allocator.hpp index 832b14ae..fde978ba 100644 --- a/mooncake-store/include/offset_allocator/offset_allocator.hpp +++ b/mooncake-store/include/offset_allocator/offset_allocator.hpp @@ -4,6 +4,7 @@ #include #include +#include #include "mutex.h" @@ -94,6 +95,8 @@ class OffsetAllocationHandle { // The real base and requested size of the allocated memory. uint64_t real_base; uint64_t requested_size; + + friend class OffsetAllocatorTest; // for unit tests }; struct OffsetAllocatorMetrics { @@ -155,6 +158,13 @@ class OffsetAllocator : public std::enable_shared_from_this { [[nodiscard]] OffsetAllocatorMetrics get_metrics() const; + // Serialize the allocator with serializer. + template + void serialize_to(T& serializer) const; + + template + static std::shared_ptr deserialize_from(T& serializer); + private: friend class OffsetAllocationHandle; @@ -166,11 +176,11 @@ class OffsetAllocator : public std::enable_shared_from_this { OffsetAllocatorMetrics get_metrics_internal() const; std::unique_ptr<__Allocator> m_allocator GUARDED_BY(m_mutex); - const uint64_t m_base; + uint64_t m_base; // The real offset and size of the allocated memory need to be multiplied by // m_multiplier - const uint64_t m_multiplier_bits; - const uint64_t m_capacity; + uint64_t m_multiplier_bits; + uint64_t m_capacity; mutable Mutex m_mutex; // Lightweight metrics maintained during allocation/deallocation @@ -179,12 +189,20 @@ class OffsetAllocator : public std::enable_shared_from_this { // Private constructor - use create() factory method instead OffsetAllocator(uint64_t base, size_t size, uint32 init_capacity, - uint32 max_capacity); + uint32 max_capacity); + + // Private constructor - initialize from serialized data + template + OffsetAllocator(T& serializer); + + friend class OffsetAllocatorTest; // for unit tests }; class __Allocator { public: __Allocator(uint32 size, uint32 init_capacity, uint32 max_capacity); + template + __Allocator(T& serializer) noexcept(false); __Allocator(__Allocator&& other); ~__Allocator(); void reset(); @@ -196,6 +214,10 @@ class __Allocator { OffsetAllocStorageReport storageReport() const; OffsetAllocStorageReportFull storageReportFull() const; + // Serialize the allocator with serializer. + template + void serialize_to(T& serializer) const; + private: uint32 insertNodeIntoBin(uint32 size, uint32 dataOffset); void removeNodeFromBin(uint32 nodeIndex); @@ -224,6 +246,103 @@ class __Allocator { Node* m_nodes; NodeIndex* m_freeNodes; uint32 m_freeOffset; + + friend class OffsetAllocatorTest; // for unit tests }; +// Template method implementations +template +void OffsetAllocator::serialize_to(T& serializer) const { + MutexLocker guard(&m_mutex); + + if (!m_allocator) { + serializer.set_error("Allocator is not initialized"); + return; + } + + // Basic member variables + serializer.write(&m_base, sizeof(m_base)); + serializer.write(&m_multiplier_bits, sizeof(m_multiplier_bits)); + serializer.write(&m_capacity, sizeof(m_capacity)); + serializer.write(&m_allocated_size, sizeof(m_allocated_size)); + serializer.write(&m_allocated_num, sizeof(m_allocated_num)); + // Serialize the allocator + m_allocator->serialize_to(serializer); +} + +template +std::shared_ptr OffsetAllocator::deserialize_from( + T& serializer) { + return std::shared_ptr(new OffsetAllocator(serializer)); +} + +template +OffsetAllocator::OffsetAllocator(T& serializer) { + // serializer.read() will throw an exception if the buffer is corrupted. + try { + serializer.read(&m_base, sizeof(m_base)); + serializer.read(&m_multiplier_bits, sizeof(m_multiplier_bits)); + serializer.read(&m_capacity, sizeof(m_capacity)); + serializer.read(&m_allocated_size, sizeof(m_allocated_size)); + serializer.read(&m_allocated_num, sizeof(m_allocated_num)); + m_allocator = std::make_unique<__Allocator>(serializer); + } catch (const std::exception& e) { + LOG(ERROR) << "Deserializing OffsetAllocator failed, error=" + << e.what(); + throw std::runtime_error("Deserializing OffsetAllocator failed"); + } +} + +template +void __Allocator::serialize_to(T& serializer) const { + if (!m_nodes || !m_freeNodes) { + serializer.set_error("Allocator is not initialized"); + return; + } + + serializer.write(&m_size, sizeof(m_size)); + serializer.write(&m_current_capacity, sizeof(m_current_capacity)); + serializer.write(&m_max_capacity, sizeof(m_max_capacity)); + serializer.write(&m_freeStorage, sizeof(m_freeStorage)); + serializer.write(&m_usedBinsTop, sizeof(m_usedBinsTop)); + serializer.write(&m_usedBins, sizeof(m_usedBins)); + serializer.write(&m_binIndices, sizeof(m_binIndices)); + serializer.write(&m_freeOffset, sizeof(m_freeOffset)); + serializer.write(m_nodes, m_current_capacity * sizeof(Node)); + serializer.write(m_freeNodes, m_current_capacity * sizeof(NodeIndex)); +} + +template +__Allocator::__Allocator(T& serializer) { + m_nodes = nullptr; + m_freeNodes = nullptr; + + // serializer.read() will throw an exception if the buffer is corrupted. + try { + // Deserialize basic member variables + serializer.read(&m_size, sizeof(m_size)); + serializer.read(&m_current_capacity, sizeof(m_current_capacity)); + serializer.read(&m_max_capacity, sizeof(m_max_capacity)); + serializer.read(&m_freeStorage, sizeof(m_freeStorage)); + serializer.read(&m_usedBinsTop, sizeof(m_usedBinsTop)); + serializer.read(&m_usedBins, sizeof(m_usedBins)); + serializer.read(&m_binIndices, sizeof(m_binIndices)); + serializer.read(&m_freeOffset, sizeof(m_freeOffset)); + + // Allocate memory for nodes and freeNodes + m_nodes = new Node[m_max_capacity]; + m_freeNodes = new NodeIndex[m_max_capacity]; + + // Deserialize the arrays + serializer.read(m_nodes, m_current_capacity * sizeof(Node)); + serializer.read(m_freeNodes, m_current_capacity * sizeof(NodeIndex)); + } catch (const std::exception& e) { + // Free memory if deserialization fails + LOG(ERROR) << "Deserializing __Allocator failed, error=" << e.what(); + if (m_nodes) delete[] m_nodes; + if (m_freeNodes) delete[] m_freeNodes; + throw std::runtime_error("Deserializing __Allocator failed"); + } +} + } // namespace mooncake::offset_allocator \ No newline at end of file diff --git a/mooncake-store/include/serializer.h b/mooncake-store/include/serializer.h new file mode 100644 index 00000000..a8a5fa08 --- /dev/null +++ b/mooncake-store/include/serializer.h @@ -0,0 +1,377 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "types.h" + +namespace mooncake { + +/** + * @brief Serialization Framework Usage Guide + * + * To implement serialization for your class, you need to provide two methods: + * + * 1. **serialize_to()** - Template method that works with both + * SerializeSizeCounter and SerializeWriter + * 2. **deserialize_from()** - Static template method that reconstructs the + * object + * + * Example implementation: + * @code + * class MyClass { + * public: + * // Serialization method (works with both counter and writer) + * template + * void serialize_to(T& serializer) const { + * serializer.write(&member1, sizeof(member1)); + * serializer.write(&member2, sizeof(member2)); + * // ... serialize other members + * } + * + * // Deserialization method + * template + * static std::shared_ptr deserialize_from(T& serializer) { + * try { + * auto obj = std::make_shared(); + * serializer.read(&obj->member1, sizeof(obj->member1)); + * serializer.read(&obj->member2, sizeof(obj->member2)); + * // ... deserialize other members + * return obj; + * } catch (const std::exception& e) { + * return nullptr; + * } + * } + * + * private: + * int member1; + * double member2; + * }; + * @endcode + * + * Usage: + * @code + * MyClass obj; + * std::vector buffer; + * serialize_to(obj, buffer); // Serialize + * auto restored = deserialize_from(buffer); // Deserialize + * @endcode + */ + +/** + * @brief A utility class for calculating the size of serialized data without + * actually writing it. + * + * This class is used in the first pass of serialization to determine the exact + * buffer size needed for storing the serialized data. It implements the same + * interface as SerializeWriter but only accumulates the size without performing + * any actual memory writes. + * + */ +class SerializeSizeCounter { + public: + SerializeSizeCounter() = default; + + ~SerializeSizeCounter() = default; + + /** + * @brief Simulates writing data by adding its size to the total count + * + * This method doesn't actually write data but accumulates the size that + * would be written. It's used to calculate the total buffer size needed for + * serialization. + * + * @param data Pointer to the data that would be written + * @param data_size Size of the data in bytes + */ + void write(const void* data, const size_t data_size) { + if (has_error_) { + return; + } + if (data == nullptr) { + set_error("data is nullptr"); + return; + } + size_ += data_size; + } + + /** + * @brief Returns the total accumulated size of all data that would be + * serialized + * + * @return Total size in bytes needed for the serialized data + */ + size_t get_size() const { return size_; } + + /** + * @brief Sets an error state with a descriptive message + * + * @param error Error message describing what went wrong + */ + void set_error(const char* error) { + has_error_ = true; + error_ = error; + } + + /** + * @brief Checks if an error occurred during size calculation + * + * @return true if an error occurred, false otherwise + */ + bool has_error() const { return has_error_; } + + /** + * @brief Returns the error message if an error occurred + * + * @return Error message string, empty if no error occurred + */ + const std::string& get_error() const { return error_; } + + private: + size_t size_{0}; ///< Accumulated size of all data that would be serialized + bool has_error_{false}; ///< Flag indicating if an error occurred during + ///< size calculation + std::string error_; ///< Error message describing what went wrong +}; + +/** + * @brief A class for writing serialized data to a pre-allocated buffer. + * + * This class handles the actual writing of serialized data to memory. It + implements the same interface as + * SerializeSizeCounter but performs actual memory writes instead of just + counting. + * + * The class is typically used in the second pass of serialization after + * SerializeSizeCounter has determined the required buffer size. + + */ +class SerializeWriter { + public: + /** + * @brief Constructs a SerializeWriter with a target buffer + * + * @param buffer Pointer to the pre-allocated buffer where data will be + * written + * @param size Total size of the buffer in bytes + */ + SerializeWriter(void* buffer, const size_t size) + : buffer_(buffer), size_(size), offset_(0), has_error_(false) {} + + ~SerializeWriter() = default; + + /** + * @brief Writes data to the buffer at the current offset position + * + * This method copies the specified data to the buffer starting at the + * current offset. + * + * @param data Pointer to the data to be written + * @param data_size Size of the data in bytes + */ + void write(const void* data, const size_t data_size) { + if (has_error_) { + return; + } + if (data == nullptr) { + set_error("null_pointer_data"); + return; + } + if (data_size + offset_ > size_) { + set_error("buffer_overflow"); + return; + } + std::memcpy(static_cast(buffer_) + offset_, data, data_size); + offset_ += data_size; + } + + /** + * @brief Sets an error state with a descriptive message + * + * Once an error is set, all subsequent write operations will be ignored + * until the error is checked and handled by the caller. + * + * @param error Error message describing what went wrong + */ + void set_error(const char* error) { + has_error_ = true; + error_ = error; + } + + /** + * @brief Checks if an error occurred during writing + * + * @return true if an error occurred, false otherwise + */ + bool has_error() const { return has_error_; } + + /** + * @brief Returns the error message if an error occurred + * + * @return Error message string, empty if no error occurred + */ + const std::string& get_error() const { return error_; } + + /** + * @brief Checks if the buffer has been completely filled + * + * This is useful for validation to ensure that the expected amount + * of data was written to the buffer. + * + * @return true if the buffer is full (offset equals buffer size), false + * otherwise + */ + bool finish_write() const { return offset_ >= size_; } + + private: + void* buffer_; ///< Pointer to the target buffer for writing data + size_t size_; ///< Total size of the buffer in bytes + size_t offset_; ///< Current write position within the buffer + bool has_error_; ///< Flag indicating if an error occurred during writing + std::string error_; ///< Error message describing what went wrong +}; + +/** + * @brief A class for reading serialized data from a buffer during + * deserialization. + * + * This class handles the reading of serialized data from a memory buffer. It + * maintains an internal offset to track the current read position and provides + * bounds checking to prevent buffer overflows. It throws an exception during + * the deserialization process if any error occurs. + */ +class SerializerReader { + public: + /** + * @brief Constructs a SerializerReader with a source buffer + * @param buffer Pointer to the buffer containing serialized data to be read + * @param size Total size of the buffer in bytes + */ + SerializerReader(const void* buffer, const size_t size) + : buffer_(buffer), size_(size), offset_(0) {} + + ~SerializerReader() = default; + + /** + * @brief Reads data from the buffer at the current offset position + * @param data Pointer to the destination where data will be copied + * @param data_size Size of the data to read in bytes + * @throws std::runtime_error if the read operation would exceed buffer + * bounds + */ + void read(void* data, const size_t data_size) { + if (offset_ + data_size > size_) { + throw std::runtime_error("buffer_overflow"); + } + std::memcpy(data, static_cast(buffer_) + offset_, + data_size); + offset_ += data_size; + } + + /** + * @brief Checks if all data in the buffer has been read + * + * This is useful for validation to ensure that the entire buffer was + * consumed during deserialization, which helps detect data corruption or + * incomplete deserialization. + * + * @return true if all data has been read (offset equals buffer size), false + * otherwise + */ + bool finish_read() const { return offset_ == size_; } + + private: + const void* + buffer_; ///< Pointer to the source buffer containing serialized data + size_t size_; ///< Total size of the buffer in bytes + size_t offset_; ///< Current read position within the buffer +}; + +/** + * @brief Serializes an object to a byte buffer using two-pass approach. + * @tparam T Type that implements serialize_to(SerializeSizeCounter&) and + * serialize_to(SerializeWriter&) + * @param target Object to serialize (must not be null) + * @param buffer Output buffer for serialized data + * @return ErrorCode indicating success or failure + */ +template +[[nodiscard]] ErrorCode serialize_to_internal( + const T* target, std::vector& buffer) { + if (target == nullptr) { + return ErrorCode::INVALID_PARAMS; + } + + // Get the size of the serialized data + SerializeSizeCounter counter; + target->serialize_to(counter); + if (counter.has_error()) { + LOG(ERROR) << "Serializing failed, error=" << counter.get_error(); + return ErrorCode::INTERNAL_ERROR; + } + + // Serialize the data to the buffer + buffer.clear(); + buffer.resize(counter.get_size()); + SerializeWriter writer(buffer.data(), buffer.size()); + target->serialize_to(writer); + + // Check if the serialization failed + if (writer.has_error()) { + LOG(ERROR) << "Serializing failed, error=" << writer.get_error(); + return ErrorCode::INTERNAL_ERROR; + } + if (!writer.finish_write()) { + LOG(ERROR) << "Serializing failed, error=wrong_data_size"; + return ErrorCode::INTERNAL_ERROR; + } + return ErrorCode::OK; +} + +template +[[nodiscard]] ErrorCode serialize_to(const T& target, + std::vector& buffer) { + return serialize_to_internal(std::addressof(target), buffer); +} + +template +[[nodiscard]] ErrorCode serialize_to(const std::shared_ptr& target, + std::vector& buffer) { + return serialize_to_internal(target.get(), buffer); +} + +/** + * @brief Deserializes an object from a byte buffer. + * @tparam T Type that implements static deserialize_from(SerializerReader&) + * @param buffer Buffer containing serialized data + * @return Shared pointer to deserialized object, or nullptr on failure + */ +template +[[nodiscard]] std::shared_ptr deserialize_from( + const std::vector& buffer) { + try { + // Deserialize the object + SerializerReader reader(buffer.data(), buffer.size()); + auto ret = T::deserialize_from(reader); + + // Check if the deserialization failed + if (ret == nullptr) { + LOG(ERROR) << "Deserializing failed, error=return_nullptr"; + return nullptr; + } + if (!reader.finish_read()) { + LOG(ERROR) << "Deserializing failed, error=wrong_data_size"; + return nullptr; + } + return ret; + } catch (const std::exception& e) { + // The deserialization method may throw an exception if it fails. + LOG(ERROR) << "Deserializing failed, error=" << e.what(); + return nullptr; + } +} + +} // namespace mooncake \ No newline at end of file diff --git a/mooncake-store/include/types.h b/mooncake-store/include/types.h index 4383ab6f..d2830a39 100644 --- a/mooncake-store/include/types.h +++ b/mooncake-store/include/types.h @@ -67,6 +67,10 @@ using EtcdLeaseId = int64_t; using UUID = std::pair; +using SerializedByte = uint8_t; // Used as basic unit of serialized data +static_assert(sizeof(SerializedByte) == 1, + "SerializedByte must be exactly 1 byte in size"); + inline std::ostream& operator<<(std::ostream& os, const UUID& uuid) noexcept { os << uuid.first << "-" << uuid.second; return os; diff --git a/mooncake-store/tests/CMakeLists.txt b/mooncake-store/tests/CMakeLists.txt index ea2e862d..421ed471 100644 --- a/mooncake-store/tests/CMakeLists.txt +++ b/mooncake-store/tests/CMakeLists.txt @@ -151,4 +151,15 @@ target_link_libraries(client_metrics_test PUBLIC ) add_test(NAME client_metrics_test COMMAND client_metrics_test) +add_executable(serializer_test serializer_test.cpp) +target_link_libraries(serializer_test PUBLIC + mooncake_store + cachelib_memory_allocator + glog + gtest + gtest_main + pthread +) +add_test(NAME serializer_test COMMAND serializer_test) + add_subdirectory(e2e) diff --git a/mooncake-store/tests/offset_allocator_test.cpp b/mooncake-store/tests/offset_allocator_test.cpp index f265c332..d255b538 100644 --- a/mooncake-store/tests/offset_allocator_test.cpp +++ b/mooncake-store/tests/offset_allocator_test.cpp @@ -1,12 +1,14 @@ #include "offset_allocator/offset_allocator.hpp" +#include "serializer.h" +#include "types.h" #include #include +#include #include -#include -using namespace mooncake::offset_allocator; +namespace mooncake::offset_allocator { // 240 bins, according to https://github.com/sebbbi/OffsetAllocator constexpr uint32 NUM_BINS = 240; @@ -91,7 +93,7 @@ class AllocationHandleWrapper { uint64_t size() const { return m_handle.size(); } // Get the underlying handle - const OffsetAllocationHandle& getHandle() const { return m_handle; } + OffsetAllocationHandle& getHandle() { return m_handle; } private: std::shared_ptr m_allocator_wrapper; @@ -102,16 +104,23 @@ class AllocationHandleWrapper { // allocation is legal. class AllocatorWrapper : public std::enable_shared_from_this { public: - // Constructor - AllocatorWrapper(uint64_t base, size_t size, uint32 maxAllocs = 128 * 1024) - : m_allocator( - OffsetAllocator::create(base, size, maxAllocs / 2, maxAllocs)), - m_base(base), - m_buffer_size(size) { - // The allocator is created with the specified base and size - // We can now properly track the allocation bounds + static std::shared_ptr create(uint64_t base, size_t size, + uint32 max_capacity) { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution init_capacity_dist(1, + max_capacity); + uint32 init_capacity = init_capacity_dist(gen); + return std::shared_ptr( + new AllocatorWrapper(base, size, init_capacity, max_capacity)); } + static std::shared_ptr create(uint64_t base, size_t size, + uint32 init_capacity, + uint32 max_capacity) { + return std::shared_ptr( + new AllocatorWrapper(base, size, init_capacity, max_capacity)); + } AllocatorWrapper(const AllocatorWrapper&) = delete; AllocatorWrapper& operator=(const AllocatorWrapper&) = delete; AllocatorWrapper(AllocatorWrapper&& other) = default; @@ -142,6 +151,15 @@ class AllocatorWrapper : public std::enable_shared_from_this { return AllocationHandleWrapper(shared_from_this(), std::move(*handle)); } + // Substitute the allocator with a new one. + void substituteAllocator(std::shared_ptr allocator) { + m_allocator = std::move(allocator); + } + + std::shared_ptr getAllocator() const { + return m_allocator; + } + // Get storage report OffsetAllocStorageReport storageReport() const { return m_allocator->storageReport(); @@ -153,6 +171,27 @@ class AllocatorWrapper : public std::enable_shared_from_this { } private: + // Constructor + AllocatorWrapper(uint64_t base, size_t size, uint32 maxAllocs = 128 * 1024) + : m_allocator( + OffsetAllocator::create(base, size, maxAllocs / 2, maxAllocs)), + m_base(base), + m_buffer_size(size) { + // The allocator is created with the specified base and size + // We can now properly track the allocation bounds + } + + // Constructor with specified init_capacity and max_capacity. + AllocatorWrapper(uint64_t base, size_t size, uint32 init_capacity, + uint32 max_capacity) + : m_allocator( + OffsetAllocator::create(base, size, init_capacity, max_capacity)), + m_base(base), + m_buffer_size(size) { + // The allocator is created with the specified base and size + // We can now properly track the allocation bounds + } + // Called by AllocationHandleWrapper when it's destroyed void onHandleDeallocated(uint64_t address, uint64_t size) { ASSERT_TRUE(m_allocated_regions.find(address) != @@ -242,14 +281,215 @@ class OffsetAllocatorTest : public ::testing::Test { void SetUp() override {} void TearDown() override {} + + OffsetAllocationHandle copyHandleWithNewAllocator( + const OffsetAllocationHandle& handle, + const std::shared_ptr& new_allocator) { + return OffsetAllocationHandle(new_allocator, handle.m_allocation, + handle.real_base, handle.requested_size); + } + + void substituteWithNewAllocator( + AllocationHandleWrapper& handle, + std::shared_ptr new_allocator) { + handle.getHandle().m_allocator = new_allocator; + } + + void assertAllocatorEQ(const std::shared_ptr& a, + const std::shared_ptr& b) { + // Compare basic member variables + ASSERT_EQ(a->m_base, b->m_base); + ASSERT_EQ(a->m_multiplier_bits, b->m_multiplier_bits); + ASSERT_EQ(a->m_capacity, b->m_capacity); + ASSERT_EQ(a->m_allocated_size, b->m_allocated_size); + ASSERT_EQ(a->m_allocated_num, b->m_allocated_num); + + // Compare __Allocator member variables + ASSERT_EQ(a->m_allocator->m_size, b->m_allocator->m_size); + ASSERT_EQ(a->m_allocator->m_current_capacity, + b->m_allocator->m_current_capacity); + ASSERT_EQ(a->m_allocator->m_max_capacity, + b->m_allocator->m_max_capacity); + ASSERT_EQ(a->m_allocator->m_freeStorage, b->m_allocator->m_freeStorage); + ASSERT_EQ(a->m_allocator->m_usedBinsTop, b->m_allocator->m_usedBinsTop); + ASSERT_EQ(a->m_allocator->m_freeOffset, b->m_allocator->m_freeOffset); + + // Compare arrays + for (uint32 i = 0; i < NUM_TOP_BINS; ++i) { + ASSERT_EQ(a->m_allocator->m_usedBins[i], + b->m_allocator->m_usedBins[i]); + } + + for (uint32 i = 0; i < NUM_LEAF_BINS; ++i) { + ASSERT_EQ(a->m_allocator->m_binIndices[i], + b->m_allocator->m_binIndices[i]); + } + + // Compare Node arrays + for (uint32 i = 0; i < a->m_allocator->m_current_capacity; ++i) { + ASSERT_EQ(a->m_allocator->m_nodes[i].dataOffset, + b->m_allocator->m_nodes[i].dataOffset); + ASSERT_EQ(a->m_allocator->m_nodes[i].dataSize, + b->m_allocator->m_nodes[i].dataSize); + ASSERT_EQ(a->m_allocator->m_nodes[i].binListPrev, + b->m_allocator->m_nodes[i].binListPrev); + ASSERT_EQ(a->m_allocator->m_nodes[i].binListNext, + b->m_allocator->m_nodes[i].binListNext); + ASSERT_EQ(a->m_allocator->m_nodes[i].neighborPrev, + b->m_allocator->m_nodes[i].neighborPrev); + ASSERT_EQ(a->m_allocator->m_nodes[i].neighborNext, + b->m_allocator->m_nodes[i].neighborNext); + ASSERT_EQ(a->m_allocator->m_nodes[i].used, + b->m_allocator->m_nodes[i].used); + } + + // Compare freeNodes array + for (uint32 i = 0; i < a->m_allocator->m_current_capacity; ++i) { + ASSERT_EQ(a->m_allocator->m_freeNodes[i], + b->m_allocator->m_freeNodes[i]); + } + } + + // Compare two allocators bytes by bytes to detect one bit difference. + bool isAllocatorEqual(const std::shared_ptr& a, + const std::shared_ptr& b) { + // Compare basic member variables + if (memcmp(&a->m_base, &b->m_base, sizeof(a->m_base)) != 0) + return false; + if (memcmp(&a->m_multiplier_bits, &b->m_multiplier_bits, + sizeof(a->m_multiplier_bits)) != 0) + return false; + if (memcmp(&a->m_capacity, &b->m_capacity, sizeof(a->m_capacity)) != 0) + return false; + if (memcmp(&a->m_allocated_size, &b->m_allocated_size, + sizeof(a->m_allocated_size)) != 0) + return false; + if (memcmp(&a->m_allocated_num, &b->m_allocated_num, + sizeof(a->m_allocated_num)) != 0) + return false; + + // Compare __Allocator member variables + if (memcmp(&a->m_allocator->m_size, &b->m_allocator->m_size, + sizeof(a->m_allocator->m_size)) != 0) + return false; + if (memcmp(&a->m_allocator->m_current_capacity, + &b->m_allocator->m_current_capacity, + sizeof(a->m_allocator->m_current_capacity)) != 0) + return false; + if (memcmp(&a->m_allocator->m_max_capacity, + &b->m_allocator->m_max_capacity, + sizeof(a->m_allocator->m_max_capacity)) != 0) + return false; + if (memcmp(&a->m_allocator->m_freeStorage, + &b->m_allocator->m_freeStorage, + sizeof(a->m_allocator->m_freeStorage)) != 0) + return false; + if (memcmp(&a->m_allocator->m_usedBinsTop, + &b->m_allocator->m_usedBinsTop, + sizeof(a->m_allocator->m_usedBinsTop)) != 0) + return false; + if (memcmp(&a->m_allocator->m_freeOffset, &b->m_allocator->m_freeOffset, + sizeof(a->m_allocator->m_freeOffset)) != 0) + return false; + + // Compare arrays + if (memcmp(a->m_allocator->m_usedBins, b->m_allocator->m_usedBins, + NUM_TOP_BINS * sizeof(a->m_allocator->m_usedBins[0])) != 0) + return false; + + if (memcmp(a->m_allocator->m_binIndices, b->m_allocator->m_binIndices, + NUM_LEAF_BINS * sizeof(a->m_allocator->m_binIndices[0])) != + 0) + return false; + + // Compare Node arrays + if (memcmp(a->m_allocator->m_nodes, b->m_allocator->m_nodes, + a->m_allocator->m_current_capacity * + sizeof(a->m_allocator->m_nodes[0])) != 0) + return false; + + // Compare freeNodes array + if (memcmp(a->m_allocator->m_freeNodes, b->m_allocator->m_freeNodes, + a->m_allocator->m_current_capacity * + sizeof(a->m_allocator->m_freeNodes[0])) != 0) + return false; + + // All comparisons passed, allocators are equal + return true; + } + + void testSerializeAllocator( + const std::shared_ptr& alloc_a, + const std::vector& handles) { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution size_dist( + 1, + 1024 * 64); // 1B to 64KB + // Serialize the allocator + std::vector buffer; + ASSERT_EQ(serialize_to(alloc_a, buffer), ErrorCode::OK); + + // Deserialize the allocator and compare + std::shared_ptr alloc_b = + deserialize_from(buffer); + ASSERT_NE(alloc_b, nullptr); + assertAllocatorEQ(alloc_a, alloc_b); + + //============== Begin test deserialization with corrupted buffer + //============== + // Set the log level to fatal to avoid the log output. + auto log_level = FLAGS_minloglevel; + FLAGS_minloglevel = google::GLOG_FATAL; + + // Remove the last byte from the buffer and try to deserialize + auto corrupted_buffer = buffer; + corrupted_buffer.pop_back(); + alloc_b = deserialize_from(corrupted_buffer); + ASSERT_TRUE(alloc_b == nullptr || !isAllocatorEqual(alloc_a, alloc_b)); + + // change a random bit from the buffer and try to deserialize + corrupted_buffer = buffer; + std::uniform_int_distribution byte_index_dist( + 0, buffer.size() - 1); + std::uniform_int_distribution bit_index_dist( + 0, 7); // 8 bits per byte + size_t byte_index = byte_index_dist(gen); + int bit_index = bit_index_dist(gen); + // Flip the random bit + corrupted_buffer[byte_index] ^= (1 << bit_index); + alloc_b = deserialize_from(corrupted_buffer); + // There are bool values whose value may not change if one bit is + // flipped, so the isAllocatorEqual compares variables using memcmp. + ASSERT_TRUE(alloc_b == nullptr || !isAllocatorEqual(alloc_a, alloc_b)); + + // Add a byte to the buffer and try to deserialize + corrupted_buffer = buffer; + corrupted_buffer.push_back(0); + alloc_b = deserialize_from(corrupted_buffer); + ASSERT_TRUE(alloc_b == nullptr || !isAllocatorEqual(alloc_a, alloc_b)); + + //============== End test deserialization with corrupted buffer + //============== + // Restore the log level. + FLAGS_minloglevel = log_level; + + // Test if the deserialized allocator can properly free allocated + // objects. + alloc_b = deserialize_from(buffer); + ASSERT_TRUE(alloc_b != nullptr); + for (const auto& handle : handles) { + OffsetAllocationHandle handle_copy = + copyHandleWithNewAllocator(handle, alloc_b); + } + } }; // Test basic allocation and deallocation TEST_F(OffsetAllocatorTest, BasicAllocation) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; // 1GB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); // Allocate handle auto handle = allocator->allocate(ALLOCATOR_SIZE); @@ -276,8 +516,7 @@ TEST_F(OffsetAllocatorTest, BasicAllocation) { TEST_F(OffsetAllocatorTest, AllocationFailure) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; // 1GB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); // Try to allocate more than available space auto handle = @@ -289,8 +528,7 @@ TEST_F(OffsetAllocatorTest, AllocationFailure) { TEST_F(OffsetAllocatorTest, MultipleAllocations) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; // 1GB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); std::vector handles; @@ -314,8 +552,7 @@ TEST_F(OffsetAllocatorTest, MultipleAllocations) { TEST_F(OffsetAllocatorTest, DifferentSizesNoOverlap) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; // 1GB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); std::vector handles; std::vector sizes = {100, 500, 1000, 2000, 50, 1500, 800, 300}; @@ -337,8 +574,7 @@ TEST_F(OffsetAllocatorTest, DifferentSizesNoOverlap) { TEST_F(OffsetAllocatorTest, StorageReports) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; // 1GB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); OffsetAllocStorageReport report = allocator->storageReport(); EXPECT_GT(report.totalFreeSpace, 0); @@ -356,8 +592,7 @@ TEST_F(OffsetAllocatorTest, StorageReports) { TEST_F(OffsetAllocatorTest, ContinuousRandomAllocationDeallocation) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; // 1GB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); std::random_device rd; std::mt19937 gen(rd()); @@ -385,8 +620,7 @@ TEST_F(OffsetAllocatorTest, FullSizeAllocation) { for (uint32 size : bin_sizes) { if (size == 0) continue; // Skip 0 size constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, size, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, size, MAX_ALLOCS); auto handle = allocator->allocate(size); ASSERT_TRUE(handle.has_value()); @@ -398,12 +632,11 @@ TEST_F(OffsetAllocatorTest, RepeatedLargeSizeAllocation) { uint32_t bin_size = bin_sizes[i]; if (bin_size < 1024) continue; // Skip small sizes constexpr uint32_t MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, bin_size + 10, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, bin_size + 10, MAX_ALLOCS); EXPECT_EQ(allocator->storageReport().totalFreeSpace, bin_size + 10); - for (uint32_t i = 0; i < 10; i++) { - auto handle = allocator->allocate(bin_size - (10 - i)); + for (uint32_t j = 0; j < 10; j++) { + auto handle = allocator->allocate(bin_size - (10 - j)); ASSERT_TRUE(handle.has_value()); } } @@ -413,8 +646,7 @@ TEST_F(OffsetAllocatorTest, RepeatedLargeSizeAllocation) { TEST_F(OffsetAllocatorTest, MaxNumAllocations) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); std::vector handles; for (uint32 i = 0; i < MAX_ALLOCS - 1; ++i) { @@ -431,8 +663,7 @@ TEST_F(OffsetAllocatorTest, MaxNumAllocations) { TEST_F(OffsetAllocatorTest, FullAllocationAfterRandomAllocationAndFree) { const uint32 ALLOCATOR_SIZE = bin_sizes[NUM_BINS - 1]; constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); std::random_device rd; std::mt19937 gen(rd()); @@ -456,8 +687,7 @@ TEST_F(OffsetAllocatorTest, FullAllocationAfterRandomAllocationAndFree) { TEST_F(OffsetAllocatorTest, AllocationSameSizeAfterFree) { constexpr uint32 ALLOCATOR_SIZE = 2048; constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); auto handle = allocator->allocate(1023); ASSERT_TRUE(handle.has_value()); @@ -474,8 +704,7 @@ TEST_F(OffsetAllocatorTest, AllocationSameSizeAfterFree) { TEST_F(OffsetAllocatorTest, RandomRepeatAllocationSameSizeAfterFree) { const uint32 ALLOCATOR_SIZE = bin_sizes[NUM_BINS - 1]; constexpr uint32 MAX_ALLOCS = 10000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); std::random_device rd; std::mt19937 gen(rd()); @@ -483,7 +712,7 @@ TEST_F(OffsetAllocatorTest, RandomRepeatAllocationSameSizeAfterFree) { std::vector handles; std::vector alloc_sizes; - for (uint32 i = 0; i < 2000; ++i) { + for (int i = 0; i < 2000; ++i) { uint32 size = size_dist(gen); auto handle = allocator->allocate(size); if (handle.has_value()) { @@ -516,8 +745,7 @@ TEST_F(OffsetAllocatorTest, BasicLargeAllocatorSize) { for (size_t buffer_size = MIN_BUFFER_SIZE; buffer_size <= MAX_BUFFER_SIZE; buffer_size *= 2) { - auto allocator = - std::make_shared(0, buffer_size, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, buffer_size, MAX_ALLOCS); size_t max_alloc_size = allocator->storageReport().largestFreeRegion; // The largest free region equals buffer size only in this specific // buffer size. @@ -548,14 +776,13 @@ TEST_F(OffsetAllocatorTest, PowerOfTwoLargeAllocatorSize) { std::mt19937 gen(rd()); for (size_t buffer_size = MIN_BUFFER_SIZE; buffer_size <= MAX_BUFFER_SIZE; buffer_size *= 2) { - auto allocator = - std::make_shared(0, buffer_size, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, buffer_size, MAX_ALLOCS); size_t max_alloc_size = buffer_size / 100; std::vector handles; std::vector alloc_sizes; std::uniform_int_distribution size_dist(1, max_alloc_size); - for (uint32 i = 0; i < 200; ++i) { + for (int i = 0; i < 200; ++i) { size_t size = size_dist(gen); auto handle = allocator->allocate(size); if (handle.has_value()) { @@ -593,8 +820,7 @@ TEST_F(OffsetAllocatorTest, MaxAllocSizeWithLargeAllocatorSize) { MAX_BUFFER_SIZE); for (int i = 0; i < 100; i++) { size_t buffer_size = buffer_size_dist(gen); - auto allocator = - std::make_shared(0, buffer_size, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, buffer_size, MAX_ALLOCS); size_t max_alloc_size = allocator->storageReport().largestFreeRegion; ASSERT_GT(max_alloc_size, buffer_size / 2); @@ -616,14 +842,13 @@ TEST_F(OffsetAllocatorTest, RandomSmallAllocWithLargeAllocatorSize) { MAX_BUFFER_SIZE); for (int i = 0; i < 100; i++) { size_t buffer_size = buffer_size_dist(gen); - auto allocator = - std::make_shared(0, buffer_size, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, buffer_size, MAX_ALLOCS); size_t max_alloc_size = buffer_size / 100; std::vector handles; std::vector alloc_sizes; std::uniform_int_distribution size_dist(1, max_alloc_size); - for (uint32 i = 0; i < 200; ++i) { + for (int j = 0; j < 200; ++j) { size_t size = size_dist(gen); auto handle = allocator->allocate(size); if (handle.has_value()) { @@ -654,8 +879,7 @@ TEST_F(OffsetAllocatorTest, RandomSmallAllocWithLargeAllocatorSize) { TEST_F(OffsetAllocatorTest, ZeroSizeAllocation) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); auto handle = allocator->allocate(0); EXPECT_FALSE(handle.has_value()) << "Zero size allocation should fail"; @@ -665,8 +889,7 @@ TEST_F(OffsetAllocatorTest, ZeroSizeAllocation) { TEST_F(OffsetAllocatorTest, OneByteAllocation) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); auto handle = allocator->allocate(1); ASSERT_TRUE(handle.has_value()); @@ -679,8 +902,7 @@ TEST_F(OffsetAllocatorTest, OneByteAllocation) { TEST_F(OffsetAllocatorTest, ExactCapacityAllocation) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); auto handle = allocator->allocate(ALLOCATOR_SIZE); ASSERT_TRUE(handle.has_value()); @@ -696,8 +918,7 @@ TEST_F(OffsetAllocatorTest, ExactCapacityAllocation) { TEST_F(OffsetAllocatorTest, OversizeAllocation) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); auto handle = allocator->allocate(ALLOCATOR_SIZE + 1); EXPECT_FALSE(handle.has_value()) @@ -708,8 +929,7 @@ TEST_F(OffsetAllocatorTest, OversizeAllocation) { TEST_F(OffsetAllocatorTest, JustBelowBinSizeAllocation) { constexpr uint32 ALLOCATOR_SIZE = 2048; constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); // Allocate size just below a bin size auto handle = allocator->allocate(1023); @@ -726,8 +946,7 @@ TEST_F(OffsetAllocatorTest, JustBelowBinSizeAllocation) { TEST_F(OffsetAllocatorTest, MaxAllocationCountEdgeCase) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 10; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); std::vector handles; @@ -754,8 +973,7 @@ TEST_F(OffsetAllocatorTest, MaxAllocationCountEdgeCase) { TEST_F(OffsetAllocatorTest, VerySmallAllocatorSize) { constexpr uint32 ALLOCATOR_SIZE = 16; // Very small constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); auto handle = allocator->allocate(16); ASSERT_TRUE(handle.has_value()); @@ -770,8 +988,7 @@ TEST_F(OffsetAllocatorTest, VerySmallAllocatorSize) { TEST_F(OffsetAllocatorTest, AllocatorSizeMinusOne) { constexpr uint32 ALLOCATOR_SIZE = 1024; constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); auto handle = allocator->allocate(ALLOCATOR_SIZE - 1); ASSERT_TRUE(handle.has_value()); @@ -782,8 +999,7 @@ TEST_F(OffsetAllocatorTest, AllocatorSizeMinusOne) { TEST_F(OffsetAllocatorTest, PowerOfTwoAllocation) { constexpr uint32 ALLOCATOR_SIZE = 2048; constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); // Test various power of 2 sizes std::vector power_of_two_sizes = {1, 2, 4, 8, 16, 32, @@ -806,8 +1022,7 @@ TEST_F(OffsetAllocatorTest, PowerOfTwoAllocation) { TEST_F(OffsetAllocatorTest, BinSizeCalculation) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); // Test that allocations are placed in appropriate bins // SmallFloat::uintToFloatRoundUp should determine the bin @@ -830,8 +1045,7 @@ TEST_F(OffsetAllocatorTest, BinSizeCalculation) { TEST_F(OffsetAllocatorTest, BinOverflowScenarios) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); // Fill up a specific bin size with many small allocations std::vector handles; @@ -858,8 +1072,7 @@ TEST_F(OffsetAllocatorTest, BinOverflowScenarios) { TEST_F(OffsetAllocatorTest, BinMergingBehavior) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); // Allocate three adjacent blocks auto handle1 = allocator->allocate(1024); @@ -889,8 +1102,7 @@ TEST_F(OffsetAllocatorTest, BinMergingBehavior) { TEST_F(OffsetAllocatorTest, BinSelectionEdgeCases) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); // Test sizes that are just below and above bin boundaries std::vector edge_sizes = { @@ -927,8 +1139,7 @@ TEST_F(OffsetAllocatorTest, BinSelectionEdgeCases) { TEST_F(OffsetAllocatorTest, BinSystemLargeAllocations) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; // 1GB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); // Test large allocations that should go into high-numbered bins std::vector large_sizes = { @@ -959,8 +1170,7 @@ TEST_F(OffsetAllocatorTest, BinSystemLargeAllocations) { TEST_F(OffsetAllocatorTest, BinSystemMixedPatterns) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); std::vector handles; @@ -984,8 +1194,7 @@ TEST_F(OffsetAllocatorTest, BinSystemMixedPatterns) { TEST_F(OffsetAllocatorTest, BinSystemRepeatedCycles) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); std::vector test_sizes = {64, 128, 256, 512, 1024, 2048, 4096}; @@ -1021,8 +1230,7 @@ TEST_F(OffsetAllocatorTest, BinSystemRepeatedCycles) { TEST_F(OffsetAllocatorTest, BinSystemNonAlignedSizes) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); // Test sizes that don't align with typical bin boundaries std::vector non_aligned_sizes = { @@ -1049,8 +1257,7 @@ TEST_F(OffsetAllocatorTest, BinSystemNonAlignedSizes) { TEST_F(OffsetAllocatorTest, BinSystemPrimeSizes) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); // Test sizes that are prime numbers (should be challenging for bin system) std::vector prime_sizes = { @@ -1081,8 +1288,7 @@ TEST_F(OffsetAllocatorTest, BinSystemPrimeSizes) { TEST_F(OffsetAllocatorTest, BinSystemFibonacciSizes) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); // Test sizes that follow the Fibonacci sequence std::vector fibonacci_sizes = { @@ -1103,8 +1309,7 @@ TEST_F(OffsetAllocatorTest, BinSystemFibonacciSizes) { TEST_F(OffsetAllocatorTest, BinSystemPageSizeMultiples) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); // Test sizes that are multiples of common page sizes (4KB, 8KB, 16KB, 64KB) std::vector page_size_multiples = { @@ -1132,8 +1337,7 @@ TEST_F(OffsetAllocatorTest, BinSystemPageSizeMultiples) { TEST_F(OffsetAllocatorTest, MetricsInterface) { constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB constexpr uint32 MAX_ALLOCS = 1000; - auto allocator = - std::make_shared(0, ALLOCATOR_SIZE, MAX_ALLOCS); + auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS); // Test initial metrics - should show empty allocator OffsetAllocatorMetrics initial_metrics = allocator->getMetrics(); @@ -1195,6 +1399,220 @@ TEST_F(OffsetAllocatorTest, MetricsInterface) { EXPECT_EQ(after_all_free.largest_free_region_, ALLOCATOR_SIZE); } +// ========== Serialization TESTS ========== + +TEST_F(OffsetAllocatorTest, SerializationEmptyAllocator) { + // Create an empty allocator + const uint64_t base = 1024 * 16; + const size_t size = 1024 * 1024; + const uint32_t init_capacity = 1000; + const uint32_t max_capacity = 10000; + std::shared_ptr alloc_a = + OffsetAllocator::create(base, size, init_capacity, max_capacity); + // test + testSerializeAllocator(alloc_a, {}); +} + +TEST_F(OffsetAllocatorTest, SerializationOneElementAllocator) { + // Create an empty allocator + const uint64_t base = 1024 * 16; + const size_t size = 1024 * 1024; + const uint32_t init_capacity = 1; + const uint32_t max_capacity = 10000; + std::shared_ptr alloc_a = + OffsetAllocator::create(base, size, init_capacity, max_capacity); + // Allocate one element + auto handle = alloc_a->allocate(1024); + ASSERT_TRUE(handle.has_value()); + // test + std::vector handles; + handles.push_back(std::move(*handle)); + testSerializeAllocator(alloc_a, handles); +} + +TEST_F(OffsetAllocatorTest, SerializationRandomAllocatedAllocator) { + // The size multiplier is larger than 1 when the allocator size is larger + // than MAX_BIN_SIZE. + constexpr size_t MIN_BUFFER_SIZE = (1ull << 31) + 1; + constexpr size_t MAX_BUFFER_SIZE = (1ull << 40); + constexpr uint32 MAX_ALLOCS = 10000; + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution buffer_size_dist(MIN_BUFFER_SIZE, + MAX_BUFFER_SIZE); + for (int i = 0; i < 100; i++) { + size_t buffer_size = buffer_size_dist(gen); + auto alloc_a = OffsetAllocator::create(0, buffer_size, 1, MAX_ALLOCS); + size_t max_alloc_size = buffer_size / 100; + + std::vector handles; + std::vector alloc_sizes; + std::uniform_int_distribution size_dist(1, max_alloc_size); + for (int j = 0; j < 200; ++j) { + size_t size = size_dist(gen); + auto handle = alloc_a->allocate(size); + if (handle.has_value()) { + handles.push_back(std::move(*handle)); + alloc_sizes.push_back(size); + } + + std::uniform_int_distribution index_dist( + 0, handles.size() - 1); + uint32 index = index_dist(gen); + std::swap(handles[index], handles.back()); + std::swap(alloc_sizes[index], alloc_sizes.back()); + uint32 test_size = alloc_sizes.back(); + handles.pop_back(); + alloc_sizes.pop_back(); + + auto handle2 = alloc_a->allocate(test_size); + ASSERT_TRUE(handle2.has_value()); + handles.push_back(std::move(*handle2)); + alloc_sizes.push_back(test_size); + } + // test + testSerializeAllocator(alloc_a, handles); + } +} + +TEST_F(OffsetAllocatorTest, AllocateAfterDeserialization) { + // Create an empty allocator + const uint64_t base = 1024 * 16; + const size_t size = 1024 * 1024; + const uint32_t init_capacity = 10; + const uint32_t max_capacity = 10000; + std::shared_ptr allocator = + AllocatorWrapper::create(base, size, init_capacity, max_capacity); + + std::random_device rd; + std::mt19937 gen(rd()); + + // Do a serias of allocations and deallocations. + std::vector handles; + for (int i = 0; i < 100; i++) { + std::uniform_int_distribution size_dist(1, 1024); + size_t size = size_dist(gen); + auto handle = allocator->allocate(size); + ASSERT_TRUE(handle.has_value()); + handles.push_back(std::move(*handle)); + + // Free a random handle for 50% probability + std::uniform_int_distribution free_dist(0, 1); + if (free_dist(gen) == 1 && !handles.empty()) { + std::uniform_int_distribution index_dist( + 0, handles.size() - 1); + size_t random_index = index_dist(gen); + std::swap(handles[random_index], handles.back()); + handles.pop_back(); + } + } + + // Serialize the allocator + std::vector buffer; + ASSERT_EQ(serialize_to(allocator->getAllocator(), buffer), ErrorCode::OK); + + // Deserialize the allocator + std::shared_ptr alloc_b = + deserialize_from(buffer); + ASSERT_NE(alloc_b, nullptr); + + // Substitute the allocator with the deserialized one. + allocator->substituteAllocator(alloc_b); + for (auto& handle : handles) { + substituteWithNewAllocator(handle, alloc_b); + } + + // Continue to do a serias of allocations and deallocations. + for (int i = 0; i < 100; i++) { + std::uniform_int_distribution size_dist(1, 1024); + size_t size = size_dist(gen); + auto handle = allocator->allocate(size); + ASSERT_TRUE(handle.has_value()); + handles.push_back(std::move(*handle)); + + // Free a random handle for 50% probability + std::uniform_int_distribution free_dist(0, 1); + if (free_dist(gen) == 1 && !handles.empty()) { + std::uniform_int_distribution index_dist( + 0, handles.size() - 1); + size_t random_index = index_dist(gen); + std::swap(handles[random_index], handles.back()); + handles.pop_back(); + } + } +} + +TEST_F(OffsetAllocatorTest, ChainedAllocationAndDeserialization) { + // The size multiplier is larger than 1 when the allocator size is larger + // than MAX_BIN_SIZE. + constexpr size_t MIN_BUFFER_SIZE = (1ull << 31) + 1; + constexpr size_t MAX_BUFFER_SIZE = (1ull << 40); + constexpr uint32 INIT_CAPACITY = 1; + constexpr uint32 MAX_ALLOCS = 10000; + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution buffer_size_dist(MIN_BUFFER_SIZE, + MAX_BUFFER_SIZE); + // Test 10 times. + for (int i = 0; i < 10; i++) { + size_t buffer_size = buffer_size_dist(gen); + auto allocator = + AllocatorWrapper::create(0, buffer_size, INIT_CAPACITY, MAX_ALLOCS); + size_t max_alloc_size = buffer_size / 100; + + std::vector handles; + std::vector alloc_sizes; + std::uniform_int_distribution size_dist(1, max_alloc_size); + // 10 times serilization and deserialization. + for (int j = 0; j < 10; j++) { + // 100 times allocation + for (int k = 0; k < 100; k++) { + size_t size = size_dist(gen); + auto handle = allocator->allocate(size); + if (handle.has_value()) { + handles.push_back(std::move(*handle)); + alloc_sizes.push_back(size); + } + + std::uniform_int_distribution index_dist( + 0, handles.size() - 1); + uint32 index = index_dist(gen); + std::swap(handles[index], handles.back()); + std::swap(alloc_sizes[index], alloc_sizes.back()); + uint32 test_size = alloc_sizes.back(); + handles.pop_back(); + alloc_sizes.pop_back(); + + auto handle2 = allocator->allocate(test_size); + ASSERT_TRUE(handle2.has_value()); + handles.push_back(std::move(*handle2)); + alloc_sizes.push_back(test_size); + } + } + // Serialize the allocator + std::vector buffer; + ASSERT_EQ(serialize_to(allocator->getAllocator(), buffer), + ErrorCode::OK); + + // Deserialize the allocator + std::shared_ptr new_alloc = + deserialize_from(buffer); + ASSERT_NE(new_alloc, nullptr); + + // Verify the allocator is equal to the original one. + assertAllocatorEQ(allocator->getAllocator(), new_alloc); + + // Substitute the allocator with the deserialized one. + allocator->substituteAllocator(new_alloc); + // Substitute the handles with the deserialized allocator. + for (auto& handle : handles) { + substituteWithNewAllocator(handle, new_alloc); + } + } +} + +} // namespace mooncake::offset_allocator + int main(int argc, char** argv) { // Initialize Google Test ::testing::InitGoogleTest(&argc, argv); diff --git a/mooncake-store/tests/serializer_test.cpp b/mooncake-store/tests/serializer_test.cpp new file mode 100644 index 00000000..34dfb2ba --- /dev/null +++ b/mooncake-store/tests/serializer_test.cpp @@ -0,0 +1,173 @@ +#include +#include + +#include "serializer.h" + +namespace mooncake::test { + +// Example class implementing serialization following the usage documentation +class ExampleClass { + public: + ExampleClass() : id_(0), value_(0.0), name_("") {} + + ExampleClass(int id, double value, const std::string& name) + : id_(id), value_(value), name_(name) {} + + // Serialization method (works with both counter and writer) + template + void serialize_to(T& serializer) const { + serializer.write(&id_, sizeof(id_)); + serializer.write(&value_, sizeof(value_)); + + // Serialize string length first, then string data + size_t name_length = name_.length(); + serializer.write(&name_length, sizeof(name_length)); + if (!name_.empty()) { + serializer.write(name_.data(), name_.length()); + } + } + + // Deserialization method + template + static std::shared_ptr deserialize_from(T& serializer) { + try { + auto obj = std::make_shared(); + + // Deserialize basic members + serializer.read(&obj->id_, sizeof(obj->id_)); + serializer.read(&obj->value_, sizeof(obj->value_)); + + // Deserialize string + size_t name_length; + serializer.read(&name_length, sizeof(name_length)); + if (name_length > 0) { + obj->name_.resize(name_length); + serializer.read(&obj->name_[0], name_length); + } + + return obj; + } catch (const std::exception& e) { + return nullptr; + } + } + + // Getters for testing + int getId() const { return id_; } + double getValue() const { return value_; } + const std::string& getName() const { return name_; } + + // Equality operator for testing + bool operator==(const ExampleClass& other) const { + return id_ == other.id_ && value_ == other.value_ && + name_ == other.name_; + } + + protected: + int id_; + double value_; + std::string name_; +}; + +class ExampleClassWithException { + public: + ExampleClassWithException() : value_(0) {} + ExampleClassWithException(int value) : value_(value) {} + + template + void serialize_to(T& serializer) const { + serializer.write(&value_, sizeof(value_)); + } + + template + static std::shared_ptr deserialize_from( + T& serializer) { + throw std::runtime_error("throw_exception"); + } + + private: + int value_; +}; + +class SerializerTest : public ::testing::Test { + protected: + void SetUp() override { + google::InitGoogleLogging("SerializerTest"); + FLAGS_logtostderr = true; + } + + void TearDown() override { google::ShutdownGoogleLogging(); } +}; + +TEST_F(SerializerTest, ExampleClassSerialization) { + // Create an example object + ExampleClass original(42, 3.14159, "Test Object"); + + // Test serialization + std::vector buffer; + ASSERT_EQ(serialize_to(original, buffer), ErrorCode::OK); + ASSERT_FALSE(buffer.empty()); + + // Test deserialization + auto restored = deserialize_from(buffer); + ASSERT_NE(restored, nullptr); + + // Verify the deserialized object matches the original + EXPECT_EQ(restored->getId(), original.getId()); + EXPECT_DOUBLE_EQ(restored->getValue(), original.getValue()); + EXPECT_EQ(restored->getName(), original.getName()); + EXPECT_TRUE(*restored == original); +} + +TEST_F(SerializerTest, ExampleClassSerializationWithSharedPtr) { + // Test with shared_ptr + auto original = + std::make_shared(777, 2.718, "Shared Pointer Test"); + + std::vector buffer; + ASSERT_EQ(serialize_to(original, buffer), ErrorCode::OK); + + auto restored = deserialize_from(buffer); + ASSERT_NE(restored, nullptr); + EXPECT_TRUE(*restored == *original); +} + +TEST_F(SerializerTest, ExampleClassSerializationNullPointer) { + // Test with null shared_ptr + std::shared_ptr null_ptr = nullptr; + + std::vector buffer; + ASSERT_EQ(serialize_to(null_ptr, buffer), ErrorCode::INVALID_PARAMS); +} + +TEST_F(SerializerTest, ExampleClassDeserializationCorruptedBuffer) { + // Create a valid object and serialize it + ExampleClass original(1, 1.0, "Test"); + std::vector buffer; + ASSERT_EQ(serialize_to(original, buffer), ErrorCode::OK); + + // Corrupt the buffer by removing the last byte + buffer.pop_back(); + + // Try to deserialize corrupted buffer + auto restored = deserialize_from(buffer); + EXPECT_EQ(restored, nullptr); +} + +TEST_F(SerializerTest, ExampleClassDeserializationWithException) { + // Create a valid object and serialize it + ExampleClassWithException original(1); + std::vector buffer; + ASSERT_EQ(serialize_to(original, buffer), ErrorCode::OK); + + // Try to deserialize the buffer, the deserialization method will throw an + // exception. + auto restored = deserialize_from(buffer); + EXPECT_EQ(restored, nullptr); +} + +} // namespace mooncake::test + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file