zbtrs2
/
Mooncake
forked from mooncake-track/Mooncake
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Store] Serialize/Deserialize Offset Allocator (#760) 

Also add generic interfaces to serialize/deserialize other objects.
This commit is contained in:
ykwd 2025-08-21 11:35:54 +08:00 committed by GitHub
parent b63322c9e8
commit aecbeaa898
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 1196 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
mooncake-store/include/offset_allocator/offset_allocator.hpp
View File

@ -4,6 +4,7 @@
#include <memory>
#include <optional>
#include <glog/logging.h>
#include "mutex.h"
@ -94,6 +95,8 @@ class OffsetAllocationHandle {
// The real base and requested size of the allocated memory.
uint64_t real_base;
uint64_t requested_size;
friend class OffsetAllocatorTest; // for unit tests
};
struct OffsetAllocatorMetrics {
@ -155,6 +158,13 @@ class OffsetAllocator : public std::enable_shared_from_this<OffsetAllocator> {
[[nodiscard]]
OffsetAllocatorMetrics get_metrics() const;
// Serialize the allocator with serializer.
template <typename T>
void serialize_to(T& serializer) const;
template <typename T>
static std::shared_ptr<OffsetAllocator> deserialize_from(T& serializer);
private:
friend class OffsetAllocationHandle;
@ -166,11 +176,11 @@ class OffsetAllocator : public std::enable_shared_from_this<OffsetAllocator> {
OffsetAllocatorMetrics get_metrics_internal() const;
std::unique_ptr<__Allocator> m_allocator GUARDED_BY(m_mutex);
const uint64_t m_base;
uint64_t m_base;
// The real offset and size of the allocated memory need to be multiplied by
// m_multiplier
const uint64_t m_multiplier_bits;
const uint64_t m_capacity;
uint64_t m_multiplier_bits;
uint64_t m_capacity;
mutable Mutex m_mutex;
// Lightweight metrics maintained during allocation/deallocation
@ -179,12 +189,20 @@ class OffsetAllocator : public std::enable_shared_from_this<OffsetAllocator> {
// Private constructor - use create() factory method instead
OffsetAllocator(uint64_t base, size_t size, uint32 init_capacity,
uint32 max_capacity);
uint32 max_capacity);
// Private constructor - initialize from serialized data
template <typename T>
OffsetAllocator(T& serializer);
friend class OffsetAllocatorTest; // for unit tests
};
class __Allocator {
public:
__Allocator(uint32 size, uint32 init_capacity, uint32 max_capacity);
template <typename T>
__Allocator(T& serializer) noexcept(false);
__Allocator(__Allocator&& other);
~__Allocator();
void reset();
@ -196,6 +214,10 @@ class __Allocator {
OffsetAllocStorageReport storageReport() const;
OffsetAllocStorageReportFull storageReportFull() const;
// Serialize the allocator with serializer.
template <typename T>
void serialize_to(T& serializer) const;
private:
uint32 insertNodeIntoBin(uint32 size, uint32 dataOffset);
void removeNodeFromBin(uint32 nodeIndex);
@ -224,6 +246,103 @@ class __Allocator {
Node* m_nodes;
NodeIndex* m_freeNodes;
uint32 m_freeOffset;
friend class OffsetAllocatorTest; // for unit tests
};
// Template method implementations
template <typename T>
void OffsetAllocator::serialize_to(T& serializer) const {
MutexLocker guard(&m_mutex);
if (!m_allocator) {
serializer.set_error("Allocator is not initialized");
return;
}
// Basic member variables
serializer.write(&m_base, sizeof(m_base));
serializer.write(&m_multiplier_bits, sizeof(m_multiplier_bits));
serializer.write(&m_capacity, sizeof(m_capacity));
serializer.write(&m_allocated_size, sizeof(m_allocated_size));
serializer.write(&m_allocated_num, sizeof(m_allocated_num));
// Serialize the allocator
m_allocator->serialize_to(serializer);
}
template <typename T>
std::shared_ptr<OffsetAllocator> OffsetAllocator::deserialize_from(
T& serializer) {
return std::shared_ptr<OffsetAllocator>(new OffsetAllocator(serializer));
}
template <typename T>
OffsetAllocator::OffsetAllocator(T& serializer) {
// serializer.read() will throw an exception if the buffer is corrupted.
try {
serializer.read(&m_base, sizeof(m_base));
serializer.read(&m_multiplier_bits, sizeof(m_multiplier_bits));
serializer.read(&m_capacity, sizeof(m_capacity));
serializer.read(&m_allocated_size, sizeof(m_allocated_size));
serializer.read(&m_allocated_num, sizeof(m_allocated_num));
m_allocator = std::make_unique<__Allocator>(serializer);
} catch (const std::exception& e) {
LOG(ERROR) << "Deserializing OffsetAllocator failed, error="
<< e.what();
throw std::runtime_error("Deserializing OffsetAllocator failed");
}
}
template <typename T>
void __Allocator::serialize_to(T& serializer) const {
if (!m_nodes || !m_freeNodes) {
serializer.set_error("Allocator is not initialized");
return;
}
serializer.write(&m_size, sizeof(m_size));
serializer.write(&m_current_capacity, sizeof(m_current_capacity));
serializer.write(&m_max_capacity, sizeof(m_max_capacity));
serializer.write(&m_freeStorage, sizeof(m_freeStorage));
serializer.write(&m_usedBinsTop, sizeof(m_usedBinsTop));
serializer.write(&m_usedBins, sizeof(m_usedBins));
serializer.write(&m_binIndices, sizeof(m_binIndices));
serializer.write(&m_freeOffset, sizeof(m_freeOffset));
serializer.write(m_nodes, m_current_capacity * sizeof(Node));
serializer.write(m_freeNodes, m_current_capacity * sizeof(NodeIndex));
}
template <typename T>
__Allocator::__Allocator(T& serializer) {
m_nodes = nullptr;
m_freeNodes = nullptr;
// serializer.read() will throw an exception if the buffer is corrupted.
try {
// Deserialize basic member variables
serializer.read(&m_size, sizeof(m_size));
serializer.read(&m_current_capacity, sizeof(m_current_capacity));
serializer.read(&m_max_capacity, sizeof(m_max_capacity));
serializer.read(&m_freeStorage, sizeof(m_freeStorage));
serializer.read(&m_usedBinsTop, sizeof(m_usedBinsTop));
serializer.read(&m_usedBins, sizeof(m_usedBins));
serializer.read(&m_binIndices, sizeof(m_binIndices));
serializer.read(&m_freeOffset, sizeof(m_freeOffset));
// Allocate memory for nodes and freeNodes
m_nodes = new Node[m_max_capacity];
m_freeNodes = new NodeIndex[m_max_capacity];
// Deserialize the arrays
serializer.read(m_nodes, m_current_capacity * sizeof(Node));
serializer.read(m_freeNodes, m_current_capacity * sizeof(NodeIndex));
} catch (const std::exception& e) {
// Free memory if deserialization fails
LOG(ERROR) << "Deserializing __Allocator failed, error=" << e.what();
if (m_nodes) delete[] m_nodes;
if (m_freeNodes) delete[] m_freeNodes;
throw std::runtime_error("Deserializing __Allocator failed");
}
}
} // namespace mooncake::offset_allocator

377
mooncake-store/include/serializer.h Normal file
View File

@ -0,0 +1,377 @@
#pragma once
#include <memory>
#include <string>
#include <cstring>
#include <cstdint>
#include <glog/logging.h>
#include "types.h"
namespace mooncake {
/**
* @brief Serialization Framework Usage Guide
*
* To implement serialization for your class, you need to provide two methods:
*
* 1. **serialize_to()** - Template method that works with both
* SerializeSizeCounter and SerializeWriter
* 2. **deserialize_from()** - Static template method that reconstructs the
* object
*
* Example implementation:
* @code
* class MyClass {
* public:
* // Serialization method (works with both counter and writer)
* template <typename T>
* void serialize_to(T& serializer) const {
* serializer.write(&member1, sizeof(member1));
* serializer.write(&member2, sizeof(member2));
* // ... serialize other members
* }
*
* // Deserialization method
* template <typename T>
* static std::shared_ptr<MyClass> deserialize_from(T& serializer) {
* try {
* auto obj = std::make_shared<MyClass>();
* serializer.read(&obj->member1, sizeof(obj->member1));
* serializer.read(&obj->member2, sizeof(obj->member2));
* // ... deserialize other members
* return obj;
* } catch (const std::exception& e) {
* return nullptr;
* }
* }
*
* private:
* int member1;
* double member2;
* };
* @endcode
*
* Usage:
* @code
* MyClass obj;
* std::vector<SerializedByte> buffer;
* serialize_to(obj, buffer); // Serialize
* auto restored = deserialize_from<MyClass>(buffer); // Deserialize
* @endcode
*/
/**
* @brief A utility class for calculating the size of serialized data without
* actually writing it.
*
* This class is used in the first pass of serialization to determine the exact
* buffer size needed for storing the serialized data. It implements the same
* interface as SerializeWriter but only accumulates the size without performing
* any actual memory writes.
*
*/
class SerializeSizeCounter {
public:
SerializeSizeCounter() = default;
~SerializeSizeCounter() = default;
/**
* @brief Simulates writing data by adding its size to the total count
*
* This method doesn't actually write data but accumulates the size that
* would be written. It's used to calculate the total buffer size needed for
* serialization.
*
* @param data Pointer to the data that would be written
* @param data_size Size of the data in bytes
*/
void write(const void* data, const size_t data_size) {
if (has_error_) {
return;
}
if (data == nullptr) {
set_error("data is nullptr");
return;
}
size_ += data_size;
}
/**
* @brief Returns the total accumulated size of all data that would be
* serialized
*
* @return Total size in bytes needed for the serialized data
*/
size_t get_size() const { return size_; }
/**
* @brief Sets an error state with a descriptive message
*
* @param error Error message describing what went wrong
*/
void set_error(const char* error) {
has_error_ = true;
error_ = error;
}
/**
* @brief Checks if an error occurred during size calculation
*
* @return true if an error occurred, false otherwise
*/
bool has_error() const { return has_error_; }
/**
* @brief Returns the error message if an error occurred
*
* @return Error message string, empty if no error occurred
*/
const std::string& get_error() const { return error_; }
private:
size_t size_{0}; ///< Accumulated size of all data that would be serialized
bool has_error_{false}; ///< Flag indicating if an error occurred during
///< size calculation
std::string error_; ///< Error message describing what went wrong
};
/**
* @brief A class for writing serialized data to a pre-allocated buffer.
*
* This class handles the actual writing of serialized data to memory. It
implements the same interface as
* SerializeSizeCounter but performs actual memory writes instead of just
counting.
*
* The class is typically used in the second pass of serialization after
* SerializeSizeCounter has determined the required buffer size.
*/
class SerializeWriter {
public:
/**
* @brief Constructs a SerializeWriter with a target buffer
*
* @param buffer Pointer to the pre-allocated buffer where data will be
* written
* @param size Total size of the buffer in bytes
*/
SerializeWriter(void* buffer, const size_t size)
: buffer_(buffer), size_(size), offset_(0), has_error_(false) {}
~SerializeWriter() = default;
/**
* @brief Writes data to the buffer at the current offset position
*
* This method copies the specified data to the buffer starting at the
* current offset.
*
* @param data Pointer to the data to be written
* @param data_size Size of the data in bytes
*/
void write(const void* data, const size_t data_size) {
if (has_error_) {
return;
}
if (data == nullptr) {
set_error("null_pointer_data");
return;
}
if (data_size + offset_ > size_) {
set_error("buffer_overflow");
return;
}
std::memcpy(static_cast<uint8_t*>(buffer_) + offset_, data, data_size);
offset_ += data_size;
}
/**
* @brief Sets an error state with a descriptive message
*
* Once an error is set, all subsequent write operations will be ignored
* until the error is checked and handled by the caller.
*
* @param error Error message describing what went wrong
*/
void set_error(const char* error) {
has_error_ = true;
error_ = error;
}
/**
* @brief Checks if an error occurred during writing
*
* @return true if an error occurred, false otherwise
*/
bool has_error() const { return has_error_; }
/**
* @brief Returns the error message if an error occurred
*
* @return Error message string, empty if no error occurred
*/
const std::string& get_error() const { return error_; }
/**
* @brief Checks if the buffer has been completely filled
*
* This is useful for validation to ensure that the expected amount
* of data was written to the buffer.
*
* @return true if the buffer is full (offset equals buffer size), false
* otherwise
*/
bool finish_write() const { return offset_ >= size_; }
private:
void* buffer_; ///< Pointer to the target buffer for writing data
size_t size_; ///< Total size of the buffer in bytes
size_t offset_; ///< Current write position within the buffer
bool has_error_; ///< Flag indicating if an error occurred during writing
std::string error_; ///< Error message describing what went wrong
};
/**
* @brief A class for reading serialized data from a buffer during
* deserialization.
*
* This class handles the reading of serialized data from a memory buffer. It
* maintains an internal offset to track the current read position and provides
* bounds checking to prevent buffer overflows. It throws an exception during
* the deserialization process if any error occurs.
*/
class SerializerReader {
public:
/**
* @brief Constructs a SerializerReader with a source buffer
* @param buffer Pointer to the buffer containing serialized data to be read
* @param size Total size of the buffer in bytes
*/
SerializerReader(const void* buffer, const size_t size)
: buffer_(buffer), size_(size), offset_(0) {}
~SerializerReader() = default;
/**
* @brief Reads data from the buffer at the current offset position
* @param data Pointer to the destination where data will be copied
* @param data_size Size of the data to read in bytes
* @throws std::runtime_error if the read operation would exceed buffer
* bounds
*/
void read(void* data, const size_t data_size) {
if (offset_ + data_size > size_) {
throw std::runtime_error("buffer_overflow");
}
std::memcpy(data, static_cast<const uint8_t*>(buffer_) + offset_,
data_size);
offset_ += data_size;
}
/**
* @brief Checks if all data in the buffer has been read
*
* This is useful for validation to ensure that the entire buffer was
* consumed during deserialization, which helps detect data corruption or
* incomplete deserialization.
*
* @return true if all data has been read (offset equals buffer size), false
* otherwise
*/
bool finish_read() const { return offset_ == size_; }
private:
const void*
buffer_; ///< Pointer to the source buffer containing serialized data
size_t size_; ///< Total size of the buffer in bytes
size_t offset_; ///< Current read position within the buffer
};
/**
* @brief Serializes an object to a byte buffer using two-pass approach.
* @tparam T Type that implements serialize_to(SerializeSizeCounter&) and
* serialize_to(SerializeWriter&)
* @param target Object to serialize (must not be null)
* @param buffer Output buffer for serialized data
* @return ErrorCode indicating success or failure
*/
template <typename T>
[[nodiscard]] ErrorCode serialize_to_internal(
const T* target, std::vector<SerializedByte>& buffer) {
if (target == nullptr) {
return ErrorCode::INVALID_PARAMS;
}
// Get the size of the serialized data
SerializeSizeCounter counter;
target->serialize_to(counter);
if (counter.has_error()) {
LOG(ERROR) << "Serializing failed, error=" << counter.get_error();
return ErrorCode::INTERNAL_ERROR;
}
// Serialize the data to the buffer
buffer.clear();
buffer.resize(counter.get_size());
SerializeWriter writer(buffer.data(), buffer.size());
target->serialize_to(writer);
// Check if the serialization failed
if (writer.has_error()) {
LOG(ERROR) << "Serializing failed, error=" << writer.get_error();
return ErrorCode::INTERNAL_ERROR;
}
if (!writer.finish_write()) {
LOG(ERROR) << "Serializing failed, error=wrong_data_size";
return ErrorCode::INTERNAL_ERROR;
}
return ErrorCode::OK;
}
template <typename T>
[[nodiscard]] ErrorCode serialize_to(const T& target,
std::vector<SerializedByte>& buffer) {
return serialize_to_internal(std::addressof(target), buffer);
}
template <typename T>
[[nodiscard]] ErrorCode serialize_to(const std::shared_ptr<T>& target,
std::vector<SerializedByte>& buffer) {
return serialize_to_internal(target.get(), buffer);
}
/**
* @brief Deserializes an object from a byte buffer.
* @tparam T Type that implements static deserialize_from(SerializerReader&)
* @param buffer Buffer containing serialized data
* @return Shared pointer to deserialized object, or nullptr on failure
*/
template <typename T>
[[nodiscard]] std::shared_ptr<T> deserialize_from(
const std::vector<SerializedByte>& buffer) {
try {
// Deserialize the object
SerializerReader reader(buffer.data(), buffer.size());
auto ret = T::deserialize_from(reader);
// Check if the deserialization failed
if (ret == nullptr) {
LOG(ERROR) << "Deserializing failed, error=return_nullptr";
return nullptr;
}
if (!reader.finish_read()) {
LOG(ERROR) << "Deserializing failed, error=wrong_data_size";
return nullptr;
}
return ret;
} catch (const std::exception& e) {
// The deserialization method may throw an exception if it fails.
LOG(ERROR) << "Deserializing failed, error=" << e.what();
return nullptr;
}
}
} // namespace mooncake

4
mooncake-store/include/types.h
View File

@ -67,6 +67,10 @@ using EtcdLeaseId = int64_t;
using UUID = std::pair<uint64_t, uint64_t>;
using SerializedByte = uint8_t; // Used as basic unit of serialized data
static_assert(sizeof(SerializedByte) == 1,
"SerializedByte must be exactly 1 byte in size");
inline std::ostream& operator<<(std::ostream& os, const UUID& uuid) noexcept {
os << uuid.first << "-" << uuid.second;
return os;

11
mooncake-store/tests/CMakeLists.txt
View File

@ -151,4 +151,15 @@ target_link_libraries(client_metrics_test PUBLIC
)
add_test(NAME client_metrics_test COMMAND client_metrics_test)
add_executable(serializer_test serializer_test.cpp)
target_link_libraries(serializer_test PUBLIC
mooncake_store
cachelib_memory_allocator
glog
gtest
gtest_main
pthread
)
add_test(NAME serializer_test COMMAND serializer_test)
add_subdirectory(e2e)

598
mooncake-store/tests/offset_allocator_test.cpp
View File

@ -1,12 +1,14 @@
#include "offset_allocator/offset_allocator.hpp"
#include "serializer.h"
#include "types.h"
#include <gtest/gtest.h>
#include <map>
#include <memory>
#include <random>
#include <vector>
using namespace mooncake::offset_allocator;
namespace mooncake::offset_allocator {
// 240 bins, according to https://github.com/sebbbi/OffsetAllocator
constexpr uint32 NUM_BINS = 240;
@ -91,7 +93,7 @@ class AllocationHandleWrapper {
uint64_t size() const { return m_handle.size(); }
// Get the underlying handle
const OffsetAllocationHandle& getHandle() const { return m_handle; }
OffsetAllocationHandle& getHandle() { return m_handle; }
private:
std::shared_ptr<AllocatorWrapper> m_allocator_wrapper;
@ -102,16 +104,23 @@ class AllocationHandleWrapper {
// allocation is legal.
class AllocatorWrapper : public std::enable_shared_from_this<AllocatorWrapper> {
public:
// Constructor
AllocatorWrapper(uint64_t base, size_t size, uint32 maxAllocs = 128 * 1024)
: m_allocator(
OffsetAllocator::create(base, size, maxAllocs / 2, maxAllocs)),
m_base(base),
m_buffer_size(size) {
// The allocator is created with the specified base and size
// We can now properly track the allocation bounds
static std::shared_ptr<AllocatorWrapper> create(uint64_t base, size_t size,
uint32 max_capacity) {
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<uint32> init_capacity_dist(1,
max_capacity);
uint32 init_capacity = init_capacity_dist(gen);
return std::shared_ptr<AllocatorWrapper>(
new AllocatorWrapper(base, size, init_capacity, max_capacity));
}
static std::shared_ptr<AllocatorWrapper> create(uint64_t base, size_t size,
uint32 init_capacity,
uint32 max_capacity) {
return std::shared_ptr<AllocatorWrapper>(
new AllocatorWrapper(base, size, init_capacity, max_capacity));
}
AllocatorWrapper(const AllocatorWrapper&) = delete;
AllocatorWrapper& operator=(const AllocatorWrapper&) = delete;
AllocatorWrapper(AllocatorWrapper&& other) = default;
@ -142,6 +151,15 @@ class AllocatorWrapper : public std::enable_shared_from_this<AllocatorWrapper> {
return AllocationHandleWrapper(shared_from_this(), std::move(*handle));
}
// Substitute the allocator with a new one.
void substituteAllocator(std::shared_ptr<OffsetAllocator> allocator) {
m_allocator = std::move(allocator);
}
std::shared_ptr<OffsetAllocator> getAllocator() const {
return m_allocator;
}
// Get storage report
OffsetAllocStorageReport storageReport() const {
return m_allocator->storageReport();
@ -153,6 +171,27 @@ class AllocatorWrapper : public std::enable_shared_from_this<AllocatorWrapper> {
}
private:
// Constructor
AllocatorWrapper(uint64_t base, size_t size, uint32 maxAllocs = 128 * 1024)
: m_allocator(
OffsetAllocator::create(base, size, maxAllocs / 2, maxAllocs)),
m_base(base),
m_buffer_size(size) {
// The allocator is created with the specified base and size
// We can now properly track the allocation bounds
}
// Constructor with specified init_capacity and max_capacity.
AllocatorWrapper(uint64_t base, size_t size, uint32 init_capacity,
uint32 max_capacity)
: m_allocator(
OffsetAllocator::create(base, size, init_capacity, max_capacity)),
m_base(base),
m_buffer_size(size) {
// The allocator is created with the specified base and size
// We can now properly track the allocation bounds
}
// Called by AllocationHandleWrapper when it's destroyed
void onHandleDeallocated(uint64_t address, uint64_t size) {
ASSERT_TRUE(m_allocated_regions.find(address) !=
@ -242,14 +281,215 @@ class OffsetAllocatorTest : public ::testing::Test {
void SetUp() override {}
void TearDown() override {}
OffsetAllocationHandle copyHandleWithNewAllocator(
const OffsetAllocationHandle& handle,
const std::shared_ptr<OffsetAllocator>& new_allocator) {
return OffsetAllocationHandle(new_allocator, handle.m_allocation,
handle.real_base, handle.requested_size);
}
void substituteWithNewAllocator(
AllocationHandleWrapper& handle,
std::shared_ptr<OffsetAllocator> new_allocator) {
handle.getHandle().m_allocator = new_allocator;
}
void assertAllocatorEQ(const std::shared_ptr<OffsetAllocator>& a,
const std::shared_ptr<OffsetAllocator>& b) {
// Compare basic member variables
ASSERT_EQ(a->m_base, b->m_base);
ASSERT_EQ(a->m_multiplier_bits, b->m_multiplier_bits);
ASSERT_EQ(a->m_capacity, b->m_capacity);
ASSERT_EQ(a->m_allocated_size, b->m_allocated_size);
ASSERT_EQ(a->m_allocated_num, b->m_allocated_num);
// Compare __Allocator member variables
ASSERT_EQ(a->m_allocator->m_size, b->m_allocator->m_size);
ASSERT_EQ(a->m_allocator->m_current_capacity,
b->m_allocator->m_current_capacity);
ASSERT_EQ(a->m_allocator->m_max_capacity,
b->m_allocator->m_max_capacity);
ASSERT_EQ(a->m_allocator->m_freeStorage, b->m_allocator->m_freeStorage);
ASSERT_EQ(a->m_allocator->m_usedBinsTop, b->m_allocator->m_usedBinsTop);
ASSERT_EQ(a->m_allocator->m_freeOffset, b->m_allocator->m_freeOffset);
// Compare arrays
for (uint32 i = 0; i < NUM_TOP_BINS; ++i) {
ASSERT_EQ(a->m_allocator->m_usedBins[i],
b->m_allocator->m_usedBins[i]);
}
for (uint32 i = 0; i < NUM_LEAF_BINS; ++i) {
ASSERT_EQ(a->m_allocator->m_binIndices[i],
b->m_allocator->m_binIndices[i]);
}
// Compare Node arrays
for (uint32 i = 0; i < a->m_allocator->m_current_capacity; ++i) {
ASSERT_EQ(a->m_allocator->m_nodes[i].dataOffset,
b->m_allocator->m_nodes[i].dataOffset);
ASSERT_EQ(a->m_allocator->m_nodes[i].dataSize,
b->m_allocator->m_nodes[i].dataSize);
ASSERT_EQ(a->m_allocator->m_nodes[i].binListPrev,
b->m_allocator->m_nodes[i].binListPrev);
ASSERT_EQ(a->m_allocator->m_nodes[i].binListNext,
b->m_allocator->m_nodes[i].binListNext);
ASSERT_EQ(a->m_allocator->m_nodes[i].neighborPrev,
b->m_allocator->m_nodes[i].neighborPrev);
ASSERT_EQ(a->m_allocator->m_nodes[i].neighborNext,
b->m_allocator->m_nodes[i].neighborNext);
ASSERT_EQ(a->m_allocator->m_nodes[i].used,
b->m_allocator->m_nodes[i].used);
}
// Compare freeNodes array
for (uint32 i = 0; i < a->m_allocator->m_current_capacity; ++i) {
ASSERT_EQ(a->m_allocator->m_freeNodes[i],
b->m_allocator->m_freeNodes[i]);
}
}
// Compare two allocators bytes by bytes to detect one bit difference.
bool isAllocatorEqual(const std::shared_ptr<OffsetAllocator>& a,
const std::shared_ptr<OffsetAllocator>& b) {
// Compare basic member variables
if (memcmp(&a->m_base, &b->m_base, sizeof(a->m_base)) != 0)
return false;
if (memcmp(&a->m_multiplier_bits, &b->m_multiplier_bits,
sizeof(a->m_multiplier_bits)) != 0)
return false;
if (memcmp(&a->m_capacity, &b->m_capacity, sizeof(a->m_capacity)) != 0)
return false;
if (memcmp(&a->m_allocated_size, &b->m_allocated_size,
sizeof(a->m_allocated_size)) != 0)
return false;
if (memcmp(&a->m_allocated_num, &b->m_allocated_num,
sizeof(a->m_allocated_num)) != 0)
return false;
// Compare __Allocator member variables
if (memcmp(&a->m_allocator->m_size, &b->m_allocator->m_size,
sizeof(a->m_allocator->m_size)) != 0)
return false;
if (memcmp(&a->m_allocator->m_current_capacity,
&b->m_allocator->m_current_capacity,
sizeof(a->m_allocator->m_current_capacity)) != 0)
return false;
if (memcmp(&a->m_allocator->m_max_capacity,
&b->m_allocator->m_max_capacity,
sizeof(a->m_allocator->m_max_capacity)) != 0)
return false;
if (memcmp(&a->m_allocator->m_freeStorage,
&b->m_allocator->m_freeStorage,
sizeof(a->m_allocator->m_freeStorage)) != 0)
return false;
if (memcmp(&a->m_allocator->m_usedBinsTop,
&b->m_allocator->m_usedBinsTop,
sizeof(a->m_allocator->m_usedBinsTop)) != 0)
return false;
if (memcmp(&a->m_allocator->m_freeOffset, &b->m_allocator->m_freeOffset,
sizeof(a->m_allocator->m_freeOffset)) != 0)
return false;
// Compare arrays
if (memcmp(a->m_allocator->m_usedBins, b->m_allocator->m_usedBins,
NUM_TOP_BINS * sizeof(a->m_allocator->m_usedBins[0])) != 0)
return false;
if (memcmp(a->m_allocator->m_binIndices, b->m_allocator->m_binIndices,
NUM_LEAF_BINS * sizeof(a->m_allocator->m_binIndices[0])) !=
0)
return false;
// Compare Node arrays
if (memcmp(a->m_allocator->m_nodes, b->m_allocator->m_nodes,
a->m_allocator->m_current_capacity *
sizeof(a->m_allocator->m_nodes[0])) != 0)
return false;
// Compare freeNodes array
if (memcmp(a->m_allocator->m_freeNodes, b->m_allocator->m_freeNodes,
a->m_allocator->m_current_capacity *
sizeof(a->m_allocator->m_freeNodes[0])) != 0)
return false;
// All comparisons passed, allocators are equal
return true;
}
void testSerializeAllocator(
const std::shared_ptr<OffsetAllocator>& alloc_a,
const std::vector<OffsetAllocationHandle>& handles) {
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<uint32> size_dist(
1,
1024 * 64); // 1B to 64KB
// Serialize the allocator
std::vector<SerializedByte> buffer;
ASSERT_EQ(serialize_to(alloc_a, buffer), ErrorCode::OK);
// Deserialize the allocator and compare
std::shared_ptr<OffsetAllocator> alloc_b =
deserialize_from<OffsetAllocator>(buffer);
ASSERT_NE(alloc_b, nullptr);
assertAllocatorEQ(alloc_a, alloc_b);
//============== Begin test deserialization with corrupted buffer
//==============
// Set the log level to fatal to avoid the log output.
auto log_level = FLAGS_minloglevel;
FLAGS_minloglevel = google::GLOG_FATAL;
// Remove the last byte from the buffer and try to deserialize
auto corrupted_buffer = buffer;
corrupted_buffer.pop_back();
alloc_b = deserialize_from<OffsetAllocator>(corrupted_buffer);
ASSERT_TRUE(alloc_b == nullptr || !isAllocatorEqual(alloc_a, alloc_b));
// change a random bit from the buffer and try to deserialize
corrupted_buffer = buffer;
std::uniform_int_distribution<size_t> byte_index_dist(
0, buffer.size() - 1);
std::uniform_int_distribution<int> bit_index_dist(
0, 7); // 8 bits per byte
size_t byte_index = byte_index_dist(gen);
int bit_index = bit_index_dist(gen);
// Flip the random bit
corrupted_buffer[byte_index] ^= (1 << bit_index);
alloc_b = deserialize_from<OffsetAllocator>(corrupted_buffer);
// There are bool values whose value may not change if one bit is
// flipped, so the isAllocatorEqual compares variables using memcmp.
ASSERT_TRUE(alloc_b == nullptr || !isAllocatorEqual(alloc_a, alloc_b));
// Add a byte to the buffer and try to deserialize
corrupted_buffer = buffer;
corrupted_buffer.push_back(0);
alloc_b = deserialize_from<OffsetAllocator>(corrupted_buffer);
ASSERT_TRUE(alloc_b == nullptr || !isAllocatorEqual(alloc_a, alloc_b));
//============== End test deserialization with corrupted buffer
//==============
// Restore the log level.
FLAGS_minloglevel = log_level;
// Test if the deserialized allocator can properly free allocated
// objects.
alloc_b = deserialize_from<OffsetAllocator>(buffer);
ASSERT_TRUE(alloc_b != nullptr);
for (const auto& handle : handles) {
OffsetAllocationHandle handle_copy =
copyHandleWithNewAllocator(handle, alloc_b);
}
}
};
// Test basic allocation and deallocation
TEST_F(OffsetAllocatorTest, BasicAllocation) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; // 1GB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
// Allocate handle
auto handle = allocator->allocate(ALLOCATOR_SIZE);
@ -276,8 +516,7 @@ TEST_F(OffsetAllocatorTest, BasicAllocation) {
TEST_F(OffsetAllocatorTest, AllocationFailure) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; // 1GB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
// Try to allocate more than available space
auto handle =
@ -289,8 +528,7 @@ TEST_F(OffsetAllocatorTest, AllocationFailure) {
TEST_F(OffsetAllocatorTest, MultipleAllocations) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; // 1GB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
std::vector<AllocationHandleWrapper> handles;
@ -314,8 +552,7 @@ TEST_F(OffsetAllocatorTest, MultipleAllocations) {
TEST_F(OffsetAllocatorTest, DifferentSizesNoOverlap) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; // 1GB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
std::vector<AllocationHandleWrapper> handles;
std::vector<uint32> sizes = {100, 500, 1000, 2000, 50, 1500, 800, 300};
@ -337,8 +574,7 @@ TEST_F(OffsetAllocatorTest, DifferentSizesNoOverlap) {
TEST_F(OffsetAllocatorTest, StorageReports) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; // 1GB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
OffsetAllocStorageReport report = allocator->storageReport();
EXPECT_GT(report.totalFreeSpace, 0);
@ -356,8 +592,7 @@ TEST_F(OffsetAllocatorTest, StorageReports) {
TEST_F(OffsetAllocatorTest, ContinuousRandomAllocationDeallocation) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; // 1GB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
std::random_device rd;
std::mt19937 gen(rd());
@ -385,8 +620,7 @@ TEST_F(OffsetAllocatorTest, FullSizeAllocation) {
for (uint32 size : bin_sizes) {
if (size == 0) continue; // Skip 0 size
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, size, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, size, MAX_ALLOCS);
auto handle = allocator->allocate(size);
ASSERT_TRUE(handle.has_value());
@ -398,12 +632,11 @@ TEST_F(OffsetAllocatorTest, RepeatedLargeSizeAllocation) {
uint32_t bin_size = bin_sizes[i];
if (bin_size < 1024) continue; // Skip small sizes
constexpr uint32_t MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, bin_size + 10, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, bin_size + 10, MAX_ALLOCS);
EXPECT_EQ(allocator->storageReport().totalFreeSpace, bin_size + 10);
for (uint32_t i = 0; i < 10; i++) {
auto handle = allocator->allocate(bin_size - (10 - i));
for (uint32_t j = 0; j < 10; j++) {
auto handle = allocator->allocate(bin_size - (10 - j));
ASSERT_TRUE(handle.has_value());
}
}
@ -413,8 +646,7 @@ TEST_F(OffsetAllocatorTest, RepeatedLargeSizeAllocation) {
TEST_F(OffsetAllocatorTest, MaxNumAllocations) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024;
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
std::vector<AllocationHandleWrapper> handles;
for (uint32 i = 0; i < MAX_ALLOCS - 1; ++i) {
@ -431,8 +663,7 @@ TEST_F(OffsetAllocatorTest, MaxNumAllocations) {
TEST_F(OffsetAllocatorTest, FullAllocationAfterRandomAllocationAndFree) {
const uint32 ALLOCATOR_SIZE = bin_sizes[NUM_BINS - 1];
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
std::random_device rd;
std::mt19937 gen(rd());
@ -456,8 +687,7 @@ TEST_F(OffsetAllocatorTest, FullAllocationAfterRandomAllocationAndFree) {
TEST_F(OffsetAllocatorTest, AllocationSameSizeAfterFree) {
constexpr uint32 ALLOCATOR_SIZE = 2048;
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto handle = allocator->allocate(1023);
ASSERT_TRUE(handle.has_value());
@ -474,8 +704,7 @@ TEST_F(OffsetAllocatorTest, AllocationSameSizeAfterFree) {
TEST_F(OffsetAllocatorTest, RandomRepeatAllocationSameSizeAfterFree) {
const uint32 ALLOCATOR_SIZE = bin_sizes[NUM_BINS - 1];
constexpr uint32 MAX_ALLOCS = 10000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
std::random_device rd;
std::mt19937 gen(rd());
@ -483,7 +712,7 @@ TEST_F(OffsetAllocatorTest, RandomRepeatAllocationSameSizeAfterFree) {
std::vector<AllocationHandleWrapper> handles;
std::vector<uint32> alloc_sizes;
for (uint32 i = 0; i < 2000; ++i) {
for (int i = 0; i < 2000; ++i) {
uint32 size = size_dist(gen);
auto handle = allocator->allocate(size);
if (handle.has_value()) {
@ -516,8 +745,7 @@ TEST_F(OffsetAllocatorTest, BasicLargeAllocatorSize) {
for (size_t buffer_size = MIN_BUFFER_SIZE; buffer_size <= MAX_BUFFER_SIZE;
buffer_size *= 2) {
auto allocator =
std::make_shared<AllocatorWrapper>(0, buffer_size, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, buffer_size, MAX_ALLOCS);
size_t max_alloc_size = allocator->storageReport().largestFreeRegion;
// The largest free region equals buffer size only in this specific
// buffer size.
@ -548,14 +776,13 @@ TEST_F(OffsetAllocatorTest, PowerOfTwoLargeAllocatorSize) {
std::mt19937 gen(rd());
for (size_t buffer_size = MIN_BUFFER_SIZE; buffer_size <= MAX_BUFFER_SIZE;
buffer_size *= 2) {
auto allocator =
std::make_shared<AllocatorWrapper>(0, buffer_size, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, buffer_size, MAX_ALLOCS);
size_t max_alloc_size = buffer_size / 100;
std::vector<AllocationHandleWrapper> handles;
std::vector<size_t> alloc_sizes;
std::uniform_int_distribution<size_t> size_dist(1, max_alloc_size);
for (uint32 i = 0; i < 200; ++i) {
for (int i = 0; i < 200; ++i) {
size_t size = size_dist(gen);
auto handle = allocator->allocate(size);
if (handle.has_value()) {
@ -593,8 +820,7 @@ TEST_F(OffsetAllocatorTest, MaxAllocSizeWithLargeAllocatorSize) {
MAX_BUFFER_SIZE);
for (int i = 0; i < 100; i++) {
size_t buffer_size = buffer_size_dist(gen);
auto allocator =
std::make_shared<AllocatorWrapper>(0, buffer_size, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, buffer_size, MAX_ALLOCS);
size_t max_alloc_size = allocator->storageReport().largestFreeRegion;
ASSERT_GT(max_alloc_size, buffer_size / 2);
@ -616,14 +842,13 @@ TEST_F(OffsetAllocatorTest, RandomSmallAllocWithLargeAllocatorSize) {
MAX_BUFFER_SIZE);
for (int i = 0; i < 100; i++) {
size_t buffer_size = buffer_size_dist(gen);
auto allocator =
std::make_shared<AllocatorWrapper>(0, buffer_size, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, buffer_size, MAX_ALLOCS);
size_t max_alloc_size = buffer_size / 100;
std::vector<AllocationHandleWrapper> handles;
std::vector<size_t> alloc_sizes;
std::uniform_int_distribution<size_t> size_dist(1, max_alloc_size);
for (uint32 i = 0; i < 200; ++i) {
for (int j = 0; j < 200; ++j) {
size_t size = size_dist(gen);
auto handle = allocator->allocate(size);
if (handle.has_value()) {
@ -654,8 +879,7 @@ TEST_F(OffsetAllocatorTest, RandomSmallAllocWithLargeAllocatorSize) {
TEST_F(OffsetAllocatorTest, ZeroSizeAllocation) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto handle = allocator->allocate(0);
EXPECT_FALSE(handle.has_value()) << "Zero size allocation should fail";
@ -665,8 +889,7 @@ TEST_F(OffsetAllocatorTest, ZeroSizeAllocation) {
TEST_F(OffsetAllocatorTest, OneByteAllocation) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto handle = allocator->allocate(1);
ASSERT_TRUE(handle.has_value());
@ -679,8 +902,7 @@ TEST_F(OffsetAllocatorTest, OneByteAllocation) {
TEST_F(OffsetAllocatorTest, ExactCapacityAllocation) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto handle = allocator->allocate(ALLOCATOR_SIZE);
ASSERT_TRUE(handle.has_value());
@ -696,8 +918,7 @@ TEST_F(OffsetAllocatorTest, ExactCapacityAllocation) {
TEST_F(OffsetAllocatorTest, OversizeAllocation) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto handle = allocator->allocate(ALLOCATOR_SIZE + 1);
EXPECT_FALSE(handle.has_value())
@ -708,8 +929,7 @@ TEST_F(OffsetAllocatorTest, OversizeAllocation) {
TEST_F(OffsetAllocatorTest, JustBelowBinSizeAllocation) {
constexpr uint32 ALLOCATOR_SIZE = 2048;
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
// Allocate size just below a bin size
auto handle = allocator->allocate(1023);
@ -726,8 +946,7 @@ TEST_F(OffsetAllocatorTest, JustBelowBinSizeAllocation) {
TEST_F(OffsetAllocatorTest, MaxAllocationCountEdgeCase) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 10;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
std::vector<AllocationHandleWrapper> handles;
@ -754,8 +973,7 @@ TEST_F(OffsetAllocatorTest, MaxAllocationCountEdgeCase) {
TEST_F(OffsetAllocatorTest, VerySmallAllocatorSize) {
constexpr uint32 ALLOCATOR_SIZE = 16; // Very small
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto handle = allocator->allocate(16);
ASSERT_TRUE(handle.has_value());
@ -770,8 +988,7 @@ TEST_F(OffsetAllocatorTest, VerySmallAllocatorSize) {
TEST_F(OffsetAllocatorTest, AllocatorSizeMinusOne) {
constexpr uint32 ALLOCATOR_SIZE = 1024;
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto handle = allocator->allocate(ALLOCATOR_SIZE - 1);
ASSERT_TRUE(handle.has_value());
@ -782,8 +999,7 @@ TEST_F(OffsetAllocatorTest, AllocatorSizeMinusOne) {
TEST_F(OffsetAllocatorTest, PowerOfTwoAllocation) {
constexpr uint32 ALLOCATOR_SIZE = 2048;
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
// Test various power of 2 sizes
std::vector<uint32> power_of_two_sizes = {1, 2, 4, 8, 16, 32,
@ -806,8 +1022,7 @@ TEST_F(OffsetAllocatorTest, PowerOfTwoAllocation) {
TEST_F(OffsetAllocatorTest, BinSizeCalculation) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
// Test that allocations are placed in appropriate bins
// SmallFloat::uintToFloatRoundUp should determine the bin
@ -830,8 +1045,7 @@ TEST_F(OffsetAllocatorTest, BinSizeCalculation) {
TEST_F(OffsetAllocatorTest, BinOverflowScenarios) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
// Fill up a specific bin size with many small allocations
std::vector<AllocationHandleWrapper> handles;
@ -858,8 +1072,7 @@ TEST_F(OffsetAllocatorTest, BinOverflowScenarios) {
TEST_F(OffsetAllocatorTest, BinMergingBehavior) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
// Allocate three adjacent blocks
auto handle1 = allocator->allocate(1024);
@ -889,8 +1102,7 @@ TEST_F(OffsetAllocatorTest, BinMergingBehavior) {
TEST_F(OffsetAllocatorTest, BinSelectionEdgeCases) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
// Test sizes that are just below and above bin boundaries
std::vector<uint32> edge_sizes = {
@ -927,8 +1139,7 @@ TEST_F(OffsetAllocatorTest, BinSelectionEdgeCases) {
TEST_F(OffsetAllocatorTest, BinSystemLargeAllocations) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024 * 1024; // 1GB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
// Test large allocations that should go into high-numbered bins
std::vector<uint32> large_sizes = {
@ -959,8 +1170,7 @@ TEST_F(OffsetAllocatorTest, BinSystemLargeAllocations) {
TEST_F(OffsetAllocatorTest, BinSystemMixedPatterns) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
std::vector<AllocationHandleWrapper> handles;
@ -984,8 +1194,7 @@ TEST_F(OffsetAllocatorTest, BinSystemMixedPatterns) {
TEST_F(OffsetAllocatorTest, BinSystemRepeatedCycles) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
std::vector<uint32> test_sizes = {64, 128, 256, 512, 1024, 2048, 4096};
@ -1021,8 +1230,7 @@ TEST_F(OffsetAllocatorTest, BinSystemRepeatedCycles) {
TEST_F(OffsetAllocatorTest, BinSystemNonAlignedSizes) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
// Test sizes that don't align with typical bin boundaries
std::vector<uint32> non_aligned_sizes = {
@ -1049,8 +1257,7 @@ TEST_F(OffsetAllocatorTest, BinSystemNonAlignedSizes) {
TEST_F(OffsetAllocatorTest, BinSystemPrimeSizes) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
// Test sizes that are prime numbers (should be challenging for bin system)
std::vector<uint32> prime_sizes = {
@ -1081,8 +1288,7 @@ TEST_F(OffsetAllocatorTest, BinSystemPrimeSizes) {
TEST_F(OffsetAllocatorTest, BinSystemFibonacciSizes) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
// Test sizes that follow the Fibonacci sequence
std::vector<uint32> fibonacci_sizes = {
@ -1103,8 +1309,7 @@ TEST_F(OffsetAllocatorTest, BinSystemFibonacciSizes) {
TEST_F(OffsetAllocatorTest, BinSystemPageSizeMultiples) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
// Test sizes that are multiples of common page sizes (4KB, 8KB, 16KB, 64KB)
std::vector<uint32> page_size_multiples = {
@ -1132,8 +1337,7 @@ TEST_F(OffsetAllocatorTest, BinSystemPageSizeMultiples) {
TEST_F(OffsetAllocatorTest, MetricsInterface) {
constexpr uint32 ALLOCATOR_SIZE = 1024 * 1024; // 1MB
constexpr uint32 MAX_ALLOCS = 1000;
auto allocator =
std::make_shared<AllocatorWrapper>(0, ALLOCATOR_SIZE, MAX_ALLOCS);
auto allocator = AllocatorWrapper::create(0, ALLOCATOR_SIZE, MAX_ALLOCS);
// Test initial metrics - should show empty allocator
OffsetAllocatorMetrics initial_metrics = allocator->getMetrics();
@ -1195,6 +1399,220 @@ TEST_F(OffsetAllocatorTest, MetricsInterface) {
EXPECT_EQ(after_all_free.largest_free_region_, ALLOCATOR_SIZE);
}
// ========== Serialization TESTS ==========
TEST_F(OffsetAllocatorTest, SerializationEmptyAllocator) {
// Create an empty allocator
const uint64_t base = 1024 * 16;
const size_t size = 1024 * 1024;
const uint32_t init_capacity = 1000;
const uint32_t max_capacity = 10000;
std::shared_ptr<OffsetAllocator> alloc_a =
OffsetAllocator::create(base, size, init_capacity, max_capacity);
// test
testSerializeAllocator(alloc_a, {});
}
TEST_F(OffsetAllocatorTest, SerializationOneElementAllocator) {
// Create an empty allocator
const uint64_t base = 1024 * 16;
const size_t size = 1024 * 1024;
const uint32_t init_capacity = 1;
const uint32_t max_capacity = 10000;
std::shared_ptr<OffsetAllocator> alloc_a =
OffsetAllocator::create(base, size, init_capacity, max_capacity);
// Allocate one element
auto handle = alloc_a->allocate(1024);
ASSERT_TRUE(handle.has_value());
// test
std::vector<OffsetAllocationHandle> handles;
handles.push_back(std::move(*handle));
testSerializeAllocator(alloc_a, handles);
}
TEST_F(OffsetAllocatorTest, SerializationRandomAllocatedAllocator) {
// The size multiplier is larger than 1 when the allocator size is larger
// than MAX_BIN_SIZE.
constexpr size_t MIN_BUFFER_SIZE = (1ull << 31) + 1;
constexpr size_t MAX_BUFFER_SIZE = (1ull << 40);
constexpr uint32 MAX_ALLOCS = 10000;
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<size_t> buffer_size_dist(MIN_BUFFER_SIZE,
MAX_BUFFER_SIZE);
for (int i = 0; i < 100; i++) {
size_t buffer_size = buffer_size_dist(gen);
auto alloc_a = OffsetAllocator::create(0, buffer_size, 1, MAX_ALLOCS);
size_t max_alloc_size = buffer_size / 100;
std::vector<OffsetAllocationHandle> handles;
std::vector<size_t> alloc_sizes;
std::uniform_int_distribution<size_t> size_dist(1, max_alloc_size);
for (int j = 0; j < 200; ++j) {
size_t size = size_dist(gen);
auto handle = alloc_a->allocate(size);
if (handle.has_value()) {
handles.push_back(std::move(*handle));
alloc_sizes.push_back(size);
}
std::uniform_int_distribution<uint32> index_dist(
0, handles.size() - 1);
uint32 index = index_dist(gen);
std::swap(handles[index], handles.back());
std::swap(alloc_sizes[index], alloc_sizes.back());
uint32 test_size = alloc_sizes.back();
handles.pop_back();
alloc_sizes.pop_back();
auto handle2 = alloc_a->allocate(test_size);
ASSERT_TRUE(handle2.has_value());
handles.push_back(std::move(*handle2));
alloc_sizes.push_back(test_size);
}
// test
testSerializeAllocator(alloc_a, handles);
}
}
TEST_F(OffsetAllocatorTest, AllocateAfterDeserialization) {
// Create an empty allocator
const uint64_t base = 1024 * 16;
const size_t size = 1024 * 1024;
const uint32_t init_capacity = 10;
const uint32_t max_capacity = 10000;
std::shared_ptr<AllocatorWrapper> allocator =
AllocatorWrapper::create(base, size, init_capacity, max_capacity);
std::random_device rd;
std::mt19937 gen(rd());
// Do a serias of allocations and deallocations.
std::vector<AllocationHandleWrapper> handles;
for (int i = 0; i < 100; i++) {
std::uniform_int_distribution<size_t> size_dist(1, 1024);
size_t size = size_dist(gen);
auto handle = allocator->allocate(size);
ASSERT_TRUE(handle.has_value());
handles.push_back(std::move(*handle));
// Free a random handle for 50% probability
std::uniform_int_distribution<size_t> free_dist(0, 1);
if (free_dist(gen) == 1 && !handles.empty()) {
std::uniform_int_distribution<size_t> index_dist(
0, handles.size() - 1);
size_t random_index = index_dist(gen);
std::swap(handles[random_index], handles.back());
handles.pop_back();
}
}
// Serialize the allocator
std::vector<SerializedByte> buffer;
ASSERT_EQ(serialize_to(allocator->getAllocator(), buffer), ErrorCode::OK);
// Deserialize the allocator
std::shared_ptr<OffsetAllocator> alloc_b =
deserialize_from<OffsetAllocator>(buffer);
ASSERT_NE(alloc_b, nullptr);
// Substitute the allocator with the deserialized one.
allocator->substituteAllocator(alloc_b);
for (auto& handle : handles) {
substituteWithNewAllocator(handle, alloc_b);
}
// Continue to do a serias of allocations and deallocations.
for (int i = 0; i < 100; i++) {
std::uniform_int_distribution<size_t> size_dist(1, 1024);
size_t size = size_dist(gen);
auto handle = allocator->allocate(size);
ASSERT_TRUE(handle.has_value());
handles.push_back(std::move(*handle));
// Free a random handle for 50% probability
std::uniform_int_distribution<size_t> free_dist(0, 1);
if (free_dist(gen) == 1 && !handles.empty()) {
std::uniform_int_distribution<size_t> index_dist(
0, handles.size() - 1);
size_t random_index = index_dist(gen);
std::swap(handles[random_index], handles.back());
handles.pop_back();
}
}
}
TEST_F(OffsetAllocatorTest, ChainedAllocationAndDeserialization) {
// The size multiplier is larger than 1 when the allocator size is larger
// than MAX_BIN_SIZE.
constexpr size_t MIN_BUFFER_SIZE = (1ull << 31) + 1;
constexpr size_t MAX_BUFFER_SIZE = (1ull << 40);
constexpr uint32 INIT_CAPACITY = 1;
constexpr uint32 MAX_ALLOCS = 10000;
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<size_t> buffer_size_dist(MIN_BUFFER_SIZE,
MAX_BUFFER_SIZE);
// Test 10 times.
for (int i = 0; i < 10; i++) {
size_t buffer_size = buffer_size_dist(gen);
auto allocator =
AllocatorWrapper::create(0, buffer_size, INIT_CAPACITY, MAX_ALLOCS);
size_t max_alloc_size = buffer_size / 100;
std::vector<AllocationHandleWrapper> handles;
std::vector<size_t> alloc_sizes;
std::uniform_int_distribution<size_t> size_dist(1, max_alloc_size);
// 10 times serilization and deserialization.
for (int j = 0; j < 10; j++) {
// 100 times allocation
for (int k = 0; k < 100; k++) {
size_t size = size_dist(gen);
auto handle = allocator->allocate(size);
if (handle.has_value()) {
handles.push_back(std::move(*handle));
alloc_sizes.push_back(size);
}
std::uniform_int_distribution<uint32> index_dist(
0, handles.size() - 1);
uint32 index = index_dist(gen);
std::swap(handles[index], handles.back());
std::swap(alloc_sizes[index], alloc_sizes.back());
uint32 test_size = alloc_sizes.back();
handles.pop_back();
alloc_sizes.pop_back();
auto handle2 = allocator->allocate(test_size);
ASSERT_TRUE(handle2.has_value());
handles.push_back(std::move(*handle2));
alloc_sizes.push_back(test_size);
}
}
// Serialize the allocator
std::vector<SerializedByte> buffer;
ASSERT_EQ(serialize_to(allocator->getAllocator(), buffer),
ErrorCode::OK);
// Deserialize the allocator
std::shared_ptr<OffsetAllocator> new_alloc =
deserialize_from<OffsetAllocator>(buffer);
ASSERT_NE(new_alloc, nullptr);
// Verify the allocator is equal to the original one.
assertAllocatorEQ(allocator->getAllocator(), new_alloc);
// Substitute the allocator with the deserialized one.
allocator->substituteAllocator(new_alloc);
// Substitute the handles with the deserialized allocator.
for (auto& handle : handles) {
substituteWithNewAllocator(handle, new_alloc);
}
}
}
} // namespace mooncake::offset_allocator
int main(int argc, char** argv) {
// Initialize Google Test
::testing::InitGoogleTest(&argc, argv);

173
mooncake-store/tests/serializer_test.cpp Normal file
View File

@ -0,0 +1,173 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
#include "serializer.h"
namespace mooncake::test {
// Example class implementing serialization following the usage documentation
class ExampleClass {
public:
ExampleClass() : id_(0), value_(0.0), name_("") {}
ExampleClass(int id, double value, const std::string& name)
: id_(id), value_(value), name_(name) {}
// Serialization method (works with both counter and writer)
template <typename T>
void serialize_to(T& serializer) const {
serializer.write(&id_, sizeof(id_));
serializer.write(&value_, sizeof(value_));
// Serialize string length first, then string data
size_t name_length = name_.length();
serializer.write(&name_length, sizeof(name_length));
if (!name_.empty()) {
serializer.write(name_.data(), name_.length());
}
}
// Deserialization method
template <typename T>
static std::shared_ptr<ExampleClass> deserialize_from(T& serializer) {
try {
auto obj = std::make_shared<ExampleClass>();
// Deserialize basic members
serializer.read(&obj->id_, sizeof(obj->id_));
serializer.read(&obj->value_, sizeof(obj->value_));
// Deserialize string
size_t name_length;
serializer.read(&name_length, sizeof(name_length));
if (name_length > 0) {
obj->name_.resize(name_length);
serializer.read(&obj->name_[0], name_length);
}
return obj;
} catch (const std::exception& e) {
return nullptr;
}
}
// Getters for testing
int getId() const { return id_; }
double getValue() const { return value_; }
const std::string& getName() const { return name_; }
// Equality operator for testing
bool operator==(const ExampleClass& other) const {
return id_ == other.id_ && value_ == other.value_ &&
name_ == other.name_;
}
protected:
int id_;
double value_;
std::string name_;
};
class ExampleClassWithException {
public:
ExampleClassWithException() : value_(0) {}
ExampleClassWithException(int value) : value_(value) {}
template <typename T>
void serialize_to(T& serializer) const {
serializer.write(&value_, sizeof(value_));
}
template <typename T>
static std::shared_ptr<ExampleClassWithException> deserialize_from(
T& serializer) {
throw std::runtime_error("throw_exception");
}
private:
int value_;
};
class SerializerTest : public ::testing::Test {
protected:
void SetUp() override {
google::InitGoogleLogging("SerializerTest");
FLAGS_logtostderr = true;
}
void TearDown() override { google::ShutdownGoogleLogging(); }
};
TEST_F(SerializerTest, ExampleClassSerialization) {
// Create an example object
ExampleClass original(42, 3.14159, "Test Object");
// Test serialization
std::vector<SerializedByte> buffer;
ASSERT_EQ(serialize_to(original, buffer), ErrorCode::OK);
ASSERT_FALSE(buffer.empty());
// Test deserialization
auto restored = deserialize_from<ExampleClass>(buffer);
ASSERT_NE(restored, nullptr);
// Verify the deserialized object matches the original
EXPECT_EQ(restored->getId(), original.getId());
EXPECT_DOUBLE_EQ(restored->getValue(), original.getValue());
EXPECT_EQ(restored->getName(), original.getName());
EXPECT_TRUE(*restored == original);
}
TEST_F(SerializerTest, ExampleClassSerializationWithSharedPtr) {
// Test with shared_ptr
auto original =
std::make_shared<ExampleClass>(777, 2.718, "Shared Pointer Test");
std::vector<SerializedByte> buffer;
ASSERT_EQ(serialize_to(original, buffer), ErrorCode::OK);
auto restored = deserialize_from<ExampleClass>(buffer);
ASSERT_NE(restored, nullptr);
EXPECT_TRUE(*restored == *original);
}
TEST_F(SerializerTest, ExampleClassSerializationNullPointer) {
// Test with null shared_ptr
std::shared_ptr<ExampleClass> null_ptr = nullptr;
std::vector<SerializedByte> buffer;
ASSERT_EQ(serialize_to(null_ptr, buffer), ErrorCode::INVALID_PARAMS);
}
TEST_F(SerializerTest, ExampleClassDeserializationCorruptedBuffer) {
// Create a valid object and serialize it
ExampleClass original(1, 1.0, "Test");
std::vector<SerializedByte> buffer;
ASSERT_EQ(serialize_to(original, buffer), ErrorCode::OK);
// Corrupt the buffer by removing the last byte
buffer.pop_back();
// Try to deserialize corrupted buffer
auto restored = deserialize_from<ExampleClass>(buffer);
EXPECT_EQ(restored, nullptr);
}
TEST_F(SerializerTest, ExampleClassDeserializationWithException) {
// Create a valid object and serialize it
ExampleClassWithException original(1);
std::vector<SerializedByte> buffer;
ASSERT_EQ(serialize_to(original, buffer), ErrorCode::OK);
// Try to deserialize the buffer, the deserialization method will throw an
// exception.
auto restored = deserialize_from<ExampleClassWithException>(buffer);
EXPECT_EQ(restored, nullptr);
}
} // namespace mooncake::test
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}