diff --git a/cmake/Jemalloc.cmake b/cmake/Jemalloc.cmake index 176c88c9b6..723ca6b081 100644 --- a/cmake/Jemalloc.cmake +++ b/cmake/Jemalloc.cmake @@ -8,6 +8,7 @@ if(USE_SANITIZER OR WIN32 OR (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") OR APPLE) return() endif() +add_definitions(-DUSE_JEMALLOC) find_path(JEMALLOC_INCLUDE_DIR NAMES jemalloc/jemalloc.h diff --git a/fdbrpc/AsyncFileCached.actor.cpp b/fdbrpc/AsyncFileCached.actor.cpp index 6354e55cd0..f91e4ab10d 100644 --- a/fdbrpc/AsyncFileCached.actor.cpp +++ b/fdbrpc/AsyncFileCached.actor.cpp @@ -29,10 +29,14 @@ static std::map, Referen EvictablePage::~EvictablePage() { if (data) { +#if defined(USE_JEMALLOC) + aligned_free(data); +#else if (pageCache->pageSize == 4096) FastAllocator<4096>::release(data); else aligned_free(data); +#endif } if (EvictablePageCache::RANDOM == pageCache->cacheEvictionType) { if (index > -1) { @@ -169,10 +173,14 @@ void AsyncFileCached::releaseZeroCopy(void* data, int length, int64_t offset) { if (o != orphanedPages.end()) { if (o->second == 1) { if (data) { +#if defined(USE_JEMALLOC) + aligned_free(data); +#else if (length == 4096) FastAllocator<4096>::release(data); else aligned_free(data); +#endif } } else { --o->second; diff --git a/fdbrpc/AsyncFileCached.actor.h b/fdbrpc/AsyncFileCached.actor.h index ea4d0c9e97..47a7b9d282 100644 --- a/fdbrpc/AsyncFileCached.actor.h +++ b/fdbrpc/AsyncFileCached.actor.h @@ -79,7 +79,11 @@ struct EvictablePageCache : ReferenceCounted { void allocate(EvictablePage* page) { try_evict(); try_evict(); +#if defined(USE_JEMALLOC) + page->data = aligned_alloc(4096, pageSize); +#else page->data = pageSize == 4096 ? FastAllocator<4096>::allocate() : aligned_alloc(4096, pageSize); +#endif if (RANDOM == cacheEvictionType) { page->index = pages.size(); pages.push_back(page); @@ -387,7 +391,11 @@ struct AFCPage : public EvictablePage, public FastAllocated { owner->orphanedPages[data] = zeroCopyRefCount; zeroCopyRefCount = 0; notReading = Void(); +#if defined(USE_JEMALLOC) + data = aligned_alloc(4096, pageCache->pageSize); +#else data = pageCache->pageSize == 4096 ? FastAllocator<4096>::allocate() : aligned_alloc(4096, pageCache->pageSize); +#endif } Future write(void const* data, int length, int offset) { diff --git a/flow/FastAlloc.h b/flow/FastAlloc.h index 6fbc6ce0af..a91102d12e 100644 --- a/flow/FastAlloc.h +++ b/flow/FastAlloc.h @@ -278,6 +278,7 @@ inline void freeFast(int size, void* ptr) { } [[nodiscard]] inline void* allocateFast4kAligned(int size) { +#if !defined(USE_JEMALLOC) // Use FastAllocator for sizes it supports to avoid internal fragmentation in some implementations of aligned_alloc if (size <= 4096) return FastAllocator<4096>::allocate(); @@ -285,10 +286,12 @@ inline void freeFast(int size, void* ptr) { return FastAllocator<8192>::allocate(); if (size <= 16384) return FastAllocator<16384>::allocate(); +#endif return aligned_alloc(4096, size); } inline void freeFast4kAligned(int size, void* ptr) { +#if !defined(USE_JEMALLOC) // Sizes supported by FastAllocator must be release via FastAllocator if (size <= 4096) return FastAllocator<4096>::release(ptr); @@ -296,6 +299,7 @@ inline void freeFast4kAligned(int size, void* ptr) { return FastAllocator<8192>::release(ptr); if (size <= 16384) return FastAllocator<16384>::release(ptr); +#endif aligned_free(ptr); }