curvefs/client: fix s3 object will not be removed

[skipci]curvefs/client: fix log level
signed-off-by: hzwuhongsong hzwuhongsong@corp.netease.com
2023-07-03 17:14:29 +08:00 · 2023-04-26 10:29:41 +08:00 · 2023-04-17 11:12:48 +08:00 · 2023-03-24 09:38:10 +08:00 · 2023-03-03 20:30:24 +08:00 · 2023-03-03 17:12:24 +08:00
42 changed files with 1019 additions and 192 deletions
--- a/curvefs/conf/client.conf
+++ b/curvefs/conf/client.conf
@ -87,6 +87,7 @@ fuseClient.dCacheLruSize=1000000
 fuseClient.enableICacheMetrics=true
 fuseClient.enableDCacheMetrics=true
 fuseClient.cto=true
+fuseClient.downloadMaxRetryTimes=3

 ### kvcache opt
 fuseClient.supportKVcache=false
@ -110,6 +111,7 @@ fuseClient.disableXattr=false
 fuseClient.maxDataSize=1024
 # default refresh data interval 30s
 fuseClient.refreshDataIntervalSec=30
+fuseClient.warmupThreadsNum=10

 #### volume
 volume.bigFileSize=1048576
--- a/curvefs/proto/metaserver.proto
+++ b/curvefs/proto/metaserver.proto
@ -218,7 +218,7 @@ message Inode {
    map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 18; // TYPE_S3 only, first is chunk index
    optional uint32 dtime = 19;
    optional uint32 openmpcount = 20; // openmpcount mount points had the file open
-    map<string, string> xattr = 21;
+    map<string, bytes> xattr = 21;
    repeated uint64 parent = 22;
 }

@ -292,7 +292,7 @@ message UpdateInodeRequest {
    map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 17;
    optional uint32 nlink = 18;
    // field 19 is left for compatibility
-    map<string, string> xattr = 20;
+    map<string, bytes> xattr = 20;
    repeated uint64 parent = 21;
    map<uint64, S3ChunkInfoList> s3ChunkInfoAdd = 22;
    optional VolumeExtentList volumeExtents = 23;
@ -385,7 +385,7 @@ message InodeAttr {
    optional uint64 rdev = 16;
    optional uint32 dtime = 17;
    optional uint32 openmpcount = 18;
-    map<string, string> xattr = 19;
+    map<string, bytes> xattr = 19;
    repeated uint64 parent = 20;
 }

@ -416,7 +416,7 @@ message BatchGetXAttrRequest {
 message XAttr {
    required uint64 inodeId = 1;
    required uint32 fsId = 2;
-    map<string, string> xAttrInfos = 3;
+    map<string, bytes> xAttrInfos = 3;
 }

 message BatchGetXAttrResponse {
--- a/curvefs/src/client/common/common.h
+++ b/curvefs/src/client/common/common.h
@ -63,7 +63,8 @@ enum class MetaServerOpType {

 std::ostream &operator<<(std::ostream &os, MetaServerOpType optype);

-const uint32_t MAXXATTRLENGTH = 256;
+const uint32_t MAX_XATTR_NAME_LENGTH = 255;
+const uint32_t MAX_XATTR_VALUE_LENGTH = 64 * 1024;

 const char kCurveFsWarmupXAttr[] = "curvefs.warmup.op";

--- a/curvefs/src/client/common/config.cpp
+++ b/curvefs/src/client/common/config.cpp
@ -291,8 +291,10 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
    conf->GetValueFatalIfFail("fuseClient.disableXattr",
                              &clientOption->disableXattr);
    conf->GetValueFatalIfFail("fuseClient.cto", &FLAGS_enableCto);
-
-
+    conf->GetValueFatalIfFail("fuseClient.downloadMaxRetryTimes",
+                              &clientOption->downloadMaxRetryTimes);
+    conf->GetValueFatalIfFail("fuseClient.warmupThreadsNum",
+                              &clientOption->warmupThreadsNum);
    LOG_IF(WARNING, conf->GetBoolValue("fuseClient.enableSplice",
                                       &clientOption->enableFuseSplice))
        << "Not found `fuseClient.enableSplice` in conf, use default value `"
--- a/curvefs/src/client/common/config.h
+++ b/curvefs/src/client/common/config.h
@ -203,6 +203,8 @@ struct FuseClientOption {
    bool enableMultiMountPointRename = false;
    bool enableFuseSplice = false;
    bool disableXattr = false;
+    uint32_t downloadMaxRetryTimes;
+    uint32_t warmupThreadsNum;
 };

 void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption);
--- a/curvefs/src/client/curve_fuse_op.cpp
+++ b/curvefs/src/client/curve_fuse_op.cpp
@ -49,7 +49,6 @@ using ::curvefs::client::FuseClient;
 using ::curvefs::client::FuseS3Client;
 using ::curvefs::client::FuseVolumeClient;
 using ::curvefs::client::common::FuseClientOption;
-using ::curvefs::client::common::MAXXATTRLENGTH;
 using ::curvefs::client::rpcclient::MdsClientImpl;
 using ::curvefs::client::rpcclient::MDSBaseClient;
 using ::curvefs::client::metric::ClientOpMetric;
@ -344,9 +343,9 @@ void FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino, const char *name,
    size_t size) {
    InflightGuard guard(&g_clientOpMetric->opGetXattr.inflightOpNum);
    LatencyUpdater updater(&g_clientOpMetric->opGetXattr.latency);
-    char buf[MAXXATTRLENGTH] = {0};
+    std::string buf;
    CURVEFS_ERROR ret = g_ClientInstance->FuseOpGetXattr(req, ino, name,
-                                                         buf, size);
+                                                         &buf, size);
    if (ret != CURVEFS_ERROR::OK) {
        g_clientOpMetric->opGetXattr.ecount << 1;
        FuseReplyErrByErrCode(req, ret);
@ -354,9 +353,9 @@ void FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino, const char *name,
    }

    if (size == 0) {
-        fuse_reply_xattr(req, strlen(buf));
+        fuse_reply_xattr(req, buf.length());
    } else {
-        fuse_reply_buf(req, buf, strlen(buf));
+        fuse_reply_buf(req, buf.data(), buf.length());
    }
 }

--- a/curvefs/src/client/dentry_cache_manager.cpp
+++ b/curvefs/src/client/dentry_cache_manager.cpp
@ -78,7 +78,6 @@ CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
            << ", parent = " << parent << ", name = " << name;
        return MetaStatusCodeToCurvefsErrCode(ret);
    }
-
    if (!curvefs::client::common::FLAGS_enableCto) {
        dCache_->Put(key, *out);
    }
--- a/curvefs/src/client/fuse_client.cpp
+++ b/curvefs/src/client/fuse_client.cpp
@ -51,7 +51,8 @@ using ::curvefs::common::S3Info;
 using ::curvefs::common::Volume;
 using ::curvefs::mds::topology::PartitionTxId;
 using ::curvefs::mds::FSStatusCode_Name;
-using ::curvefs::client::common::MAXXATTRLENGTH;
+using ::curvefs::client::common::MAX_XATTR_NAME_LENGTH;
+using ::curvefs::client::common::MAX_XATTR_VALUE_LENGTH;
 using ::curvefs::client::common::FileHandle;

 #define RETURN_IF_UNSUCCESS(action)                                            \
@ -135,7 +136,7 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
    warmUpFile_.exist = false;
    bgCmdStop_.store(false, std::memory_order_release);
    bgCmdTaskThread_ = Thread(&FuseClient::WarmUpTask, this);
-    taskFetchMetaPool_.Start(WARMUP_THREADS);
+    taskFetchMetaPool_.Start(option_.warmupThreadsNum);
    return ret3;
 }

@ -1066,7 +1067,7 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
 }

 CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
-                                         const char* name, void* value,
+                                         const char* name, std::string* value,
                                         size_t size) {
    VLOG(9) << "FuseOpGetXattr, ino: " << ino
            << ", name: " << name << ", size = " << size;
@ -1074,7 +1075,6 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
        return CURVEFS_ERROR::NOTSUPPORT;
    }

-    std::string xValue;
    InodeAttr inodeAttr;
    CURVEFS_ERROR ret = inodeManager_->GetInodeAttr(ino, &inodeAttr);
    if (ret != CURVEFS_ERROR::OK) {
@ -1083,17 +1083,20 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
        return ret;
    }

-    ret = xattrManager_->GetXattr(name, &xValue, &inodeAttr, enableSumInDir_);
+    ret = xattrManager_->GetXattr(name, value, &inodeAttr, enableSumInDir_);
    if (CURVEFS_ERROR::OK != ret) {
        LOG(ERROR) << "xattrManager get xattr failed, name = " << name;
        return ret;
    }

    ret = CURVEFS_ERROR::NODATA;
-    if (xValue.length() > 0) {
-        if ((size == 0 && xValue.length() <= MAXXATTRLENGTH) ||
-            (size >= xValue.length() && xValue.length() <= MAXXATTRLENGTH)) {
-            memcpy(value, xValue.c_str(), xValue.length());
+    if (value->length() > 0) {
+        if ((size == 0 && value->length() <= MAX_XATTR_VALUE_LENGTH) ||
+            (size >= value->length() &&
+                value->length() <= MAX_XATTR_VALUE_LENGTH)) {
+            VLOG(1) << "FuseOpGetXattr name = " << name
+                    << ", length = " << value->length()
+                    << ", value = " << *value;
            ret = CURVEFS_ERROR::OK;
        } else {
            ret = CURVEFS_ERROR::OUT_OF_RANGE;
@ -1105,15 +1108,17 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseClient::FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
                                         const char* name, const char* value,
                                         size_t size, int flags) {
-    VLOG(1) << "FuseOpSetXattr ino: " << ino << ", name: " << name
-            << ", value: " << value;
    if (option_.disableXattr) {
        return CURVEFS_ERROR::NOTSUPPORT;
    }

    std::string strname(name);
    std::string strvalue(value, size);
-    if (strname.length() > MAXXATTRLENGTH  || size > MAXXATTRLENGTH) {
+    VLOG(1) << "FuseOpSetXattr ino: " << ino << ", name: " << name
+            << ", size = " << size
+            << ", strvalue: " << strvalue;
+    if (strname.length() > MAX_XATTR_NAME_LENGTH  ||
+        size > MAX_XATTR_VALUE_LENGTH) {
        LOG(ERROR) << "xattr length is too long, name = " << name
                   << ", name length = " << strname.length()
                   << ", value length = " << size;
--- a/curvefs/src/client/fuse_client.h
+++ b/curvefs/src/client/fuse_client.h
@ -55,7 +55,6 @@

 #define DirectIOAlignment 512
 #define WARMUP_CHECKINTERVAL_US 1000*1000
-#define WARMUP_THREADS 10

 using ::curve::common::Atomic;
 using ::curve::common::InterruptibleSleeper;
@ -191,7 +190,7 @@ class FuseClient {
                                        struct stat* attrOut);

    virtual CURVEFS_ERROR FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
-                                         const char* name, void* value,
+                                         const char* name, std::string* value,
                                         size_t size);

    virtual CURVEFS_ERROR FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
@ -268,9 +267,9 @@ class FuseClient {
    void SetEnableSumInDir(bool enable) {
        enableSumInDir_ = enable;
    }
-    std::list<fuse_ino_t>& GetReadAheadFiles() {
+    void GetReadAheadFiles(std::list<fuse_ino_t>* readAheadFiles) {
        std::unique_lock<std::mutex> lck(fetchMtx_);
-        return readAheadFiles_;
+        *readAheadFiles = std::move(readAheadFiles_);
    }

    void GetWarmUpFile(WarmUpFileContext_t* warmUpFile) {
--- a/curvefs/src/client/fuse_s3_client.cpp
+++ b/curvefs/src/client/fuse_s3_client.cpp
@ -54,6 +54,7 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
    if (ret != CURVEFS_ERROR::OK) {
        return ret;
    }
+    downloadMaxRetryTimes_ = option.downloadMaxRetryTimes;

    // init kvcache
    if (FLAGS_supportKVcache && !InitKVCache(option.kvClientManagerOpt)) {
@ -91,7 +92,7 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
                               inodeManager_, mdsClient_, fsCacheManager,
                               nullptr, true);
    }
-
+    isWarmUping_.store(false);
    bgFetchStop_.store(false, std::memory_order_release);
    bgFetchThread_ = Thread(&FuseS3Client::BackGroundFetch, this);
    initbgFetchThread_ = true;
@ -150,8 +151,7 @@ void FuseS3Client::GetWarmUpFileList(const WarmUpFileContext_t&warmUpFile,

 void FuseS3Client::BackGroundFetch() {
    while (!bgFetchStop_.load(std::memory_order_acquire)) {
-        LOG_EVERY_N(WARNING, 100)
-            << "fetch thread start.";
+        usleep(WARMUP_CHECKINTERVAL_US);
        if (hasWarmUpTask()) {  // new warmup task
            WarmUpFileContext_t warmUpFile;
            GetWarmUpFile(&warmUpFile);
@ -166,15 +166,30 @@ void FuseS3Client::BackGroundFetch() {
        }
        {   // file need warmup
            std::list<fuse_ino_t> readAheadFiles;
-            readAheadFiles.swap(GetReadAheadFiles());
-            for (auto iter : readAheadFiles) {
-                VLOG(9) << "BackGroundFetch: " << iter;
-                fetchDataEnqueue(iter);
+            GetReadAheadFiles(&readAheadFiles);
+            if (!readAheadFiles.empty()) {
+                LOG(INFO) << "num of files is need loaded is: "
+                        << readAheadFiles.size();
+                for (auto iter : readAheadFiles) {
+                    VLOG(9) << "BackGroundFetch: " << iter;
+                    fetchDataEnqueue(iter);
+                }
            }
        }
-        LOG_EVERY_N(WARNING, 100)
-            << "fetch thread end.";
-        usleep(WARMUP_CHECKINTERVAL_US);
+        {  // objs will be downloaded
+            {
+                std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+                if (needWarmupObjs_.empty()) {
+                    continue;
+                }
+            }
+            if (isWarmUping_.exchange(true)) {
+                continue;
+            }
+            std::thread downloadThread =
+              std::thread(&FuseS3Client::WarmUpAllObjs, this);
+            downloadThread.detach();
+        }
    }
    return;
 }
@ -189,14 +204,12 @@ void FuseS3Client::fetchDataEnqueue(fuse_ino_t ino) {
                    << ", inodeid = " << ino;
            return;
        }
-        google::protobuf::Map<uint64_t, S3ChunkInfoList> *s3ChunkInfoMap
-          = nullptr;
+        google::protobuf::Map<uint64_t, S3ChunkInfoList> s3ChunkInfoMap;
        {
            ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-            s3ChunkInfoMap = inodeWrapper->GetChunkInfoMap();
+            s3ChunkInfoMap = *inodeWrapper->GetChunkInfoMap();
        }
-        if (nullptr == s3ChunkInfoMap ||
-          s3ChunkInfoMap->empty()) {
+        if (s3ChunkInfoMap.empty()) {
            return;
        }
        travelChunks(ino, s3ChunkInfoMap);
@ -205,8 +218,7 @@ void FuseS3Client::fetchDataEnqueue(fuse_ino_t ino) {
 }

 // travel and download all objs belong to the chunk
-void FuseS3Client::travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo,
-  std::list<std::pair<std::string, uint64_t>>* prefetchObjs) {
+void FuseS3Client::travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo) {
    uint64_t blockSize = s3Adaptor_->GetBlockSize();
    uint64_t chunkSize = s3Adaptor_->GetChunkSize();
    uint64_t offset, len, chunkid, compaction;
@ -227,7 +239,8 @@ void FuseS3Client::travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo,
        if (len < blockSize) {  // just one block
            auto objectName = curvefs::common::s3util::GenObjName(
                chunkid, blockIndexBegin, compaction, fsId, ino);
-            prefetchObjs->push_back(std::make_pair(objectName, len));
+            std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+            needWarmupObjs_.push_back(std::make_pair(objectName, len));
        } else {
            // the offset in the block
            uint64_t blockPos = chunkPos % blockSize;
@ -258,7 +271,8 @@ void FuseS3Client::travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo,
                travelStartIndex = blockIndexBegin + 1;
                auto objectName = curvefs::common::s3util::GenObjName(
                  chunkid, blockIndexBegin, compaction, fsId, ino);
-                prefetchObjs->push_back(std::make_pair(
+                std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+                needWarmupObjs_.push_back(std::make_pair(
                  objectName, firstBlockSize));
            } else {
                travelStartIndex = blockIndexBegin;
@ -271,7 +285,8 @@ void FuseS3Client::travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo,
                  chunkid, blockIndexEnd, compaction, fsId, ino);
                // there is no need to care about the order
                // in which objects are downloaded
-                prefetchObjs->push_back(
+                std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+                needWarmupObjs_.push_back(
                  std::make_pair(objectName, lastBlockSize));
            } else {
                travelEndIndex = blockIndexEnd;
@ -294,7 +309,11 @@ void FuseS3Client::travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo,
              blockIndex <= travelEndIndex ; blockIndex++) {
                auto objectName = curvefs::common::s3util::GenObjName(
                    chunkid, blockIndex, compaction, fsId, ino);
-                prefetchObjs->push_back(std::make_pair(objectName, blockSize));
+                {
+                    std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+                    needWarmupObjs_.push_back(
+                      std::make_pair(objectName, blockSize));
+                }
            }
        }
    }
@ -302,14 +321,24 @@ void FuseS3Client::travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo,

 // TODO(hzwuhongsong): These logics are very similar to other place,
 // try to merge it
-void FuseS3Client::WarmUpAllObjs(
-  const std::list<std::pair<std::string, uint64_t>> &prefetchObjs) {
+void FuseS3Client::WarmUpAllObjs() {
+    std::list<std::pair<std::string, uint64_t>> needWarmupObjs;
+    {
+        std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+        LOG(INFO) << "num of objs need loaded is: " << needWarmupObjs_.size();
+        needWarmupObjs = std::move(needWarmupObjs_);
+    }
    std::atomic<uint64_t> pendingReq(0);
    curve::common::CountDownEvent cond(1);
    // callback function
    GetObjectAsyncCallBack cb =
        [&](const S3Adapter *adapter,
            const std::shared_ptr<GetObjectAsyncContext> &context) {
+            if (bgFetchStop_.load()) {
+                LOG(INFO) << "need stop warmup";
+                cond.Signal();
+                return;
+            }
            if (context->retCode == 0) {
                VLOG(9) << "Get Object success: " << context->key;
                int ret = s3Adaptor_->GetDiskCacheManager()->WriteReadDirect(
@ -325,16 +354,27 @@ void FuseS3Client::WarmUpAllObjs(
                delete []context->buf;
                return;
            }
-            // todo: retry
+            if (++context->retry >= downloadMaxRetryTimes_) {
+                if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == 1) {
+                    VLOG(6) << "pendingReq is over";
+                    cond.Signal();
+                }
+                LOG(WARNING) << "Up to max retry times, "
+                             << "download object failed, key: "
+                             << context->key;
+                delete []context->buf;
+                return;
+            }
+
            LOG(WARNING) << "Get Object failed, key: " << context->key
                         << ", offset: " << context->offset;
            s3Adaptor_->GetS3Client()->DownloadAsync(context);
    };

-    pendingReq.fetch_add(prefetchObjs.size(), std::memory_order_seq_cst);
-    if (pendingReq.load(std::memory_order_seq_cst)) {
-        VLOG(9) << "wait for pendingReq";
-        for (auto iter : prefetchObjs) {
+    pendingReq.fetch_add(needWarmupObjs.size(), std::memory_order_seq_cst);
+    if (pendingReq.load()) {
+        VLOG(9) << "wait for pendingReq" << pendingReq.load();
+        for (auto iter : needWarmupObjs) {
            VLOG(9) << "download start: " << iter.first;
            std::string name = iter.first;
            uint64_t readLen = iter.second;
@ -350,25 +390,26 @@ void FuseS3Client::WarmUpAllObjs(
            context->offset = 0;
            context->len = readLen;
            context->cb = cb;
+            context->retry = 0;
            s3Adaptor_->GetS3Client()->DownloadAsync(context);
        }
        if (pendingReq.load())
            cond.Wait();
    }
+    isWarmUping_.exchange(false);
+    LOG(INFO) << "num of objs is loaded over ";
 }

-void FuseS3Client::travelChunks(fuse_ino_t ino, google::protobuf::Map<uint64_t,
-    S3ChunkInfoList> *s3ChunkInfoMap) {
+void FuseS3Client::travelChunks(
+    fuse_ino_t ino,
+    const google::protobuf::Map<uint64_t, S3ChunkInfoList>& s3ChunkInfoMap) {
    VLOG(9) << "travel chunk start: " << ino
-            << ", size: " << s3ChunkInfoMap->size();
-    std::list<std::pair<std::string, uint64_t>> prefetchObjs;
-    for (auto &iter : *s3ChunkInfoMap) {
+            << ", size: " << s3ChunkInfoMap.size();
+    for (auto const& iter : s3ChunkInfoMap) {
        VLOG(9) << "travel chunk: " << iter.first;
-        travelChunk(ino, iter.second, &prefetchObjs);
+        travelChunk(ino, iter.second);
    }
-    WarmUpAllObjs(prefetchObjs);
    VLOG(9) << "travel chunks end";
-    return;
 }

 void FuseS3Client::UnInit() {
--- a/curvefs/src/client/fuse_s3_client.h
+++ b/curvefs/src/client/fuse_s3_client.h
@ -105,14 +105,13 @@ class FuseS3Client : public FuseClient {
    // then can downlaod the objs belong to it
    void fetchDataEnqueue(fuse_ino_t ino);
    // travel all chunks
-    void travelChunks(fuse_ino_t ino, google::protobuf::Map<uint64_t,
-      S3ChunkInfoList> *s3ChunkInfoMap);
+    void travelChunks(
+        fuse_ino_t ino,
+        const google::protobuf::Map<uint64_t, S3ChunkInfoList>& s3ChunkInfoMap);
    // travel and download all objs belong to the chunk
-    void travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo,
-      std::list<std::pair<std::string, uint64_t>>* prefetchObjs);
+    void travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo);
    // warmup all the prefetchObjs
-    void WarmUpAllObjs(const std::list<
-      std::pair<std::string, uint64_t>> &prefetchObjs);
+    void WarmUpAllObjs();

 private:
    // s3 adaptor
@ -121,7 +120,10 @@ class FuseS3Client : public FuseClient {
    bool initbgFetchThread_;
    Thread bgFetchThread_;
    std::atomic<bool> bgFetchStop_;
-    std::mutex fetchMtx_;
+    uint32_t downloadMaxRetryTimes_;
+    std::mutex warmupObjsMtx_;
+    std::atomic<bool> isWarmUping_;
+    std::list<std::pair<std::string, uint64_t>> needWarmupObjs_;
 };


--- a/curvefs/src/client/inode_wrapper.cpp
+++ b/curvefs/src/client/inode_wrapper.cpp
@ -457,7 +457,7 @@ CURVEFS_ERROR InodeWrapper::Sync(bool internal) {
 }

 void InodeWrapper::Async(MetaServerClientDone *done, bool internal) {
-    VLOG(3) << "async inode: " << inode_.ShortDebugString();
+    VLOG(9) << "async inode: " << inode_.ShortDebugString();

    switch (inode_.type()) {
        case FsFileType::TYPE_S3:
--- a/curvefs/src/client/kvclient/kvclient_manager.cpp
+++ b/curvefs/src/client/kvclient/kvclient_manager.cpp
@ -34,13 +34,11 @@ namespace client {
 KVClientManager *g_kvClientManager = nullptr;
 KVClientMetric *g_kvClientMetric = nullptr;

-#define ONRETURN(TYPE, RES, KEY, ERRORLOG)                                     \
+#define ONRETURN(TYPE, RES)                                                    \
    if (RES) {                                                                 \
        g_kvClientMetric->kvClient##TYPE.qps.count << 1;                       \
-        VLOG(9) << "##TYPE key = " << KEY << " OK";                            \
    } else {                                                                   \
        g_kvClientMetric->kvClient##TYPE.eps.count << 1;                       \
-        LOG(ERROR) << "##TYPE key = " << KEY << " error = " << ERRORLOG;       \
    }

 bool KVClientManager::Init(const KVClientManagerOpt &config,
@ -63,27 +61,23 @@ void KVClientManager::Set(std::shared_ptr<SetKVCacheTask> task) {
        std::string error_log;
        auto res =
            client_->Set(task->key, task->value, task->length, &error_log);
-        ONRETURN(Set, res, task->key, error_log);
+        ONRETURN(Set, res);

        task->done(task);
    });
 }

-bool KVClientManager::Get(std::shared_ptr<GetKvCacheContext> task) {
-    assert(nullptr != task->value);
-    return Get(task->key, task->value, task->offset, task->length);
-}
+void KVClientManager::Get(std::shared_ptr<GetKVCacheTask> task) {
+    threadPool_.Enqueue([task, this]() {
+        LatencyGuard guard(&g_kvClientMetric->kvClientGet.latency);

+        std::string error_log;
+        task->res = client_->Get(task->key, task->value, task->offset,
+                                task->length, &error_log);
+        ONRETURN(Get, task->res);

-bool KVClientManager::Get(const std::string &key, char *value, uint64_t offset,
-                          uint64_t length) {
-    LatencyGuard guard(&g_kvClientMetric->kvClientGet.latency);
-
-    assert(nullptr != value);
-    std::string error_log;
-    auto res = client_->Get(key, value, offset, length, &error_log);
-    ONRETURN(Get, res, key, error_log);
-    return res;
+        task->done(task);
+    });
 }

 }  // namespace client
--- a/curvefs/src/client/kvclient/kvclient_manager.h
+++ b/curvefs/src/client/kvclient/kvclient_manager.h
@ -44,6 +44,7 @@ namespace client {

 class KVClientManager;
 class SetKVCacheTask;
+class GetKVCacheTask;
 using curve::common::TaskThreadPool;
 using curvefs::client::common::KVClientManagerOpt;

@ -52,6 +53,8 @@ extern KVClientMetric *g_kvClientMetric;

 typedef std::function<void(const std::shared_ptr<SetKVCacheTask> &)>
    SetKVCacheDone;
+typedef std::function<void(const std::shared_ptr<GetKVCacheTask> &)>
+    GetKVCacheDone;

 struct SetKVCacheTask {
    std::string key;
@ -60,16 +63,22 @@ struct SetKVCacheTask {
    SetKVCacheDone done;
    SetKVCacheTask() = default;
    SetKVCacheTask(const std::string &k, const char *val, const uint64_t len)
-        : key(k), value(val), length(len) {}
+        : key(k), value(val), length(len) {
+        done = [](const std::shared_ptr<SetKVCacheTask> &) {};
+    }
 };

-struct GetKvCacheContext {
+struct GetKVCacheTask {
    const std::string &key;
    char *value;
    uint64_t offset;
    uint64_t length;
-    GetKvCacheContext(const std::string &k, char *v, uint64_t off, uint64_t len)
-        : key(k), value(v), offset(off), length(len) {}
+    bool res;
+    GetKVCacheDone done;
+    GetKVCacheTask(const std::string &k, char *v, uint64_t off, uint64_t len)
+        : key(k), value(v), offset(off), length(len), res(false) {
+        done = [](const std::shared_ptr<GetKVCacheTask> &) {};
+    }
 };

 class KVClientManager {
@ -87,10 +96,7 @@ class KVClientManager {
     */
    void Set(std::shared_ptr<SetKVCacheTask> task);

-    bool Get(const std::string &key, char *value, uint64_t offset,
-             uint64_t length);
-
-    bool Get(std::shared_ptr<GetKvCacheContext> task);
+    void Get(std::shared_ptr<GetKVCacheTask> task);

 private:
    void Uninit();
--- a/curvefs/src/client/kvclient/memcache_client.h
+++ b/curvefs/src/client/kvclient/memcache_client.h
@ -92,12 +92,9 @@ class MemCachedClient : public KVClient {
                return false;
            }
        }
-
        memcached_behavior_set(client_, MEMCACHED_BEHAVIOR_DISTRIBUTION,
                               MEMCACHED_DISTRIBUTION_CONSISTENT);
        memcached_behavior_set(client_, MEMCACHED_BEHAVIOR_RETRY_TIMEOUT, 5);
-        memcached_behavior_set(client_,
-                               MEMCACHED_BEHAVIOR_REMOVE_FAILED_SERVERS, 1);

        return PushServer();
    }
@ -117,9 +114,13 @@ class MemCachedClient : public KVClient {
        auto res = memcached_set(tcli, key.c_str(), key.length(), value,
                                 value_len, 0, 0);
        if (MEMCACHED_SUCCESS == res) {
+            VLOG(9) << "Set key = " << key << " OK";
            return true;
        }
        *errorlog = ResError(res);
+        memcached_free(tcli);
+        tcli = nullptr;
+        LOG(ERROR) << "Set key = " << key << " error = " << *errorlog;
        return false;
    }

@ -135,13 +136,23 @@ class MemCachedClient : public KVClient {
        memcached_return_t ue;
        char *res = memcached_get(tcli, key.c_str(), key.length(),
                                  &value_length, &flags, &ue);
-        if (res != nullptr && value) {
+        if (MEMCACHED_SUCCESS == ue && res != nullptr && value &&
+            value_length >= length) {
+            VLOG(9) << "Get key = " << key << " OK";
            memcpy(value, res + offset, length);
            free(res);
            return true;
        }

        *errorlog = ResError(ue);
+        if (ue != MEMCACHED_NOTFOUND) {
+          LOG(ERROR) << "Get key = " << key << " error = " << *errorlog
+                     << ", get_value_len = " << value_length
+                     << ", expect_value_len = " << length;
+          memcached_free(tcli);
+          tcli = nullptr;
+        }
+
        return false;
    }

--- a/curvefs/src/client/s3/client_s3_cache_manager.cpp
+++ b/curvefs/src/client/s3/client_s3_cache_manager.cpp
@ -513,7 +513,16 @@ bool FileCacheManager::ReadKVRequestFromRemoteCache(const std::string &name,
        return false;
    }

-    return g_kvClientManager->Get(name, databuf, offset, length);
+    auto task = std::make_shared<GetKVCacheTask>(name, databuf, offset, length);
+    CountDownEvent event(1);
+    task->done = [&](const std::shared_ptr<GetKVCacheTask> &task) {
+        event.Signal();
+        return;
+    };
+    g_kvClientManager->Get(task);
+    event.Wait();
+
+    return task->res;
 }

 bool FileCacheManager::ReadKVRequestFromS3(const std::string &name,
--- a/curvefs/src/mds/fs_manager.cpp
+++ b/curvefs/src/mds/fs_manager.cpp
@ -263,11 +263,6 @@ FSStatusCode FsManager::CreateFs(const ::curvefs::mds::CreateFsRequest* request,
    const auto& fsType = request->fstype();
    const auto& detail = request->fsdetail();

-    // check fsname
-    if (!CheckFsName(fsName)) {
-        return FSStatusCode::FSNAME_INVALID;
-    }
-
    NameLockGuard lock(nameLock_, fsName);
    FsInfoWrapper wrapper;
    bool skipCreateNewFs = false;
@ -299,6 +294,11 @@ FSStatusCode FsManager::CreateFs(const ::curvefs::mds::CreateFsRequest* request,
        }
    }

+    // check fsname
+    if (!CheckFsName(fsName)) {
+        return FSStatusCode::FSNAME_INVALID;
+    }
+
    // check s3info
    if (!skipCreateNewFs && detail.has_s3info()) {
        const auto& s3Info = detail.s3info();
--- a/curvefs/src/mds/topology/topology.cpp
+++ b/curvefs/src/mds/topology/topology.cpp
@ -1735,12 +1735,12 @@ TopoStatusCode TopologyImpl::AllocOrGetMemcacheCluster(
        int randId =
            static_cast<int>(butil::fast_rand()) % memcacheClusterMap_.size();
        auto iter = memcacheClusterMap_.cbegin();
-        for (int i = 0; i < randId; ++i, ++iter) continue;
-        *cluster = iter->second;
-        if (!storage_->StorageFs2MemcacheCluster(fsId, cluster->clusterid())) {
+        std::advance(iter, randId);
+        if (!storage_->StorageFs2MemcacheCluster(fsId, iter->first)) {
            ret = TopoStatusCode::TOPO_STORGE_FAIL;
        } else {
-            fs2MemcacheCluster_[fsId] = cluster->clusterid();
+            fs2MemcacheCluster_[fsId] = iter->first;
+            *cluster = iter->second;
        }
    }
    return ret;
--- a/curvefs/src/mds/topology/topology_item.h
+++ b/curvefs/src/mds/topology/topology_item.h
@ -823,6 +823,8 @@ class MemcacheServer {
    MemcacheServer() : port_(0) {}
    explicit MemcacheServer(const MemcacheServerInfo& info)
        : ip_(info.ip()), port_(info.port()) {}
+    explicit MemcacheServer(const std::string&& ip, uint32_t port)
+        : ip_(ip), port_(port) {}

    MemcacheServer& operator=(const MemcacheServerInfo& info) {
        ip_ = info.ip();
@ -888,6 +890,19 @@ class MemcacheCluster {
        return info;
    }

+    bool operator==(const MemcacheCluster& rhs) const {
+        if (rhs.id_ != id_ || rhs.servers_.size() != servers_.size()) {
+            return false;
+        }
+        for (auto const& server : servers_) {
+            if (std::find(rhs.servers_.cbegin(), rhs.servers_.cend(), server) ==
+                rhs.servers_.cend()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
    std::list<MemcacheServer> GetServers() const {
        return servers_;
    }
@ -899,6 +914,10 @@ class MemcacheCluster {
    bool ParseFromString(const std::string& value);
    bool SerializeToString(std::string* value) const;

+    void SetId(MetaServerIdType id) {
+        id_ = id;
+    }
+
 private:
    MetaServerIdType id_;
    std::list<MemcacheServer> servers_;
--- a/curvefs/src/mds/topology/topology_manager.cpp
+++ b/curvefs/src/mds/topology/topology_manager.cpp
@ -1327,6 +1327,20 @@ void TopologyManager::RegistMemcacheCluster(
    // register memcacheCluster as server
    WriteLockGuard lock(registMemcacheClusterMutex_);

+    // idempotence
+    std::list<MemcacheCluster> clusterList = topology_->ListMemcacheClusters();
+    MemcacheCluster mCluster(
+        0, std::list<MemcacheServer>(request->servers().begin(),
+                                     request->servers().end()));
+    for (auto const& cluster : clusterList) {
+        mCluster.SetId(cluster.GetId());
+        if (cluster == mCluster) {
+            // has registered memcache cluster
+            response->set_clusterid(cluster.GetId());
+            return;
+        }
+    }
+
    // Guarantee the uniqueness of memcacheServer
    std::list<MemcacheServer> serverRegisted = topology_->ListMemcacheServers();
    std::list<MemcacheServer> serverList;
--- a/curvefs/src/mds/topology/topology_storage_codec.cpp
+++ b/curvefs/src/mds/topology/topology_storage_codec.cpp
@ -22,7 +22,9 @@
 #include "curvefs/src/mds/topology/topology_storage_codec.h"

 #include <string>
+#include <utility>
 #include "curvefs/src/mds/common/storage_key.h"
+#include "src/common/encode.h"

 namespace curvefs {
 namespace mds {
@ -178,6 +180,13 @@ std::string TopologyStorageCodec::EncodeFs2MemcacheClusterKey(FsIdType fsId) {
    return key;
 }

+bool TopologyStorageCodec::DecodeFs2MemcacheClusterKey(const std::string& value,
+                                                       FsIdType* data) {
+    size_t prefixLen = TOPOLOGY_PREFIX_LENGTH;
+    *data = curve::common::DecodeBigEndian(&(value[prefixLen]));
+    return true;
+}
+
 }  // namespace topology
 }  // namespace mds
 }  // namespace curvefs
--- a/curvefs/src/mds/topology/topology_storage_codec.h
+++ b/curvefs/src/mds/topology/topology_storage_codec.h
@ -89,6 +89,7 @@ class TopologyStorageCodec {
                                  MemcacheCluster* data);

    std::string EncodeFs2MemcacheClusterKey(FsIdType fsId);
+    bool DecodeFs2MemcacheClusterKey(const std::string& value, FsIdType* data);
 };


--- a/curvefs/src/mds/topology/topology_storge_etcd.cpp
+++ b/curvefs/src/mds/topology/topology_storge_etcd.cpp
@ -21,6 +21,7 @@
 */
 #include "curvefs/src/mds/topology/topology_storge_etcd.h"

+#include <exception>
 #include <string>
 #include <vector>
 #include <map>
@ -625,8 +626,18 @@ bool TopologyStorageEtcd::LoadFs2MemcacheCluster(
        return false;
    }
    for (auto const& data : out) {
-        fs2MemcacheCluster->emplace(
-            std::make_pair(std::stoul(data.first), std::stoul(data.second)));
+        FsIdType id;
+        codec_->DecodeFs2MemcacheClusterKey(data.first, &id);
+        if (fs2MemcacheCluster->find(id) != fs2MemcacheCluster->end()) {
+            // Duplicated id
+            return false;
+        }
+        try {
+            fs2MemcacheCluster->emplace(id, std::stoul(data.second));
+        } catch (std::exception) {
+            // for std::stoul(data.second) exceptions
+            return false;
+        }
    }
    return true;
 }
--- a/curvefs/src/metaserver/trash.cpp
+++ b/curvefs/src/metaserver/trash.cpp
@ -168,6 +168,21 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
        clientAdaptorOption.chunkSize = s3Info.chunksize();
        s3Adaptor_->Reinit(clientAdaptorOption, s3Info.ak(), s3Info.sk(),
            s3Info.endpoint(), s3Info.bucketname());
+        ret = inodeStorage_->PaddingInodeS3ChunkInfo(item.fsId,
+          item.inodeId, inode.mutable_s3chunkinfomap());
+        if (ret != MetaStatusCode::OK) {
+            LOG(ERROR) << "GetInode chunklist fail, fsId = " << item.fsId
+                << ", inodeId = " << item.inodeId
+                << ", retCode = " << MetaStatusCode_Name(ret);
+            return ret;
+        }
+        if (inode.s3chunkinfomap().empty()) {
+            LOG(WARNING) << "GetInode chunklist empty, fsId = " << item.fsId
+                << ", inodeId = " << item.inodeId;
+            return MetaStatusCode::NOT_FOUND;
+        }
+        VLOG(9) << "DeleteInodeAndData, inode: "
+            << inode.ShortDebugString();
        int retVal = s3Adaptor_->Delete(inode);
        if (retVal != 0) {
            LOG(ERROR) << "S3ClientAdaptor delete s3 data failed"
@ -176,7 +191,6 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
            return MetaStatusCode::S3_DELETE_ERR;
        }
    }
-
    ret = inodeStorage_->Delete(Key4Inode(item.fsId, item.inodeId));
    if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
        LOG(ERROR) << "Delete Inode fail, fsId = " << item.fsId
--- a/curvefs/test/client/client_memcache_test.cpp
+++ b/curvefs/test/client/client_memcache_test.cpp
@ -73,6 +73,7 @@ class MemCachedTest : public ::testing::Test {
            std::this_thread::sleep_for(std::chrono::milliseconds(100));
            retry++;
        } while (1);
+        LOG(INFO) << "=============== memcache start ok";
    }

    void TearDown() {
@ -116,11 +117,18 @@ TEST_F(MemCachedTest, MultiThreadTask) {
    // get
    for (int i = 0; i < 5; i++) {
        workers.emplace_back([&, i]() {
+            CountDownEvent taskEnvent(1);
            char *result = new char[4];
-            auto context = std::make_shared<GetKvCacheContext>(kvstr[i].first,
-                                                               result, 0, 4);
-            ASSERT_EQ(true, manager_.Get(context));
+            auto task =
+                std::make_shared<GetKVCacheTask>(kvstr[i].first, result, 0, 4);
+            task->done =
+                [&taskEnvent](const std::shared_ptr<GetKVCacheTask> &task) {
+                    taskEnvent.Signal();
+                };
+            manager_.Get(task);
+            taskEnvent.Wait();
            ASSERT_EQ(0, memcmp(result, kvstr[i].second.c_str(), 4));
+            ASSERT_TRUE(task->res);
        });
    }
    for (auto &iter : workers) {
--- a/curvefs/test/client/rpcclient/base_client_test.cpp
+++ b/curvefs/test/client/rpcclient/base_client_test.cpp
@ -419,6 +419,30 @@ TEST_F(BaseClientTest, test_RefreshSession) {
        << response.ShortDebugString();
 }

+TEST_F(BaseClientTest, test_AllocOrGetMemcacheCluster) {
+    AllocOrGetMemcacheClusterResponse response;
+    AllocOrGetMemcacheClusterResponse resp;
+    response.set_statuscode(mds::topology::TOPO_OK);
+    EXPECT_CALL(mockTopologyService_, AllocOrGetMemcacheCluster(_, _, _, _))
+        .WillOnce(DoAll(SetArgPointee<2>(response),
+                        Invoke(RpcService<AllocOrGetMemcacheClusterRequest,
+                                          AllocOrGetMemcacheClusterResponse>)));
+
+    brpc::Controller cntl;
+    cntl.set_timeout_ms(1000);
+    brpc::Channel ch;
+    ASSERT_EQ(0, ch.Init(addr_.c_str(), nullptr));
+
+    mdsbasecli_.AllocOrGetMemcacheCluster(1, &resp, &cntl, &ch);
+
+    ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText();
+    ASSERT_TRUE(
+        google::protobuf::util::MessageDifferencer::Equals(resp, response))
+        << "resp:\n"
+        << resp.ShortDebugString() << "response:\n"
+        << response.ShortDebugString();
+}
+
 }  // namespace rpcclient
 }  // namespace client
 }  // namespace curvefs
--- a/curvefs/test/client/rpcclient/mds_client_test.cpp
+++ b/curvefs/test/client/rpcclient/mds_client_test.cpp
@ -24,6 +24,7 @@
 #include <gmock/gmock-more-actions.h>
 #include <google/protobuf/util/message_differencer.h>
 #include <gtest/gtest.h>
+#include <cstdint>

 #include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/test/client/rpcclient/mock_mds_base_client.h"
@ -57,6 +58,12 @@ void UmountFsRpcFailed(const std::string &fsName, const Mountpoint &mountPt,
    cntl->SetFailed(112, "Not connected to");
 }

+void AllocOrGetMemcacheClusterRpcFailed(
+    uint32_t fsId, AllocOrGetMemcacheClusterResponse* response,
+    brpc::Controller* cntl, brpc::Channel* channel) {
+    cntl->SetFailed(112, "Not connected to");
+}
+
 void GetFsInfoByFsnameRpcFailed(const std::string &fsName,
                                GetFsInfoResponse *response,
                                brpc::Controller *cntl,
@ -998,6 +1005,41 @@ TEST_F(MdsClientImplTest, TestReleaseVolumeBlockGroup) {
    }
 }

+TEST_F(MdsClientImplTest, test_AllocOrGetMemcacheCluster) {
+    AllocOrGetMemcacheClusterResponse response;
+    MemcacheClusterInfo cluster1;
+    cluster1.set_clusterid(1);
+    mds::topology::MemcacheServerInfo server;
+    server.set_ip("127.0.0.1");
+    server.set_port(1);
+    *cluster1.add_servers() = server;
+    response.set_allocated_cluster(new MemcacheClusterInfo(cluster1));
+
+    // 1. ok
+    response.set_statuscode(curvefs::mds::topology::TOPO_OK);
+    EXPECT_CALL(mockmdsbasecli_, AllocOrGetMemcacheCluster(_, _, _, _))
+        .WillOnce(SetArgPointee<1>(response));
+    MemcacheClusterInfo cluster2;
+    ASSERT_EQ(true,
+              mdsclient_.AllocOrGetMemcacheCluster(1, &cluster2));
+
+    // 2. no memcached
+    response.set_statuscode(
+        curvefs::mds::topology::TOPO_MEMCACHECLUSTER_NOT_FOUND);
+    EXPECT_CALL(mockmdsbasecli_, AllocOrGetMemcacheCluster(_, _, _, _))
+        .WillOnce(SetArgPointee<1>(response));
+    ASSERT_EQ(false,
+              mdsclient_.AllocOrGetMemcacheCluster(1, &cluster2));
+
+    // 3. rpc error
+    brpc::Controller cntl;
+    cntl.SetFailed(ECONNRESET, "error connect reset");
+    EXPECT_CALL(mockmdsbasecli_, AllocOrGetMemcacheCluster(_, _, _, _))
+        .WillRepeatedly(Invoke(AllocOrGetMemcacheClusterRpcFailed));
+    ASSERT_EQ(false,
+              mdsclient_.AllocOrGetMemcacheCluster(1, &cluster2));
+}
+
 }  // namespace rpcclient
 }  // namespace client
 }  // namespace curvefs
--- a/curvefs/test/client/rpcclient/mock_mds_base_client.h
+++ b/curvefs/test/client/rpcclient/mock_mds_base_client.h
@ -118,6 +118,11 @@ class MockMDSBaseClient : public MDSBaseClient {
             ReleaseBlockGroupResponse *response,
             brpc::Controller *cntl,
             brpc::Channel *channel));
+
+    MOCK_METHOD4(AllocOrGetMemcacheCluster,
+                 void(uint32_t fsId,
+                      AllocOrGetMemcacheClusterResponse* response,
+                      brpc::Controller* cntl, brpc::Channel* channel));
 };
 }  // namespace rpcclient
 }  // namespace client
--- a/curvefs/test/client/rpcclient/mock_topology_service.h
+++ b/curvefs/test/client/rpcclient/mock_topology_service.h
@ -67,6 +67,14 @@ class MockTopologyService : public curvefs::mds::topology::TopologyService {
                 request,
             ::curvefs::mds::topology::GetCopysetOfPartitionResponse* response,
             ::google::protobuf::Closure* done));
+    MOCK_METHOD4(
+        AllocOrGetMemcacheCluster,
+        void(::google::protobuf::RpcController* controller,
+             const ::curvefs::mds::topology::AllocOrGetMemcacheClusterRequest*
+                 request,
+             ::curvefs::mds::topology::AllocOrGetMemcacheClusterResponse*
+                 response,
+             ::google::protobuf::Closure* done));
 };
 }  // namespace rpcclient
 }  // namespace client
--- a/curvefs/test/client/test_disk_cache_manager_impl.cpp
+++ b/curvefs/test/client/test_disk_cache_manager_impl.cpp
@ -97,6 +97,7 @@ class TestDiskCacheManagerImpl : public ::testing::Test {

 TEST_F(TestDiskCacheManagerImpl, Init) {
    S3ClientAdaptorOption s3AdaptorOption;
+    s3AdaptorOption.diskCacheOpt.threads = 10;
    EXPECT_CALL(*diskCacheManager_, Init(_, _)).WillOnce(Return(-1));
    int ret = diskCacheManagerImpl_->Init(s3AdaptorOption);
    ASSERT_EQ(-1, ret);
--- a/curvefs/test/client/test_fuse_s3_client.cpp
+++ b/curvefs/test/client/test_fuse_s3_client.cpp
@ -91,6 +91,15 @@ class TestFuseS3Client : public ::testing::Test {
                                                 s3ClientAdaptor_);
        InitOptionBasic(&fuseClientOption_);
        InitFSInfo(client_);
+        fuseClientOption_.s3Opt.s3AdaptrOpt.asyncThreadNum = 1;
+        fuseClientOption_.dummyServerStartPort = 5000;
+        fuseClientOption_.maxNameLength = 20u;
+        fuseClientOption_.listDentryThreads = 2;
+        fuseClientOption_.warmupThreadsNum = 10;
+        auto fsInfo = std::make_shared<FsInfo>();
+        fsInfo->set_fsid(fsId);
+        fsInfo->set_fsname("s3fs");
+        client_->SetFsInfo(fsInfo);
        client_->Init(fuseClientOption_);
        PrepareFsInfo();
    }
@ -117,6 +126,7 @@ class TestFuseS3Client : public ::testing::Test {
        opt->dummyServerStartPort = 5000;
        opt->maxNameLength = 20u;
        opt->listDentryThreads = 2;
+        opt->warmupThreadsNum = 10;
    }

    void InitFSInfo(std::shared_ptr<FuseS3Client> client) {
@ -1018,11 +1028,9 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotSummaryInfo) {
    fuse_ino_t ino = 1;
    const char name[] = "security.selinux";
    size_t size = 100;
-    char value[100];
-    std::memset(value, 0, 100);
+    std::string value;

-    CURVEFS_ERROR ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    CURVEFS_ERROR ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::NODATA, ret);
 }

@ -1033,8 +1041,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir) {
    const char rname[] = "curve.dir.rfbytes";
    const char name[] = "curve.dir.fbytes";
    size_t size = 100;
-    char value[100];
-    std::memset(value, 0, 100);
+    std::string value;

    // out
    uint32_t fsId = 1;
@ -1110,10 +1117,9 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir) {
        .WillOnce(
            DoAll(SetArgPointee<1>(attrs1), Return(CURVEFS_ERROR::OK)));

-    CURVEFS_ERROR ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    CURVEFS_ERROR ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
-    ASSERT_EQ(std::string(value), "4596");
+    ASSERT_EQ(value, "4596");

    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
        .WillOnce(
@ -1125,10 +1131,9 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir) {
        .WillOnce(
            DoAll(SetArgPointee<1>(attrs), Return(CURVEFS_ERROR::OK)));

-    ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
-    ASSERT_EQ(std::string(value), "4396");
+    ASSERT_EQ(value, "4396");
 }

 TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir_Failed) {
@ -1138,8 +1143,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir_Failed) {
    const char rname[] = "curve.dir.rfbytes";
    const char name[] = "curve.dir.fbytes";
    size_t size = 100;
-    char value[100];
-    std::memset(value, 0, 100);
+    std::string value;

    // out
    uint32_t fsId = 1;
@ -1176,8 +1180,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir_Failed) {
    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
        .WillOnce(DoAll(SetArgPointee<1>(inode),
                        Return(CURVEFS_ERROR::INTERNAL)));
-    CURVEFS_ERROR ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    CURVEFS_ERROR ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);

    // list dentry failed
@ -1186,8 +1189,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir_Failed) {
            DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
        .WillOnce(Return(CURVEFS_ERROR::NOTEXIST));
-    ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);

    // BatchGetInodeAttr failed
@ -1199,8 +1201,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir_Failed) {
            DoAll(SetArgPointee<1>(dlist), Return(CURVEFS_ERROR::OK)));
    EXPECT_CALL(*inodeManager_, BatchGetInodeAttr(_, _))
        .WillOnce(Return(CURVEFS_ERROR::INTERNAL));
-    ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);

    // AddUllStringToFirst  XATTRFILES failed
@ -1213,8 +1214,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir_Failed) {
    EXPECT_CALL(*inodeManager_, BatchGetInodeAttr(_, _))
        .WillOnce(
            DoAll(SetArgPointee<1>(attrs), Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);

    // AddUllStringToFirst  XATTRSUBDIRS failed
@ -1229,8 +1229,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir_Failed) {
    EXPECT_CALL(*inodeManager_, BatchGetInodeAttr(_, _))
        .WillOnce(
            DoAll(SetArgPointee<1>(attrs), Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);

    // AddUllStringToFirst  XATTRENTRIES failed
@ -1245,8 +1244,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir_Failed) {
    EXPECT_CALL(*inodeManager_, BatchGetInodeAttr(_, _))
        .WillOnce(
            DoAll(SetArgPointee<1>(attrs), Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);

    // AddUllStringToFirst  XATTRFBYTES failed
@ -1261,8 +1259,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir_Failed) {
    EXPECT_CALL(*inodeManager_, BatchGetInodeAttr(_, _))
        .WillOnce(
            DoAll(SetArgPointee<1>(attrs), Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 }

@ -1273,8 +1270,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir) {
    fuse_ino_t ino = 1;
    const char name[] = "curve.dir.rentries";
    size_t size = 100;
-    char value[100];
-    std::memset(value, 0, 100);
+    std::string value;

    // out
    uint32_t fsId = 1;
@ -1330,10 +1326,9 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir) {
        .WillOnce(
            DoAll(SetArgPointee<1>(xattrs), Return(CURVEFS_ERROR::OK)));

-    CURVEFS_ERROR ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    CURVEFS_ERROR ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
-    ASSERT_EQ(std::string(value), "6");
+    ASSERT_EQ(value, "6");
 }

 TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) {
@ -1344,8 +1339,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) {
    const char name[] = "curve.dir.entries";
    const char rname[] = "curve.dir.rentries";
    size_t size = 100;
-    char value[100];
-    std::memset(value, 0, 100);
+    std::string value;

    // out
    uint32_t fsId = 1;
@ -1387,16 +1381,14 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) {
    // get inode failed
    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
        .WillOnce(Return(CURVEFS_ERROR::INTERNAL));
-    CURVEFS_ERROR ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    CURVEFS_ERROR ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);

    // AddUllStringToFirst failed
    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
        .WillOnce(DoAll(SetArgPointee<1>(inode),
                        Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
    inode.mutable_xattr()->find(XATTRFBYTES)->second = "100";

@ -1407,8 +1399,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) {
            DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
        .WillOnce(Return(CURVEFS_ERROR::NOTEXIST));
-    ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);

    // BatchGetInodeAttr failed
@ -1423,8 +1414,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) {
            DoAll(SetArgPointee<1>(emptyDlist), Return(CURVEFS_ERROR::OK)));
    EXPECT_CALL(*inodeManager_, BatchGetXAttr(_, _))
        .WillOnce(Return(CURVEFS_ERROR::INTERNAL));
-    ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);

    // AddUllStringToFirst  XATTRFILES failed
@ -1441,8 +1431,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) {
    EXPECT_CALL(*inodeManager_, BatchGetXAttr(_, _))
        .WillOnce(
            DoAll(SetArgPointee<1>(xattrs), Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);

    // AddUllStringToFirst  XATTRSUBDIRS failed
@ -1460,8 +1449,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) {
    EXPECT_CALL(*inodeManager_, BatchGetXAttr(_, _))
        .WillOnce(
            DoAll(SetArgPointee<1>(xattrs), Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);

    // AddUllStringToFirst  XATTRENTRIES failed
@ -1479,8 +1467,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) {
    EXPECT_CALL(*inodeManager_, BatchGetXAttr(_, _))
        .WillOnce(
            DoAll(SetArgPointee<1>(xattrs), Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 }

@ -1861,9 +1848,9 @@ TEST_F(TestFuseS3Client, FuseOpSetXattr_TooLong) {
    fuse_req_t req;
    fuse_ino_t ino = 1;
    const char name[] = "security.selinux";
-    size_t size = 300;
-    char value[300];
-    std::memset(value, 0, 300);
+    size_t size = 64 * 1024 + 1;
+    char value[64 * 1024 + 1];
+    std::memset(value, 0, size);

    CURVEFS_ERROR ret = client_->FuseOpSetXattr(
        req, ino, name, value, size, 0);
--- a/curvefs/test/client/test_fuse_volume_client.cpp
+++ b/curvefs/test/client/test_fuse_volume_client.cpp
@ -97,6 +97,7 @@ class TestFuseVolumeClient : public ::testing::Test {
        fuseClientOption_.listDentryLimit = listDentryLimit_;
        fuseClientOption_.listDentryThreads = listDentryThreads_;
        fuseClientOption_.maxNameLength = 20u;
+        fuseClientOption_.warmupThreadsNum = 10;

        spaceManager_ = new MockSpaceManager();
        volumeStorage_ = new MockVolumeStorage();
--- a/curvefs/test/mds/mock/mock_topology.h
+++ b/curvefs/test/mds/mock/mock_topology.h
@ -446,6 +446,12 @@ class MockTopologyManager : public TopologyManager {
    MOCK_METHOD2(GetLatestPartitionsTxId,
                 void(const std::vector<PartitionTxId> &txIds,
                      std::vector<PartitionTxId> *needUpdate));
+
+    MOCK_METHOD2(RegistMemcacheCluster,
+                 void(const RegistMemcacheClusterRequest*,
+                      RegistMemcacheClusterResponse*));
+
+    MOCK_METHOD1(ListMemcacheCluster, void(ListMemcacheClusterResponse*));
 };

 }  // namespace topology
--- a/curvefs/test/mds/topology/test_topology.cpp
+++ b/curvefs/test/mds/topology/test_topology.cpp
@ -2343,6 +2343,35 @@ TEST_F(TestTopology, GenCopysetAddrBatch_2pool) {
    ASSERT_EQ(pool1Count, 3);
    ASSERT_EQ(pool2Count, 12);
 }
+
+TEST_F(TestTopology, test_AddMemcacheCluster_success) {
+    MemcacheCluster data(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)});
+
+    EXPECT_CALL(*storage_, StorageMemcacheCluster(_)).WillOnce(Return(true));
+
+    int ret = topology_->AddMemcacheCluster(data);
+
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, ret);
+
+    ASSERT_EQ(*topology_->ListMemcacheClusters().begin(), data);
+}
+
+TEST_F(TestTopology, test_AddMemcacheCluster_fail) {
+    MemcacheCluster data(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)});
+
+    EXPECT_CALL(*storage_, StorageMemcacheCluster(_)).WillOnce(Return(false));
+
+    int ret = topology_->AddMemcacheCluster(data);
+
+    ASSERT_EQ(TopoStatusCode::TOPO_STORGE_FAIL, ret);
+}
+
 }  // namespace topology
 }  // namespace mds
 }  // namespace curvefs
--- a/curvefs/test/mds/topology/test_topology_manager.cpp
+++ b/curvefs/test/mds/topology/test_topology_manager.cpp
@ -23,12 +23,17 @@
 #include <brpc/channel.h>
 #include <brpc/controller.h>
 #include <brpc/server.h>
+#include <gmock/gmock-matchers.h>
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include <google/protobuf/util/message_differencer.h>
+#include <algorithm>
+#include <iostream>
+#include <vector>

 #include "curvefs/proto/topology.pb.h"
 #include "curvefs/src/mds/common/mds_define.h"
+#include "curvefs/src/mds/topology/topology_item.h"
 #include "curvefs/src/mds/topology/topology_manager.h"
 #include "curvefs/test/mds/mock/mock_metaserver.h"
 #include "curvefs/test/mds/mock/mock_metaserver_client.h"
@ -218,6 +223,14 @@ class TestTopologyManager : public ::testing::Test {
        PrepareAddPartition(fsId + 1, poolId, copysetId, pId3, 1, 100, 2);
    }

+    void PrepareAddMemcacheCluster(MemcacheClusterIdType id,
+                                   const std::list<MemcacheServer>& servers) {
+        MemcacheCluster cluster(id, servers);
+        EXPECT_CALL(*storage_, StorageMemcacheCluster(_))
+            .WillOnce(Return(true));
+        topology_->AddMemcacheCluster(cluster);
+    }
+
 protected:
    std::shared_ptr<MockIdGenerator> idGenerator_;
    std::shared_ptr<MockTokenGenerator> tokenGenerator_;
@ -2842,6 +2855,175 @@ TEST_F(TestTopologyManager, test_GetCopysetMembers_Success) {
    ASSERT_THAT(*addrs.begin(), AnyOf("ip2:8887", "ip2:8888", "ip2:8889"));
 }

+TEST_F(TestTopologyManager, test_RegistMemcacheCluster_Success) {
+    RegistMemcacheClusterRequest request;
+    RegistMemcacheClusterResponse response;
+    MemcacheServerInfo server;
+    server.set_ip("127.0.0.1");
+    server.set_port(1);
+    *request.add_servers() = server;
+
+    MemcacheClusterIdType mcCId(1);
+
+    EXPECT_CALL(*idGenerator_, GenMemCacheClusterId()).WillOnce(Return(mcCId));
+    EXPECT_CALL(*storage_, StorageMemcacheCluster(_)).WillOnce(Return(true));
+
+    serviceManager_->RegistMemcacheCluster(&request, &response);
+
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response.statuscode());
+    ASSERT_TRUE(response.has_clusterid());
+    ASSERT_EQ(mcCId, response.clusterid());
+
+    // test register same cluster
+    serviceManager_->RegistMemcacheCluster(&request, &response);
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response.statuscode());
+
+    // test register register server
+    server.set_port(2);
+    *request.add_servers() = server;
+    serviceManager_->RegistMemcacheCluster(&request, &response);
+    ASSERT_EQ(TopoStatusCode::TOPO_IP_PORT_DUPLICATED, response.statuscode());
+}
+
+TEST_F(TestTopologyManager, test_RegistMemcacheCluster_AllocateIdFail) {
+    RegistMemcacheClusterRequest request;
+    RegistMemcacheClusterResponse response;
+    MemcacheServerInfo server;
+    server.set_ip("127.0.0.1");
+    server.set_port(1);
+    *request.add_servers() = server;
+
+    MemcacheClusterIdType mcCId(1);
+
+    EXPECT_CALL(*idGenerator_, GenMemCacheClusterId())
+        .WillOnce(Return(UNINITIALIZE_ID));
+
+    serviceManager_->RegistMemcacheCluster(&request, &response);
+
+    ASSERT_EQ(TopoStatusCode::TOPO_ALLOCATE_ID_FAIL, response.statuscode());
+}
+
+TEST_F(TestTopologyManager, test_ListMemcacheCluster_Success) {
+    std::vector<MemcacheCluster> clusters;
+    clusters.emplace_back(MemcacheCluster(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)}));
+    clusters.emplace_back(MemcacheCluster(
+        2, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 4),
+                                     MemcacheServer("127.0.0.1", 5),
+                                     MemcacheServer("127.0.0.1", 6)}));
+    for (auto const & cluster : clusters) {
+        PrepareAddMemcacheCluster(cluster.GetId(), cluster.GetServers());
+    }
+
+    ListMemcacheClusterResponse response;
+    serviceManager_->ListMemcacheCluster(&response);
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response.statuscode());
+    auto clusterList = response.memcacheclusters();
+    ASSERT_EQ(clusterList.size(), clusters.size());
+    for (auto const& cluster : clusterList) {
+        ASSERT_TRUE(std::find(clusters.cbegin(), clusters.cend(),
+                              static_cast<MemcacheCluster>(cluster)) !=
+                    clusters.cend());
+    }
+}
+
+TEST_F(TestTopologyManager, test_ListMemcacheCluster_NotFound) {
+    ListMemcacheClusterResponse response;
+    serviceManager_->ListMemcacheCluster(&response);
+    ASSERT_EQ(TopoStatusCode::TOPO_MEMCACHECLUSTER_NOT_FOUND,
+              response.statuscode());
+}
+
+TEST_F(TestTopologyManager, test_AllocOrGetMemcacheCluster_1_Success) {
+    std::vector<MemcacheCluster> clusters;
+    clusters.emplace_back(MemcacheCluster(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)}));
+    for (auto const& cluster : clusters) {
+        PrepareAddMemcacheCluster(cluster.GetId(), cluster.GetServers());
+    }
+
+    EXPECT_CALL(*storage_, StorageFs2MemcacheCluster(_, _))
+        .WillOnce(Return(true));
+
+    AllocOrGetMemcacheClusterRequest request;
+    AllocOrGetMemcacheClusterResponse response;
+    serviceManager_->AllocOrGetMemcacheCluster(&request, &response);
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response.statuscode());
+    ASSERT_TRUE(response.has_cluster());
+    ASSERT_EQ(clusters[0],
+              static_cast<MemcacheCluster>(response.cluster()));
+}
+
+TEST_F(TestTopologyManager, test_AllocOrGetMemcacheCluster_Success) {
+    std::vector<MemcacheCluster> clusters;
+    clusters.emplace_back(MemcacheCluster(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)}));
+    clusters.emplace_back(MemcacheCluster(
+        2, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 4),
+                                     MemcacheServer("127.0.0.1", 5),
+                                     MemcacheServer("127.0.0.1", 6)}));
+    for (auto const& cluster : clusters) {
+        PrepareAddMemcacheCluster(cluster.GetId(), cluster.GetServers());
+    }
+
+    EXPECT_CALL(*storage_, StorageFs2MemcacheCluster(_, _))
+        .WillOnce(Return(true));
+
+    AllocOrGetMemcacheClusterRequest request1;
+    request1.set_fsid(1);
+    AllocOrGetMemcacheClusterResponse response1;
+    serviceManager_->AllocOrGetMemcacheCluster(&request1, &response1);
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response1.statuscode());
+    ASSERT_TRUE(response1.has_cluster());
+
+
+    // same fsid
+    AllocOrGetMemcacheClusterResponse response2;
+    serviceManager_->AllocOrGetMemcacheCluster(&request1, &response2);
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response2.statuscode());
+    ASSERT_TRUE(response2.has_cluster());
+    ASSERT_EQ(static_cast<MemcacheCluster>(response1.cluster()),
+              static_cast<MemcacheCluster>(response2.cluster()));
+}
+
+TEST_F(TestTopologyManager, test_AllocOrGetMemcacheCluster_NotFound) {
+    AllocOrGetMemcacheClusterRequest request;
+    AllocOrGetMemcacheClusterResponse response;
+    serviceManager_->AllocOrGetMemcacheCluster(&request, &response);
+    ASSERT_EQ(TopoStatusCode::TOPO_MEMCACHECLUSTER_NOT_FOUND,
+              response.statuscode());
+}
+
+TEST_F(TestTopologyManager, test_AllocOrGetMemcacheCluster_StorageFail) {
+    std::vector<MemcacheCluster> clusters;
+    clusters.emplace_back(MemcacheCluster(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)}));
+    clusters.emplace_back(MemcacheCluster(
+        2, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 4),
+                                     MemcacheServer("127.0.0.1", 5),
+                                     MemcacheServer("127.0.0.1", 6)}));
+    for (auto const& cluster : clusters) {
+        PrepareAddMemcacheCluster(cluster.GetId(), cluster.GetServers());
+    }
+
+    AllocOrGetMemcacheClusterRequest request;
+    request.set_fsid(1);
+    AllocOrGetMemcacheClusterResponse response;
+
+    EXPECT_CALL(*storage_, StorageFs2MemcacheCluster(_, _))
+        .WillOnce(Return(false));
+    serviceManager_->AllocOrGetMemcacheCluster(&request, &response);
+    ASSERT_EQ(TopoStatusCode::TOPO_STORGE_FAIL, response.statuscode());
+}
+
 }  // namespace topology
 }  // namespace mds
 }  // namespace curvefs
--- a/curvefs/test/mds/topology/test_topology_service.cpp
+++ b/curvefs/test/mds/topology/test_topology_service.cpp
@ -28,6 +28,7 @@
 #include <brpc/server.h>
 #include <memory>

+#include "curvefs/src/mds/topology/topology_item.h"
 #include "curvefs/test/mds/mock/mock_topology.h"
 #include "curvefs/src/mds/topology/topology_service.h"
 #include "curvefs/src/mds/topology/topology_storge.h"
@ -1127,6 +1128,120 @@ TEST_F(TestTopologyService, test_GetCopysetOfPartition_fail) {
    ASSERT_EQ(TopoStatusCode::TOPO_COPYSET_NOT_FOUND, response.statuscode());
 }

+TEST_F(TestTopologyService, test_RegistMemcache_success) {
+    TopologyService_Stub stub(&channel_);
+    brpc::Controller cntl;
+    RegistMemcacheClusterRequest request;
+    MemcacheServerInfo server;
+    server.set_ip("127.0.0.1");
+    server.set_port(1);
+    *request.add_servers() = server;
+    server.set_port(2);
+    *request.add_servers() = server;
+
+    RegistMemcacheClusterResponse response;
+
+    RegistMemcacheClusterResponse reps;
+    reps.set_statuscode(TopoStatusCode::TOPO_OK);
+    EXPECT_CALL(*manager_, RegistMemcacheCluster(_, _))
+        .WillRepeatedly(SetArgPointee<1>(reps));
+
+    stub.RegistMemcacheCluster(&cntl, &request, &response, nullptr);
+
+    if (cntl.Failed()) {
+        FAIL() << cntl.ErrorText() << std::endl;
+    }
+
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response.statuscode());
+}
+
+TEST_F(TestTopologyService, test_RegistMemcache_fail) {
+    TopologyService_Stub stub(&channel_);
+    brpc::Controller cntl;
+    RegistMemcacheClusterRequest request;
+    MemcacheServerInfo server;
+    server.set_ip("127.0.0.1");
+    server.set_port(1);
+    *request.add_servers() = server;
+    server.set_port(2);
+    *request.add_servers() = server;
+
+    RegistMemcacheClusterResponse response;
+
+    RegistMemcacheClusterResponse reps;
+    reps.set_statuscode(TopoStatusCode::TOPO_IP_PORT_DUPLICATED);
+    EXPECT_CALL(*manager_, RegistMemcacheCluster(_, _))
+        .WillRepeatedly(SetArgPointee<1>(reps));
+
+    stub.RegistMemcacheCluster(&cntl, &request, &response, nullptr);
+
+    if (cntl.Failed()) {
+        FAIL() << cntl.ErrorText() << std::endl;
+    }
+
+    ASSERT_EQ(TopoStatusCode::TOPO_IP_PORT_DUPLICATED, response.statuscode());
+}
+
+TEST_F(TestTopologyService, test_ListMemcache_success) {
+    TopologyService_Stub stub(&channel_);
+    brpc::Controller cntl;
+    ListMemcacheClusterRequest request;
+    ListMemcacheClusterResponse response;
+
+    ListMemcacheClusterResponse reps;
+    reps.set_statuscode(TopoStatusCode::TOPO_OK);
+    MemcacheClusterInfo cluster;
+    cluster.set_clusterid(1);
+    MemcacheServerInfo server;
+    server.set_ip("127.0.0.1");
+    server.set_port(1);
+    *cluster.add_servers() = server;
+    *reps.add_memcacheclusters() = cluster;
+    EXPECT_CALL(*manager_, ListMemcacheCluster(_))
+        .WillRepeatedly(SetArgPointee<0>(reps));
+
+    stub.ListMemcacheCluster(&cntl, &request, &response, nullptr);
+
+    if (cntl.Failed()) {
+        FAIL() << cntl.ErrorText() << std::endl;
+    }
+
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response.statuscode());
+
+    auto memcacheclusters1 = response.memcacheclusters();
+    auto memcacheclusters2 = reps.memcacheclusters();
+    ASSERT_EQ(memcacheclusters1.size(), memcacheclusters2.size());
+    for (auto const& cluster1 : memcacheclusters1) {
+        ASSERT_NE(
+            std::find_if(memcacheclusters2.cbegin(), memcacheclusters2.cend(),
+                         [=](const MemcacheClusterInfo& cluster2) {
+                             return static_cast<MemcacheCluster>(cluster1) ==
+                                    static_cast<MemcacheCluster>(cluster2);
+                         }),
+            memcacheclusters2.cend());
+    }
+}
+
+TEST_F(TestTopologyService, test_ListMemcache_fail) {
+    TopologyService_Stub stub(&channel_);
+    brpc::Controller cntl;
+    ListMemcacheClusterRequest request;
+    ListMemcacheClusterResponse response;
+
+    ListMemcacheClusterResponse reps;
+    reps.set_statuscode(TopoStatusCode::TOPO_MEMCACHECLUSTER_NOT_FOUND);
+    EXPECT_CALL(*manager_, ListMemcacheCluster(_))
+        .WillRepeatedly(SetArgPointee<0>(reps));
+
+    stub.ListMemcacheCluster(&cntl, &request, &response, nullptr);
+
+    if (cntl.Failed()) {
+        FAIL() << cntl.ErrorText() << std::endl;
+    }
+
+    ASSERT_EQ(TopoStatusCode::TOPO_MEMCACHECLUSTER_NOT_FOUND,
+              response.statuscode());
+}

 }  // namespace topology
 }  // namespace mds
--- a/curvefs/test/mds/topology/test_topology_storage_etcd.cpp
+++ b/curvefs/test/mds/topology/test_topology_storage_etcd.cpp
@ -20,8 +20,11 @@
 * Author: wanghai01
 */

+#include <gmock/gmock-matchers.h>
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
+#include <string>
+#include <utility>

 #include "curvefs/src/mds/topology/topology_storge_etcd.h"
 #include "curvefs/test/mds/mock/mock_topology.h"
@ -545,7 +548,7 @@ TEST_F(TestTopologyStorageEtcd, test_LoadPartition_IdDuplicated) {
    ASSERT_FALSE(ret);
 }

-TEST_F(TestTopologyStorageEtcd, test_StotagePool_success) {
+TEST_F(TestTopologyStorageEtcd, test_StoragePool_success) {
    Pool::RedundanceAndPlaceMentPolicy rap;
    rap.replicaNum = 3;
    rap.copysetNum = 3;
@ -559,7 +562,7 @@ TEST_F(TestTopologyStorageEtcd, test_StotagePool_success) {
    ASSERT_TRUE(ret);
 }

-TEST_F(TestTopologyStorageEtcd, test_StotagePool_putInfoEtcdFail) {
+TEST_F(TestTopologyStorageEtcd, test_StoragePool_putInfoEtcdFail) {
    Pool::RedundanceAndPlaceMentPolicy rap;
    rap.replicaNum = 3;
    rap.copysetNum = 3;
@ -573,7 +576,7 @@ TEST_F(TestTopologyStorageEtcd, test_StotagePool_putInfoEtcdFail) {
    ASSERT_FALSE(ret);
 }

-TEST_F(TestTopologyStorageEtcd, test_StotageZone_success) {
+TEST_F(TestTopologyStorageEtcd, test_StorageZone_success) {
    Zone data(0x31, "zone", 0x21);

    EXPECT_CALL(*kvStorageClient_, Put(_, _))
@ -583,7 +586,7 @@ TEST_F(TestTopologyStorageEtcd, test_StotageZone_success) {
    ASSERT_TRUE(ret);
 }

-TEST_F(TestTopologyStorageEtcd, test_StotageZone_putInfoEtcdFail) {
+TEST_F(TestTopologyStorageEtcd, test_StorageZone_putInfoEtcdFail) {
    Zone data(0x31, "zone", 0x21);

    EXPECT_CALL(*kvStorageClient_, Put(_, _))
@ -593,7 +596,7 @@ TEST_F(TestTopologyStorageEtcd, test_StotageZone_putInfoEtcdFail) {
    ASSERT_FALSE(ret);
 }

-TEST_F(TestTopologyStorageEtcd, test_StotageServer_success) {
+TEST_F(TestTopologyStorageEtcd, test_StorageServer_success) {
    Server data(0x41, "server", "127.0.0.1", 8080, "127.0.0.1", 8080, 0x31,
                0x21);

@ -604,7 +607,7 @@ TEST_F(TestTopologyStorageEtcd, test_StotageServer_success) {
    ASSERT_TRUE(ret);
 }

-TEST_F(TestTopologyStorageEtcd, test_StotageServer_putInfoEtcdFail) {
+TEST_F(TestTopologyStorageEtcd, test_StorageServer_putInfoEtcdFail) {
    Server data(0x41, "server", "127.0.0.1", 8080, "127.0.0.1", 8080, 0x31,
                0x21);

@ -615,7 +618,7 @@ TEST_F(TestTopologyStorageEtcd, test_StotageServer_putInfoEtcdFail) {
    ASSERT_FALSE(ret);
 }

-TEST_F(TestTopologyStorageEtcd, test_StotageMetaServer_success) {
+TEST_F(TestTopologyStorageEtcd, test_StorageMetaServer_success) {
    MetaServer data(0x51, "metaserver", "token", 0x41, "127.0.0.1", 8080,
                    "127.0.0.1", 8080,
                    OnlineState::OFFLINE);
@ -627,7 +630,7 @@ TEST_F(TestTopologyStorageEtcd, test_StotageMetaServer_success) {
    ASSERT_TRUE(ret);
 }

-TEST_F(TestTopologyStorageEtcd, test_StotageMetaServer_putInfoEtcdFail) {
+TEST_F(TestTopologyStorageEtcd, test_StorageMetaServer_putInfoEtcdFail) {
    MetaServer data(0x51, "metaserver", "token", 0x41, "127.0.0.1", 8080,
                    "127.0.0.1", 8080,
                    OnlineState::OFFLINE);
@ -639,7 +642,7 @@ TEST_F(TestTopologyStorageEtcd, test_StotageMetaServer_putInfoEtcdFail) {
    ASSERT_FALSE(ret);
 }

-TEST_F(TestTopologyStorageEtcd, test_StotageCopyset_success) {
+TEST_F(TestTopologyStorageEtcd, test_StorageCopyset_success) {
    CopySetInfo data(0x11, 0x61);
    data.SetEpoch(100);
    data.SetCopySetMembers({0x51, 0x52, 0x53});
@ -651,7 +654,7 @@ TEST_F(TestTopologyStorageEtcd, test_StotageCopyset_success) {
    ASSERT_TRUE(ret);
 }

-TEST_F(TestTopologyStorageEtcd, test_StotageCopyset_putInfoEtcdFail) {
+TEST_F(TestTopologyStorageEtcd, test_StorageCopyset_putInfoEtcdFail) {
    CopySetInfo data(0x11, 0x61);
    data.SetEpoch(100);
    data.SetCopySetMembers({0x51, 0x52, 0x53});
@ -663,7 +666,7 @@ TEST_F(TestTopologyStorageEtcd, test_StotageCopyset_putInfoEtcdFail) {
    ASSERT_FALSE(ret);
 }

-TEST_F(TestTopologyStorageEtcd, test_StotagePartition_success) {
+TEST_F(TestTopologyStorageEtcd, test_StoragePartition_success) {
    Partition data(0x01, 0x11, 0x61, 0x71, 0, 100);

    EXPECT_CALL(*kvStorageClient_, Put(_, _))
@ -673,7 +676,7 @@ TEST_F(TestTopologyStorageEtcd, test_StotagePartition_success) {
    ASSERT_TRUE(ret);
 }

-TEST_F(TestTopologyStorageEtcd, test_StotagePartition_putInfoEtcdFail) {
+TEST_F(TestTopologyStorageEtcd, test_StoragePartition_putInfoEtcdFail) {
    Partition data(0x01, 0x11, 0x61, 0x71, 0, 100);

    EXPECT_CALL(*kvStorageClient_, Put(_, _))
@ -822,7 +825,7 @@ TEST_F(TestTopologyStorageEtcd, test_LoadClusterInfo_decodeError) {
    ASSERT_FALSE(ret);
 }

-TEST_F(TestTopologyStorageEtcd, test_StotageClusterInfo_success) {
+TEST_F(TestTopologyStorageEtcd, test_StorageClusterInfo_success) {
    ClusterInformation data;
    data.clusterId = "xxx";

@ -833,7 +836,7 @@ TEST_F(TestTopologyStorageEtcd, test_StotageClusterInfo_success) {
    ASSERT_TRUE(ret);
 }

-TEST_F(TestTopologyStorageEtcd, test_StotageClusterInfo_fail) {
+TEST_F(TestTopologyStorageEtcd, test_StorageClusterInfo_fail) {
    ClusterInformation data;
    data.clusterId = "xxx";

@ -844,6 +847,206 @@ TEST_F(TestTopologyStorageEtcd, test_StotageClusterInfo_fail) {
    ASSERT_FALSE(ret);
 }

+TEST_F(TestTopologyStorageEtcd, test_LoadMemcacheCluster_success) {
+    MemcacheCluster data(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)});
+
+    std::string key = codec_->EncodeMemcacheClusterKey(data.GetId());
+    std::string value;
+    ASSERT_TRUE(codec_->EncodeMemcacheClusterData(data, &value));
+    std::vector<std::string> list;
+    list.push_back(value);
+
+    EXPECT_CALL(*kvStorageClient_,
+                List(_, _, Matcher<std::vector<std::string>*>(_)))
+        .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK)));
+
+    MemcacheClusterIdType maxId(0);
+    std::unordered_map<MemcacheClusterIdType, MemcacheCluster>
+            clusterMap;
+    bool ret = storage_->LoadMemcacheCluster(&clusterMap, &maxId);
+    ASSERT_TRUE(ret);
+
+    ASSERT_EQ(1, clusterMap.size());
+    ASSERT_EQ(data, clusterMap[data.GetId()]);
+    ASSERT_EQ(1, maxId);
+}
+
+TEST_F(TestTopologyStorageEtcd,
+       test_LoadMemcacheCluster_success_listEtcdEmpty) {
+    std::vector<std::string> list;
+    EXPECT_CALL(*kvStorageClient_,
+                List(_, _, Matcher<std::vector<std::string>*>(_)))
+        .WillOnce(DoAll(SetArgPointee<2>(list),
+                        Return(EtcdErrCode::EtcdKeyNotExist)));
+
+    MemcacheClusterIdType maxId(0);
+    std::unordered_map<MemcacheClusterIdType, MemcacheCluster> clusterMap;
+    bool ret = storage_->LoadMemcacheCluster(&clusterMap, &maxId);
+    ASSERT_TRUE(ret);
+
+    ASSERT_EQ(0, clusterMap.size());
+    ASSERT_EQ(0, maxId);
+}
+
+TEST_F(TestTopologyStorageEtcd, test_LoadMemcacheCluster_decodeError) {
+    std::vector<std::string> list;
+    list.emplace_back("xxx");
+    EXPECT_CALL(*kvStorageClient_,
+                List(_, _, Matcher<std::vector<std::string>*>(_)))
+        .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK)));
+
+    MemcacheClusterIdType maxId(0);
+    std::unordered_map<MemcacheClusterIdType, MemcacheCluster> clusterMap;
+    bool ret = storage_->LoadMemcacheCluster(&clusterMap, &maxId);
+    ASSERT_FALSE(ret);
+}
+
+TEST_F(TestTopologyStorageEtcd, test_LoadMemcacheCluster_IdDuplicated) {
+    MemcacheCluster data(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)});
+
+    std::string key = codec_->EncodeMemcacheClusterKey(data.GetId());
+    std::string value;
+    ASSERT_TRUE(codec_->EncodeMemcacheClusterData(data, &value));
+
+    std::vector<std::string> list;
+    list.push_back(value);
+    list.push_back(value);
+    EXPECT_CALL(*kvStorageClient_,
+                List(_, _, Matcher<std::vector<std::string>*>(_)))
+        .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK)));
+
+    MemcacheClusterIdType maxId(0);
+    std::unordered_map<MemcacheClusterIdType, MemcacheCluster> clusterMap;
+    bool ret = storage_->LoadMemcacheCluster(&clusterMap, &maxId);
+    ASSERT_FALSE(ret);
+}
+
+TEST_F(TestTopologyStorageEtcd, test_StorageMemcacheCluster_success) {
+    MemcacheCluster data(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)});
+
+    EXPECT_CALL(*kvStorageClient_, Put(_, _))
+        .WillOnce(Return(EtcdErrCode::EtcdOK));
+
+    bool ret = storage_->StorageMemcacheCluster(data);
+    ASSERT_TRUE(ret);
+}
+
+TEST_F(TestTopologyStorageEtcd, test_StorageMemcacheCluster_putInfoEtcdFail) {
+    MemcacheCluster data(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)});
+
+    EXPECT_CALL(*kvStorageClient_, Put(_, _))
+        .WillOnce(Return(EtcdErrCode::EtcdUnknown));
+
+    bool ret = storage_->StorageMemcacheCluster(data);
+    ASSERT_FALSE(ret);
+}
+
+TEST_F(TestTopologyStorageEtcd, test_LoadFs2MemcacheCluster_success) {
+    std::unordered_map<FsIdType, MemcacheClusterIdType> data{
+        {1, 1}, {2, 2}, {3, 3}, {4, 4}};
+    std::vector<std::pair<std::string, std::string>> list;
+    for (auto const& kv : data) {
+        auto key = codec_->EncodeFs2MemcacheClusterKey(kv.first);
+        auto value = std::to_string(kv.second);
+        list.emplace_back(key, value);
+    }
+
+    EXPECT_CALL(
+        *kvStorageClient_,
+        List(_, _,
+             Matcher<std::vector<std::pair<std::string, std::string>>*>(_)))
+        .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK)));
+
+    std::unordered_map<FsIdType, MemcacheClusterIdType> loadData;
+    bool ret = storage_->LoadFs2MemcacheCluster(&loadData);
+    ASSERT_TRUE(ret);
+
+    ASSERT_EQ(data.size(), loadData.size());
+    ASSERT_EQ(data, loadData);
+}
+
+TEST_F(TestTopologyStorageEtcd,
+       test_LoadFs2MemcacheCluster_success_listEtcdEmpty) {
+    std::vector<std::pair<std::string, std::string>> list;
+    EXPECT_CALL(
+        *kvStorageClient_,
+        List(_, _,
+             Matcher<std::vector<std::pair<std::string, std::string>>*>(_)))
+        .WillOnce(DoAll(SetArgPointee<2>(list),
+                        Return(EtcdErrCode::EtcdKeyNotExist)));
+
+    std::unordered_map<FsIdType, MemcacheClusterIdType> loadData;
+    bool ret = storage_->LoadFs2MemcacheCluster(&loadData);
+
+    ASSERT_TRUE(ret);
+    ASSERT_TRUE(loadData.empty());
+}
+
+TEST_F(TestTopologyStorageEtcd, test_LoadFs2MemcacheCluster_decodeValueError) {
+    std::vector<std::pair<std::string, std::string>> list;
+    list.emplace_back("xxx", "xxx");
+
+    EXPECT_CALL(
+        *kvStorageClient_,
+        List(_, _,
+             Matcher<std::vector<std::pair<std::string, std::string>>*>(_)))
+        .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK)));
+
+    std::unordered_map<FsIdType, MemcacheClusterIdType> loadData;
+    bool ret = storage_->LoadFs2MemcacheCluster(&loadData);
+
+    ASSERT_FALSE(ret);
+}
+
+TEST_F(TestTopologyStorageEtcd, test_LoadFs2MemcacheCluster_IdDuplicated) {
+    std::unordered_map<FsIdType, MemcacheClusterIdType> data{
+        {1, 1}};
+    std::vector<std::pair<std::string, std::string>> list;
+    for (auto const& kv : data) {
+        auto key = codec_->EncodeFs2MemcacheClusterKey(kv.first);
+        auto value = std::to_string(kv.second);
+        list.emplace_back(key, value);
+    }
+    list.emplace_back(codec_->EncodeFs2MemcacheClusterKey(1), "2");
+    EXPECT_CALL(
+        *kvStorageClient_,
+        List(_, _,
+             Matcher<std::vector<std::pair<std::string, std::string>>*>(_)))
+        .WillOnce(DoAll(SetArgPointee<2>(list), Return(EtcdErrCode::EtcdOK)));
+
+    std::unordered_map<FsIdType, MemcacheClusterIdType> loadData;
+    bool ret = storage_->LoadFs2MemcacheCluster(&loadData);
+    ASSERT_FALSE(ret);
+}
+
+TEST_F(TestTopologyStorageEtcd, test_StorageFs2MemcacheCluster_success) {
+    EXPECT_CALL(*kvStorageClient_, Put(_, _))
+        .WillRepeatedly(Return(EtcdErrCode::EtcdOK));
+        bool ret = storage_->StorageFs2MemcacheCluster(1, 1);
+        ASSERT_TRUE(ret);
+}
+
+TEST_F(TestTopologyStorageEtcd,
+       test_StorageFs2MemcacheCluster_putInfoEtcdFail) {
+    EXPECT_CALL(*kvStorageClient_, Put(_, _))
+        .WillOnce(Return(EtcdErrCode::EtcdUnknown));
+
+    bool ret = storage_->StorageFs2MemcacheCluster(1, 1);
+    ASSERT_FALSE(ret);
+}
+
 }  // namespace topology
 }  // namespace mds
 }  // namespace curvefs
--- a/curvefs/test/mds/topology/topology_storage_codec_test.cpp
+++ b/curvefs/test/mds/topology/topology_storage_codec_test.cpp
@ -26,6 +26,7 @@
 #include <string>
 #include <set>

+#include "curvefs/src/mds/topology/topology_item.h"
 #include "curvefs/src/mds/topology/topology_storage_codec.h"
 #include "curvefs/test/mds/topology/test_topology_helper.h"

@ -180,6 +181,21 @@ TEST_F(TopologyStorageCodecTest, TestEncodeKeyNumEqual) {
    ASSERT_EQ(5 * keyNum + keyNum * keyRow, keySet.size());
 }

+TEST_F(TopologyStorageCodecTest, TestMemcacheClusterEncodeDecodeEqual) {
+    MemcacheCluster data(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)});
+
+    std::string value;
+    ASSERT_TRUE(testObj.EncodeMemcacheClusterData(data, &value));
+
+    MemcacheCluster out;
+    ASSERT_TRUE(testObj.DecodeMemcacheClusterData(value, &out));
+
+    ASSERT_EQ(data, out);
+}
+
 }  // namespace topology
 }  // namespace mds
 }  // namespace curvefs
--- a/curvefs/test/metaserver/BUILD
+++ b/curvefs/test/metaserver/BUILD
@ -26,7 +26,6 @@ cc_test(
            "mock_metaserver_s3.h",
            "metaserver_s3_adaptor_test.h",
            "metaserver_s3_adaptor_test.cpp",
-            "mock_metaserver_s3_adaptor.h",
            "metaserver_s3_test.cpp",
            "mock_s3compact_inode.h",
            "s3compact_test.cpp",
@ -46,6 +45,7 @@ cc_test(
            "//curvefs/test/metaserver/storage:metaserver_storage_test_utils",
            "//curvefs/test/metaserver/mock:metaserver_test_mock",
            "@com_google_absl//absl/types:optional",
+            "//curvefs/test/client/rpcclient:rpcclient_test_mock",
    ],
 )

--- a/curvefs/test/metaserver/trash_test.cpp
+++ b/curvefs/test/metaserver/trash_test.cpp
@ -28,6 +28,8 @@
 #include "curvefs/src/metaserver/storage/rocksdb_storage.h"
 #include "curvefs/test/metaserver/storage/utils.h"
 #include "src/fs/ext4_filesystem_impl.h"
+#include "curvefs/test/client/rpcclient/mock_mds_client.h"
+#include "curvefs/test/metaserver/mock_metaserver_s3_adaptor.h"

 using ::testing::AtLeast;
 using ::testing::StrEq;
@ -45,6 +47,7 @@ namespace {
 auto localfs = curve::fs::Ext4FileSystemImpl::getInstance();
 }

+using ::curvefs::client::rpcclient::MockMdsClient;
 using ::curvefs::metaserver::storage::KVStorage;
 using ::curvefs::metaserver::storage::StorageOptions;
 using ::curvefs::metaserver::storage::RocksDBStorage;
@ -103,7 +106,29 @@ class TestTrash : public ::testing::Test {
        inode.set_gid(0);
        inode.set_mode(0);
        inode.set_nlink(0);
-        inode.set_type(FsFileType::TYPE_FILE);
+        inode.set_type(FsFileType::TYPE_S3);
+        return inode;
+    }
+
+    Inode GenInodeHasChunks(uint32_t fsId, uint64_t inodeId) {
+        Inode inode;
+        inode.set_fsid(fsId);
+        inode.set_inodeid(inodeId);
+        inode.set_length(4096);
+        inode.set_ctime(0);
+        inode.set_ctime_ns(0);
+        inode.set_mtime(0);
+        inode.set_mtime_ns(0);
+        inode.set_atime(0);
+        inode.set_atime_ns(0);
+        inode.set_uid(0);
+        inode.set_gid(0);
+        inode.set_mode(0);
+        inode.set_nlink(0);
+        inode.set_type(FsFileType::TYPE_S3);
+
+        S3ChunkInfoList s3ChunkInfoList;
+        inode.mutable_s3chunkinfomap()->insert({0, s3ChunkInfoList});
        return inode;
    }

@ -119,17 +144,18 @@ TEST_F(TestTrash, testAdd3ItemAndDelete) {
    option.scanPeriodSec = 1;
    option.expiredAfterSec = 1;

+    option.mdsClient = std::make_shared<MockMdsClient>();
+    option.s3Adaptor = std::make_shared<MockS3ClientAdaptor>();
    trashManager_->Init(option);
    trashManager_->Run();
-
    auto trash1 = std::make_shared<TrashImpl>(inodeStorage_);
    auto trash2 = std::make_shared<TrashImpl>(inodeStorage_);
    trashManager_->Add(1, trash1);
    trashManager_->Add(2, trash2);

-    inodeStorage_->Insert(GenInode(1, 1));
-    inodeStorage_->Insert(GenInode(1, 2));
-    inodeStorage_->Insert(GenInode(2, 1));
+    inodeStorage_->Insert(GenInodeHasChunks(1, 1));
+    inodeStorage_->Insert(GenInodeHasChunks(1, 2));
+    inodeStorage_->Insert(GenInodeHasChunks(2, 1));

    ASSERT_EQ(inodeStorage_->Size(), 3);

@ -138,17 +164,42 @@ TEST_F(TestTrash, testAdd3ItemAndDelete) {
    trash2->Add(2, 1, 0);

    std::this_thread::sleep_for(std::chrono::seconds(5));
-
    std::list<TrashItem> list;

    trashManager_->ListItems(&list);

    ASSERT_EQ(0, list.size());
-
    ASSERT_EQ(inodeStorage_->Size(), 0);

    trashManager_->Fini();
 }

+TEST_F(TestTrash, testAdd3ItemAndNoDelete) {
+    TrashOption option;
+    option.scanPeriodSec = 1;
+    option.expiredAfterSec = 1;
+    option.mdsClient = std::make_shared<MockMdsClient>();
+    option.s3Adaptor = std::make_shared<MockS3ClientAdaptor>();
+    trashManager_->Init(option);
+    trashManager_->Run();
+
+    auto trash1 = std::make_shared<TrashImpl>(inodeStorage_);
+    trashManager_->Add(1, trash1);
+
+    inodeStorage_->Insert(GenInode(1, 1));
+    inodeStorage_->Insert(GenInode(1, 2));
+    inodeStorage_->Insert(GenInode(2, 1));
+    ASSERT_EQ(inodeStorage_->Size(), 3);
+    trash1->Add(1, 1, 0);
+    trash1->Add(1, 2, 0);
+    std::this_thread::sleep_for(std::chrono::seconds(5));
+    std::list<TrashItem> list;
+
+    trashManager_->ListItems(&list);
+    ASSERT_EQ(0, list.size());
+    ASSERT_EQ(inodeStorage_->Size(), 3);
+    trashManager_->Fini();
+}
+
 }  // namespace metaserver
 }  // namespace curvefs
--- a/src/common/encode.h
+++ b/src/common/encode.h
@ -23,6 +23,7 @@
 #define SRC_COMMON_ENCODE_H_

 #include <stdint.h>
+#include <cstdint>

 namespace curve {
 namespace common {
@ -39,6 +40,13 @@ static inline void EncodeBigEndian(char* buf, uint64_t value) {
    buf[7] = value & 0xff;
 }

+inline uint64_t DecodeBigEndian(const char* buf) {
+    return (uint64_t(buf[0]) << 56) | (uint64_t(buf[1]) << 48) |
+           (uint64_t(buf[2]) << 40) | (uint64_t(buf[3]) << 32) |
+           (uint64_t(buf[4]) << 24) | (uint64_t(buf[5]) << 16) |
+           (uint64_t(buf[6]) << 8) | uint64_t(buf[7]);
+}
+
 inline void EncodeBigEndian_uint32(char* buf, uint32_t value) {
    buf[0] = (value >> 24) & 0xff;
    buf[1] = (value >> 16) & 0xff;
--- a/src/common/s3_adapter.h
+++ b/src/common/s3_adapter.h
@ -115,6 +115,7 @@ struct GetObjectAsyncContext : public Aws::Client::AsyncCallerContext {
    size_t len;
    GetObjectAsyncCallBack cb;
    int retCode;
+    int retry;
 };

 /*
Author	SHA1	Message	Date
hzwuhongsong	b23c3e3484	curvefs/client: fix s3 object will not be removed	2023-07-03 17:14:29 +08:00
hzwuhongsong	2dc5b4ada2	[skipci]curvefs/client: fix log level signed-off-by: hzwuhongsong hzwuhongsong@corp.netease.com	2023-04-26 10:29:41 +08:00
Cyber-SiKu	d7f6b5bff3	[fix]mds/fsManager:check fs name Moved to after checking whether fs exists to ensure compatibility Signed-off-by: Cyber-SiKu <Cyber-SiKu@outlook.com>	2023-04-17 11:12:48 +08:00
ilixiaocui	3cba3502e8	curvefs/client: memcache client set nullptr after operation fail Signed-off-by: ilixiaocui <ilixiaocui@163.com>	2023-03-24 09:38:10 +08:00
ilixiaocui	c1a66b902b	curvefs/client: bug fix about memcache 1. When running the vdbech task, the memory usage of the mount point is very high 2. When running the vdbech task, the mount will coredump or the task will have data inconsistency 3. The memcache server fails over and then restarts and cannot be used by the client anymore Signed-off-by: ilixiaocui <ilixiaocui@163.com>	2023-03-03 20:30:24 +08:00
Cyber-SiKu	1420440d85	[fix]curvefs/client: test coredump Signed-off-by: Cyber-SiKu <Cyber-SiKu@outlook.com>	2023-03-03 17:12:24 +08:00
SiKu	c1c9962fcb	Revert "[feat]curvefs/client: warmup manager" This reverts commit `260e4630be`.	2023-03-02 19:25:22 +08:00
SiKu	b96f56bde1	Revert "[fix]curvefs/client/warmupManager:fix FetchS3ObjectsPoolMutex_" This reverts commit `90f64fe044`.	2023-03-02 19:25:22 +08:00
Cyber-SiKu	90f64fe044	[fix]curvefs/client/warmupManager:fix FetchS3ObjectsPoolMutex_ Signed-off-by: Cyber-SiKu <Cyber-SiKu@outlook.com>	2023-02-21 16:26:06 +08:00
Cyber-SiKu	260e4630be	[feat]curvefs/client: warmup manager 1. add WarmupManager 2. add WarmupManagerS3Impl 3. add query warmup progress in tools-v2 Signed-off-by: Cyber-SiKu <Cyber-SiKu@outlook.com>	2023-02-21 16:26:06 +08:00
hzwuhongsong	503aa3483b	curvefs/client:ptimize warmup performance	2023-02-14 11:22:55 +08:00
CodeFarmerPK	549c0fa75f	curvefs/client: limited download retry times Signed-off-by: CodeFarmerPK <CodeFarmer.PK@Gmail.com>	2023-02-14 11:22:55 +08:00
wanghai01	378684ca97	fix getxattr return wrong length Signed-off-by: wanghai01 <seanhaizi@163.com>	2023-02-02 15:13:55 +08:00
Cyber-SiKu	aa957d3c2e	[fix]curvefs/client: fix client core dump use object instaed of pointer Signed-off-by: Cyber-SiKu <Cyber-SiKu@outlook.com>	2023-01-30 10:15:10 +08:00
Cyber-SiKu	bd2c5b3438	[fix]curvefs/client init core dump && add ut 1. fix 2. add storage codec ut 3. add DecodeBigEndian 4. add storage etcd ut 5. add topology manager ut 6. add topology service ut 7. add topology ut 8. add clinet ut Signed-off-by: Cyber-SiKu <Cyber-SiKu@outlook.com>	2022-12-14 18:43:11 +08:00