[fix] corvedump & log too much

Signed-off-by: Cyber-SiKu <Cyber-SiKu@outlook.com>

.gitignore

@@ -144,3 +144,12 @@ tools-v2/proto/curvefs/*
 tools-v2/*/*.test
 tools-v2/__debug_bin
 tools-v2/vendor/
+
+.note
+.playground
+GPATH
+GRTAGS
+GTAGS
+curvefs/docker/debian9/libhashkit.so.2
+curvefs/docker/debian9/libmemcached.so.11
+curvefs/docker/debian9/libssl.so.3

Makefile

@@ -70,3 +70,6 @@ install:
 
 image:
 	@bash util/image.sh $(stor) $(tag) $(os)
+
+playground:
+	@bash util/playground.sh

WORKSPACE

@@ -228,6 +228,24 @@ http_archive(
   sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
 )
 
+# fmt
+http_archive(
+  name = "fmt",
+  url = "https://github.com/fmtlib/fmt/archive/9.1.0.tar.gz",
+  sha256 = "5dea48d1fcddc3ec571ce2058e13910a0d4a6bab4cc09a809d8b1dd1c88ae6f2",
+  strip_prefix = "fmt-9.1.0",
+  build_file = "//:thirdparties/fmt.BUILD",
+)
+
+# spdlog
+http_archive(
+  name = "spdlog",
+  urls = ["https://github.com/gabime/spdlog/archive/refs/tags/v1.11.0.tar.gz"],
+  strip_prefix = "spdlog-1.11.0",
+  sha256 = "ca5cae8d6cac15dae0ec63b21d6ad3530070650f68076f3a4a862ca293a858bb",
+  build_file = "//:thirdparties/spdlog.BUILD",
+)
+
 # Bazel platform rules.
 http_archive(
     name = "platforms",
@@ -248,14 +266,14 @@ new_local_repository(
 http_archive(
     name = "hedron_compile_commands",
 
-    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here. 
+    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here.
     # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README).
     urls = [
         "https://curve-build.nos-eastchina1.126.net/bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
         "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
     ],
     strip_prefix = "bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f",
-    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..." 
+    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..."
 )
 load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
 hedron_compile_commands_setup()

curvefs/conf/client.conf

@@ -77,16 +77,8 @@ rpc.healthCheckIntervalSec=0
 
 #### fuseClient
 # TODO(xuchaojie): add unit
-fuseClient.attrTimeOut=1.0
-fuseClient.entryTimeOut=1.0
 fuseClient.listDentryLimit=65536
-fuseClient.flushPeriodSec=5
-fuseClient.maxNameLength=255
-fuseClient.iCacheLruSize=65536
-fuseClient.dCacheLruSize=1000000
-fuseClient.enableICacheMetrics=true
-fuseClient.enableDCacheMetrics=true
-fuseClient.cto=true
+fuseClient.downloadMaxRetryTimes=3
 
 ### kvcache opt
 fuseClient.supportKVcache=false
@@ -110,6 +102,33 @@ fuseClient.disableXattr=false
 fuseClient.maxDataSize=1024
 # default refresh data interval 30s
 fuseClient.refreshDataIntervalSec=30
+fuseClient.warmupThreadsNum=10
+
+#### filesystem metadata
+# {
+# fs.disableXattr:
+#   if you want to get curvefs specified xattr,
+#   you can mount another fs with |fs.disableXattr| is true
+#
+# fs.lookupCache.negativeTimeout:
+#   entry which not found will be cached if |timeout| > 0
+fs.cto=true
+fs.maxNameLength=255
+fs.disableXattr=true
+fs.accessLogging=true
+fs.kernelCache.attrTimeout=3600
+fs.kernelCache.dirAttrTimeout=3600
+fs.kernelCache.entryTimeout=3600
+fs.kernelCache.dirEntryTimeout=3600
+fs.lookupCache.negativeTimeout=0
+fs.lookupCache.lruSize=100000
+fs.dirCache.lruSize=5000000
+fs.openFile.lruSize=65536
+fs.attrWatcher.lruSize=5000000
+fs.rpc.listDentryLimit=65536
+fs.deferSync.delay=3
+fs.deferSync.deferDirMtime=false
+# }
 
 #### volume
 volume.bigFileSize=1048576

curvefs/docker/debian9/Dockerfile

@@ -1,11 +1,9 @@
 FROM opencurvedocker/curve-base:debian9
 ENV TZ=Asia/Shanghai
-RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
-COPY curvefs /curvefs
 COPY entrypoint.sh /
-COPY curvefs/tools/sbin/curvefs_tool /usr/bin
-COPY libmemcached.so /usr/lib/
-COPY libmemcached.so.11 /usr/lib/
-COPY libhashkit.so.2 /usr/lib
-RUN chmod a+x /entrypoint.sh
+COPY curvefs /curvefs
+COPY libmemcached.so libmemcached.so.11 libhashkit.so.2 /usr/lib/
+RUN mkdir -p /etc/curvefs /core /etc/curve && chmod a+x /entrypoint.sh \
+    && cp /curvefs/tools/sbin/curvefs_tool /usr/bin \
+    && cp curvefs/tools-v2/sbin/curve /usr/bin/
 ENTRYPOINT ["/entrypoint.sh"]

curvefs/proto/metaserver.proto

@@ -218,7 +218,7 @@ message Inode {
     map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 18; // TYPE_S3 only, first is chunk index
     optional uint32 dtime = 19;
     optional uint32 openmpcount = 20; // openmpcount mount points had the file open
-    map<string, string> xattr = 21;
+    map<string, bytes> xattr = 21;
     repeated uint64 parent = 22;
 }
 
@@ -292,7 +292,7 @@ message UpdateInodeRequest {
     map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 17;
     optional uint32 nlink = 18;
     // field 19 is left for compatibility
-    map<string, string> xattr = 20;
+    map<string, bytes> xattr = 20;
     repeated uint64 parent = 21;
     map<uint64, S3ChunkInfoList> s3ChunkInfoAdd = 22;
     optional VolumeExtentList volumeExtents = 23;
@@ -385,7 +385,7 @@ message InodeAttr {
     optional uint64 rdev = 16;
     optional uint32 dtime = 17;
     optional uint32 openmpcount = 18;
-    map<string, string> xattr = 19;
+    map<string, bytes> xattr = 19;
     repeated uint64 parent = 20;
 }
 
@@ -416,7 +416,7 @@ message BatchGetXAttrRequest {
 message XAttr {
     required uint64 inodeId = 1;
     required uint32 fsId = 2;
-    map<string, string> xAttrInfos = 3;
+    map<string, bytes> xAttrInfos = 3;
 }
 
 message BatchGetXAttrResponse {

curvefs/src/client/BUILD

@@ -43,6 +43,10 @@ cc_library(
             "s3/*.h",
             "volume/*.cpp",
             "volume/*.h",
+            "filesystem/*.cpp",
+            "filesystem/*.h",
+            "warmup/*.h",
+            "warmup/*.cpp",
         ],
         exclude = ["main.cpp"],
     ),
@@ -69,10 +73,13 @@ cc_library(
         "//curvefs/src/volume",
         "//curvefs/src/common:metric_utils",
         "//curvefs/src/common:dynamic_vlog",
+        "//curvefs/src/common:threading",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/meta:type_traits",
         "@com_google_absl//absl/types:optional",
         "@com_google_googletest//:gtest_prod",
+        "@spdlog//:spdlog",
     ],
 )

curvefs/src/client/async_request_closure.cpp

@@ -28,7 +28,7 @@
 #include <mutex>
 
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/inode_wrapper.h"
 
 namespace curvefs {
@@ -63,7 +63,7 @@ CURVEFS_ERROR UpdateVolumeExtentClosure::Wait() {
         cond_.wait(lk);
     }
 
-    return MetaStatusCodeToCurvefsErrCode(GetStatusCode());
+    return ToFSError(GetStatusCode());
 }
 
 void UpdateVolumeExtentClosure::Run() {

curvefs/src/client/async_request_closure.h

@@ -28,12 +28,15 @@
 
 #include <memory>
 
-#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 namespace curvefs {
 namespace client {
 
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
+using ::curvefs::client::filesystem::ToFSError;
+
 class InodeWrapper;
 
 namespace internal {

curvefs/src/client/client_operator.cpp

@@ -24,13 +24,14 @@
 
 #include "src/common/uuid.h"
 #include "curvefs/src/client/client_operator.h"
-
+#include "curvefs/src/client/filesystem/error.h"
 namespace curvefs {
 namespace client {
 
 using ::curve::common::UUIDGenerator;
 using ::curvefs::metaserver::DentryFlag;
 using ::curvefs::mds::topology::PartitionTxId;
+using ::curvefs::client::filesystem::ToFSError;
 
 #define LOG_ERROR(action, rc) \
     LOG(ERROR) << action << " failed, retCode = " << rc \
@@ -95,7 +96,7 @@ CURVEFS_ERROR RenameOperator::GetTxId(uint32_t fsId,
     if (rc != MetaStatusCode::OK) {
         LOG_ERROR("GetTxId", rc);
     }
-    return MetaStatusCodeToCurvefsErrCode(rc);
+    return ToFSError(rc);
 }
 
 CURVEFS_ERROR RenameOperator::GetLatestTxIdWithLock() {
@@ -204,7 +205,7 @@ CURVEFS_ERROR RenameOperator::PrepareRenameTx(
         LOG_ERROR("PrepareRenameTx", rc);
     }
 
-    return MetaStatusCodeToCurvefsErrCode(rc);
+    return ToFSError(rc);
 }
 
 CURVEFS_ERROR RenameOperator::PrepareTx() {
@@ -397,8 +398,6 @@ CURVEFS_ERROR RenameOperator::UpdateInodeParent() {
 }
 
 void RenameOperator::UpdateCache() {
-    dentryManager_->DeleteCache(parentId_, name_);
-    dentryManager_->InsertOrReplaceCache(newDentry_);
     SetTxId(srcPartitionId_, srcTxId_ + 1);
     SetTxId(dstPartitionId_, dstTxId_ + 1);
 }

curvefs/src/client/common/common.cpp

@@ -77,6 +77,7 @@ std::ostream &operator<<(std::ostream &os, MetaServerOpType optype) {
 }
 
 const char kCurveFsWarmupOpAdd[] = "add";
+const char kCurveFsWarmupOpQuery[] = "query";
 const char kCurveFsWarmupTypeList[] = "list";
 const char kCurveFsWarmupTypeSingle[] = "single";
 
@@ -85,6 +86,9 @@ WarmupOpType GetWarmupOpType(const std::string& op) {
     if (op == kCurveFsWarmupOpAdd) {
         ret = WarmupOpType::kWarmupOpAdd;
     }
+    if (op == kCurveFsWarmupOpQuery) {
+        ret = WarmupOpType::kWarmupOpQuery;
+    }
     return ret;
 }
 

curvefs/src/client/common/common.h

@@ -63,13 +63,15 @@ enum class MetaServerOpType {
 
 std::ostream &operator<<(std::ostream &os, MetaServerOpType optype);
 
-const uint32_t MAXXATTRLENGTH = 256;
+const uint32_t MAX_XATTR_NAME_LENGTH = 255;
+const uint32_t MAX_XATTR_VALUE_LENGTH = 64 * 1024;
 
 const char kCurveFsWarmupXAttr[] = "curvefs.warmup.op";
 
 enum class WarmupOpType {
     kWarmupOpUnknown = 0,
     kWarmupOpAdd = 1,
+    kWarmupOpQuery = 2,
 };
 
 WarmupOpType GetWarmupOpType(const std::string& op);

curvefs/src/client/common/config.cpp

@@ -50,6 +50,7 @@ DEFINE_bool(enableCto, true, "acheieve cto consistency");
 DEFINE_bool(useFakeS3, false,
             "Use fake s3 to inject more metadata for testing metaserver");
 DEFINE_bool(supportKVcache, false, "use kvcache to speed up sharing");
+DEFINE_bool(access_logging, true, "enable access log");
 
 void InitMdsOption(Configuration *conf, MdsOption *mdsOpt) {
     conf->GetValueFatalIfFail("mdsOpt.mdsMaxRetryMS", &mdsOpt->mdsMaxRetryMS);
@@ -243,6 +244,48 @@ void InitKVClientManagerOpt(Configuration *conf,
                               &config->getThreadPooln);
 }
 
+void InitFileSystemOption(Configuration* c, FileSystemOption* option) {
+    c->GetValueFatalIfFail("fs.cto", &option->cto);
+    c->GetValueFatalIfFail("fs.cto", &FLAGS_enableCto);
+    c->GetValueFatalIfFail("fs.disableXattr", &option->disableXattr);
+    c->GetValueFatalIfFail("fs.maxNameLength", &option->maxNameLength);
+    c->GetValueFatalIfFail("fs.accessLogging", &FLAGS_access_logging);
+    { // kernel cache option
+        auto o = &option->kernelCacheOption;
+        c->GetValueFatalIfFail("fs.kernelCache.attrTimeout", &o->attrTimeout);
+        c->GetValueFatalIfFail("fs.kernelCache.dirAttrTimeout", &o->dirAttrTimeout);
+        c->GetValueFatalIfFail("fs.kernelCache.entryTimeout", &o->entryTimeout);
+        c->GetValueFatalIfFail("fs.kernelCache.dirEntryTimeout", &o->dirEntryTimeout);
+    }
+    { // memory lookup cache option
+        auto o = &option->lookupCacheOption;
+        c->GetValueFatalIfFail("fs.lookupCache.lruSize", &o->lruSize);
+        c->GetValueFatalIfFail("fs.lookupCache.negativeTimeout",
+                               &o->negativeTimeout);
+    }
+    { // dir cache option
+        auto o = &option->dirCacheOption;
+        c->GetValueFatalIfFail("fs.dirCache.lruSize", &o->lruSize);
+    }
+    { // open file option
+        auto o = &option->openFileOption;
+        c->GetValueFatalIfFail("fs.openFile.lruSize", &o->lruSize);
+    }
+    { // attr watcher option
+        auto o = &option->attrWatcherOption;
+        c->GetValueFatalIfFail("fs.attrWatcher.lruSize", &o->lruSize);
+    }
+    { // rpc option
+        auto o = &option->rpcOption;
+        c->GetValueFatalIfFail("fs.rpc.listDentryLimit", &o->listDentryLimit);
+    }
+    { // defer sync option
+        auto o = &option->deferSyncOption;
+        c->GetValueFatalIfFail("fs.deferSync.delay", &o->delay);
+        c->GetValueFatalIfFail("fs.deferSync.deferDirMtime", &o->deferDirMtime);
+    }
+}
+
 void SetBrpcOpt(Configuration *conf) {
     curve::common::GflagsLoadValueFromConfIfCmdNotSet dummy;
     dummy.Load(conf, "defer_close_second", "rpc.defer.close.second",
@@ -263,47 +306,25 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
     InitLeaseOpt(conf, &clientOption->leaseOpt);
     InitRefreshDataOpt(conf, &clientOption->refreshDataOption);
     InitKVClientManagerOpt(conf, &clientOption->kvClientManagerOpt);
+    InitFileSystemOption(conf, &clientOption->fileSystemOption);
 
-    conf->GetValueFatalIfFail("fuseClient.attrTimeOut",
-                              &clientOption->attrTimeOut);
-    conf->GetValueFatalIfFail("fuseClient.entryTimeOut",
-                              &clientOption->entryTimeOut);
     conf->GetValueFatalIfFail("fuseClient.listDentryLimit",
                               &clientOption->listDentryLimit);
     conf->GetValueFatalIfFail("fuseClient.listDentryThreads",
                               &clientOption->listDentryThreads);
-    conf->GetValueFatalIfFail("fuseClient.flushPeriodSec",
-                              &clientOption->flushPeriodSec);
-    conf->GetValueFatalIfFail("fuseClient.maxNameLength",
-                              &clientOption->maxNameLength);
-    conf->GetValueFatalIfFail("fuseClient.iCacheLruSize",
-                              &clientOption->iCacheLruSize);
-    conf->GetValueFatalIfFail("fuseClient.dCacheLruSize",
-                              &clientOption->dCacheLruSize);
-    conf->GetValueFatalIfFail("fuseClient.enableICacheMetrics",
-                              &clientOption->enableICacheMetrics);
-    conf->GetValueFatalIfFail("fuseClient.enableDCacheMetrics",
-                              &clientOption->enableDCacheMetrics);
     conf->GetValueFatalIfFail("client.dummyServer.startPort",
                               &clientOption->dummyServerStartPort);
     conf->GetValueFatalIfFail("fuseClient.enableMultiMountPointRename",
                               &clientOption->enableMultiMountPointRename);
-    conf->GetValueFatalIfFail("fuseClient.disableXattr",
-                              &clientOption->disableXattr);
-    conf->GetValueFatalIfFail("fuseClient.cto", &FLAGS_enableCto);
-
-
+    conf->GetValueFatalIfFail("fuseClient.downloadMaxRetryTimes",
+                              &clientOption->downloadMaxRetryTimes);
+    conf->GetValueFatalIfFail("fuseClient.warmupThreadsNum",
+                              &clientOption->warmupThreadsNum);
     LOG_IF(WARNING, conf->GetBoolValue("fuseClient.enableSplice",
                                        &clientOption->enableFuseSplice))
         << "Not found `fuseClient.enableSplice` in conf, use default value `"
         << std::boolalpha << clientOption->enableFuseSplice << '`';
 
-    // if enableCto, attr and entry cache must invalid
-    if (FLAGS_enableCto) {
-        clientOption->attrTimeOut = 0;
-        clientOption->entryTimeOut = 0;
-    }
-
     SetBrpcOpt(conf);
 }
 

curvefs/src/client/common/config.h

@@ -175,6 +175,58 @@ struct RefreshDataOption {
     uint64_t maxDataSize = 1024;
     uint32_t refreshDataIntervalSec = 30;
 };
+
+// { filesystem option
+struct KernelCacheOption {
+    uint32_t entryTimeout;
+    uint32_t dirEntryTimeout;
+    uint32_t attrTimeout;
+    uint32_t dirAttrTimeout;
+};
+
+struct LookupCacheOption {
+    uint64_t lruSize;
+    uint32_t negativeTimeout;
+};
+
+struct DirCacheOption {
+    uint64_t lruSize;
+    uint32_t timeout;
+};
+
+struct AttrWatcherOption {
+    uint64_t lruSize;
+};
+
+struct OpenFileOption {
+    uint64_t lruSize;
+    uint32_t deferSyncSecond;
+};
+
+struct RPCOption {
+    uint32_t listDentryLimit;
+};
+
+struct DeferSyncOption {
+    uint32_t delay;
+    bool deferDirMtime;
+};
+
+struct FileSystemOption {
+    bool cto;
+    bool disableXattr;
+    uint32_t maxNameLength;
+    uint32_t blockSize = 0x10000u;
+    KernelCacheOption kernelCacheOption;
+    LookupCacheOption lookupCacheOption;
+    DirCacheOption dirCacheOption;
+    OpenFileOption openFileOption;
+    AttrWatcherOption attrWatcherOption;
+    RPCOption rpcOption;
+    DeferSyncOption deferSyncOption;
+};
+// }
+
 struct FuseClientOption {
     MdsOption mdsOpt;
     MetaCacheOpt metaCacheOpt;
@@ -188,21 +240,15 @@ struct FuseClientOption {
     LeaseOpt leaseOpt;
     RefreshDataOption refreshDataOption;
     KVClientManagerOpt kvClientManagerOpt;
+    FileSystemOption fileSystemOption;
 
-    double attrTimeOut;
-    double entryTimeOut;
     uint32_t listDentryLimit;
     uint32_t listDentryThreads;
-    uint32_t flushPeriodSec;
-    uint32_t maxNameLength;
-    uint64_t iCacheLruSize;
-    uint64_t dCacheLruSize;
-    bool enableICacheMetrics;
-    bool enableDCacheMetrics;
     uint32_t dummyServerStartPort;
     bool enableMultiMountPointRename = false;
     bool enableFuseSplice = false;
-    bool disableXattr = false;
+    uint32_t downloadMaxRetryTimes;
+    uint32_t warmupThreadsNum;
 };
 
 void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption);

curvefs/src/client/curve_fuse_op.h

@@ -37,7 +37,7 @@
 extern "C" {
 #endif
 
-int InitGlog(const char *confPath, const char *argv0);
+int InitLog(const char *confPath, const char *argv0);
 
 int InitFuseClient(const char *confPath, const char *fsName,
         const char* fsType, const char* mdsAddr);

curvefs/src/client/dentry_cache_manager.cpp

@@ -44,31 +44,13 @@ namespace client {
 
 using curve::common::WriteLockGuard;
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
-
-void DentryCacheManagerImpl::InsertOrReplaceCache(const Dentry &dentry) {
-    std::string key = GetDentryCacheKey(dentry.parentinodeid(), dentry.name());
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        NameLockGuard lock(nameLock_, key);
-        dCache_->Put(key, dentry);
-    }
-}
-
-void DentryCacheManagerImpl::DeleteCache(uint64_t parentId,
-                                         const std::string &name) {
-    std::string key = GetDentryCacheKey(parentId, name);
-    NameLockGuard lock(nameLock_, key);
-    dCache_->Remove(key);
-}
+using ::curvefs::client::filesystem::ToFSError;
 
 CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
                                                 const std::string &name,
                                                 Dentry *out) {
     std::string key = GetDentryCacheKey(parent, name);
     NameLockGuard lock(nameLock_, key);
-    bool ok = dCache_->Get(key, out);
-    if (ok) {
-        return CURVEFS_ERROR::OK;
-    }
 
     MetaStatusCode ret = metaClient_->GetDentry(fsId_, parent, name, out);
     if (ret != MetaStatusCode::OK) {
@@ -76,11 +58,7 @@ CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
             << "metaClient_ GetDentry failed, MetaStatusCode = " << ret
             << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
             << ", parent = " << parent << ", name = " << name;
-        return MetaStatusCodeToCurvefsErrCode(ret);
-    }
-
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        dCache_->Put(key, *out);
+        return ToFSError(ret);
     }
     return CURVEFS_ERROR::OK;
 }
@@ -95,12 +73,9 @@ CURVEFS_ERROR DentryCacheManagerImpl::CreateDentry(const Dentry &dentry) {
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", parent = " << dentry.parentinodeid()
                    << ", name = " << dentry.name();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
 
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        dCache_->Put(key, dentry);
-    }
     return CURVEFS_ERROR::OK;
 }
 
@@ -109,14 +84,13 @@ CURVEFS_ERROR DentryCacheManagerImpl::DeleteDentry(uint64_t parent,
                                                    FsFileType type) {
     std::string key = GetDentryCacheKey(parent, name);
     NameLockGuard lock(nameLock_, key);
-    dCache_->Remove(key);
 
     MetaStatusCode ret = metaClient_->DeleteDentry(fsId_, parent, name, type);
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", parent = " << parent << ", name = " << name;
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
     return CURVEFS_ERROR::OK;
 }
@@ -150,7 +124,7 @@ CURVEFS_ERROR DentryCacheManagerImpl::ListDentry(uint64_t parent,
                        << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                        << ", parent = " << parent << ", last = " << last
                        << ", count = " << limit << ", onlyDir = " << onlyDir;
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
 
         if (!onlyDir) {

curvefs/src/client/dentry_cache_manager.h

@@ -33,20 +33,18 @@
 #include <utility>
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
-#include "curvefs/src/client/error_code.h"
 #include "src/common/concurrent/concurrent.h"
-#include "src/common/lru_cache.h"
 #include "src/common/concurrent/name_lock.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 using ::curvefs::metaserver::Dentry;
-using ::curve::common::LRUCache;
-using ::curve::common::CacheMetrics;
 
 namespace curvefs {
 namespace client {
 
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
 
 static const char* kDentryKeyDelimiter = ":";
 
@@ -59,12 +57,6 @@ class DentryCacheManager {
         fsId_ = fsId;
     }
 
-    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics) = 0;
-
-    virtual void InsertOrReplaceCache(const Dentry& dentry) = 0;
-
-    virtual void DeleteCache(uint64_t parentId, const std::string& name) = 0;
-
     virtual CURVEFS_ERROR GetDentry(uint64_t parent,
         const std::string &name, Dentry *out) = 0;
 
@@ -85,29 +77,11 @@ class DentryCacheManager {
 class DentryCacheManagerImpl : public DentryCacheManager {
  public:
     DentryCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()),
-        dCache_(nullptr) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
 
     explicit DentryCacheManagerImpl(
         const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient),
-        dCache_(nullptr) {}
-
-    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics) override {
-        if (enableCacheMetrics) {
-            dCache_ = std::make_shared<
-                LRUCache<std::string, Dentry>>(cacheSize,
-                    std::make_shared<CacheMetrics>("dcache"));
-        } else {
-            dCache_ = std::make_shared<
-                LRUCache<std::string, Dentry>>(cacheSize);
-        }
-        return CURVEFS_ERROR::OK;
-    }
-
-    void InsertOrReplaceCache(const Dentry& dentry) override;
-
-    void DeleteCache(uint64_t parentId, const std::string& name) override;
+      : metaClient_(metaClient) {}
 
     CURVEFS_ERROR GetDentry(uint64_t parent,
         const std::string &name, Dentry *out) override;
@@ -128,8 +102,6 @@ class DentryCacheManagerImpl : public DentryCacheManager {
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
-    // key is parentId + name
-    std::shared_ptr<LRUCache<std::string, Dentry>> dCache_;
     curve::common::GenericNameLock<Mutex> nameLock_;
 };
 

curvefs/src/client/error_code.cpp

@@ -1,98 +0,0 @@
-/*
- *  Copyright (c) 2021 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-
-/*
- * Project: curve
- * Created Date: Thur May 27 2021
- * Author: xuchaojie
- */
-
-#include "curvefs/src/client/error_code.h"
-
-#include <map>
-#include <string>
-
-namespace curvefs {
-namespace client {
-
-using ::curvefs::metaserver::MetaStatusCode;
-
-static const std::map<CURVEFS_ERROR, std::string> err2Msg = {
-    {CURVEFS_ERROR::OK, "OK"},
-    {CURVEFS_ERROR::INTERNAL, "internal error"},
-    {CURVEFS_ERROR::UNKNOWN, "unknown"},
-    {CURVEFS_ERROR::EXISTS, "inode or dentry already exist"},
-    {CURVEFS_ERROR::NOTEXIST, "inode or dentry not exist"},
-    {CURVEFS_ERROR::NO_SPACE, "no space to alloc"},
-    {CURVEFS_ERROR::BAD_FD, "bad fd"},
-    {CURVEFS_ERROR::INVALIDPARAM, "invalid param"},
-    {CURVEFS_ERROR::NOPERMISSION, "no permission"},
-    {CURVEFS_ERROR::NOTEMPTY, "dir not empty"},
-    {CURVEFS_ERROR::NOFLUSH, "no flush"},
-    {CURVEFS_ERROR::NOTSUPPORT, "not support"},
-    {CURVEFS_ERROR::NAMETOOLONG, "name too long"},
-    {CURVEFS_ERROR::MOUNT_POINT_EXIST, "mount point already exist"},
-    {CURVEFS_ERROR::MOUNT_FAILED, "mount failed"},
-};
-
-std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
-    os << static_cast<int>(code) << "[" << [code]() {
-        auto it = err2Msg.find(code);
-        if (it != err2Msg.end()) {
-            return it->second;
-        }
-
-        return std::string{"Unknown"};
-    }() << "]";
-
-    return os;
-}
-
-CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
-    MetaStatusCode code) {
-    CURVEFS_ERROR ret = CURVEFS_ERROR::UNKNOWN;
-    switch (code) {
-        case MetaStatusCode::OK:
-            ret = CURVEFS_ERROR::OK;
-            break;
-
-        case MetaStatusCode::NOT_FOUND:
-            ret = CURVEFS_ERROR::NOTEXIST;
-            break;
-
-        case MetaStatusCode::PARAM_ERROR:
-            ret = CURVEFS_ERROR::INVALIDPARAM;
-            break;
-
-        case MetaStatusCode::INODE_EXIST:
-        case MetaStatusCode::DENTRY_EXIST:
-            ret = CURVEFS_ERROR::EXISTS;
-            break;
-
-        case MetaStatusCode::SYM_LINK_EMPTY:
-        case MetaStatusCode::RPC_ERROR:
-            ret = CURVEFS_ERROR::INTERNAL;
-            break;
-
-        default:
-            ret = CURVEFS_ERROR::UNKNOWN;
-    }
-    return ret;
-}
-
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/README.md

@@ -0,0 +1,78 @@
+CurveFS Client Metadata Cache Design
+===
+
+```
++----------------+
++  Kernel Cache  |
++----------------+
+    |       ^
+(1) |       | (4)
+    v       |
++----------------+
++  CurveFS Fuse  |
++----------------+
+    |       ^
+(2) |       | (3)
+    v       |
++----------------+
++   MetaServer   |
++----------------+
+```
+
+(1) kernel cache timeout
+---
+
+* requests are always sent to fuse layer, except `lookup` and `getattr` request which affected by caching.
+* if entry or attribute cache timeout, vfs layer will launch request to fuse layer.
+
+(2) revalidate cache
+---
+
+* for `open` and `opendir` request, fuse layer should revalidate cache by comparing mtime, if modified, fuse layer should:
+    * `open`: return **ESTALE** to trigger vfs layer to invoke the `open` again with ignoring cache.
+    * `opendir`: drop all directory cache.
+* others, proxy request to metaserver directly.
+
+
+(3) retrive fresh metadata
+---
+
+* for `readdir` request, fuse layer should cache direcoty entries and their attributes.
+* others, no caching.
+
+(4) reply with timeout
+---
+
+* reply entry or attribute to vfs layer with corresponding cache timeout.
+* four type timeouts provided: `entryTimeout`, `dirEntryTimeout`, `attrTimeout`, `dirAtttTimeout`.
+* fuse layer should remeber the `mtime` of attribute while reply it to vfs layer.
+
+Cache Layer Level
+===
+
+```
++-------------------------------------------------------+
+|                       Kernel Cache                    |  (L1 Cache)
++-------------------------------------------------------+
+                            ^  (update length)
+                            |
++--------------------------------------------------------+
+|                       Open File Cache                  |  (L2 Cache)
+|                   (only length and chunk)              |
++--------------------------------------------------------+
+         ^                            ^
+         | (reply entry/attr)         |  (reply entry/attr)
++-----------------+                   |
+| Directory Cache |  (L3 Cache)       |
++-----------------+                   |
+         ^                            |
+         |                            |
++-----------------+ +------------------------------------+
+|     readdir     | | lookup、getattr、setattr、create... |
++-----------------+ +------------------------------------+
+         ^                            ^
+         |                            |
++--------------------------------------------------------+
+|                        MetaServer                      |
++--------------------------------------------------------+
+```

curvefs/src/client/filesystem/access_log.h

@@ -0,0 +1,83 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-17
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <butil/time.h>
+#include <spdlog/spdlog.h>
+#include <spdlog/sinks/basic_file_sink.h>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/common/config.h"
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
+
+namespace curvefs {
+namespace client {
+namespace common {
+
+DECLARE_bool(access_logging);
+
+}
+namespace filesystem {
+
+using ::absl::StrFormat;
+using ::curvefs::client::common::FLAGS_access_logging;
+using MessageHandler = std::function<std::string()>;
+
+static std::shared_ptr<spdlog::logger> Logger;
+
+bool InitAccessLog(const std::string& prefix) {
+    Logger = spdlog::basic_logger_mt("fuse_access", prefix + "/access.log");
+    spdlog::flush_every(std::chrono::seconds(1));
+    return true;
+}
+
+struct AccessLogGuard {
+    AccessLogGuard(MessageHandler handler)
+        : enable(FLAGS_access_logging),
+          handler(handler) {
+        if (!enable) {
+            return;
+        }
+
+        timer.start();
+    }
+
+    ~AccessLogGuard() {
+        if (!enable) {
+            return;
+        }
+
+        timer.stop();
+        Logger->info("{0} <{1:.6f}>", handler(), timer.u_elapsed() / 1e6);
+    }
+
+    bool enable;
+    MessageHandler handler;
+    butil::Timer timer;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_

curvefs/src/client/filesystem/attr_watcher.cpp

@@ -0,0 +1,91 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/src/client/filesystem/utils.h"
+#include "curvefs/src/client/filesystem/attr_watcher.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+AttrWatcher::AttrWatcher(AttrWatcherOption option,
+                         std::shared_ptr<OpenFiles> openFiles,
+                         std::shared_ptr<DirCache> dirCache)
+    : modifiedAt_(std::make_shared<LRUType>(option.lruSize)),
+      openFiles_(openFiles),
+      dirCache_(dirCache) {}
+
+void AttrWatcher::RemeberMtime(const InodeAttr& attr) {
+    WriteLockGuard lk(rwlock_);
+    modifiedAt_->Put(attr.inodeid(), AttrMtime(attr));
+}
+
+bool AttrWatcher::GetMtime(Ino ino, TimeSpec* time) {
+    ReadLockGuard lk(rwlock_);
+    return modifiedAt_->Get(ino, time);
+}
+
+void AttrWatcher::UpdateDirEntryAttr(Ino ino, const InodeAttr& attr) {
+    std::shared_ptr<DirEntryList> entries;
+    for (const auto parent : attr.parent()) {
+        bool yes = dirCache_->Get(parent, &entries);
+        if (!yes) {
+            continue;
+        }
+
+        entries->UpdateAttr(ino, attr);
+
+        VLOG(1) << "Write back attribute to dir entry cache: ino = " << ino
+                << ", attr = " << attr.ShortDebugString();
+    }
+}
+
+AttrWatcherGuard::AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
+                                   InodeAttr* attr,
+                                   bool writeBack)
+    : watcher(watcher), attr(attr), writeBack(writeBack) {
+    InodeAttr open;
+    Ino ino = attr->inodeid();
+    bool yes = watcher->openFiles_->GetFileAttr(ino, &open);
+    if (!yes) {
+        return;
+    }
+
+    attr->set_length(open.length());
+    attr->set_mtime(open.mtime());
+    attr->set_mtime_ns(open.mtime_ns());
+    if (AttrCtime(open) > AttrCtime(*attr)) {
+        attr->set_ctime(open.ctime());
+        attr->set_ctime_ns(open.ctime_ns());
+    }
+}
+
+AttrWatcherGuard::~AttrWatcherGuard() {
+    watcher->RemeberMtime(*attr);
+    if (writeBack) {
+        watcher->UpdateDirEntryAttr(attr->inodeid(), *attr);
+    }
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/attr_watcher.h

@@ -0,0 +1,92 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/openfile.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::AttrWatcherOption;
+
+class AttrWatcher {
+ public:
+    using LRUType = LRUCache<Ino, struct TimeSpec>;
+
+ public:
+    AttrWatcher(AttrWatcherOption option,
+                std::shared_ptr<OpenFiles> openFiles,
+                std::shared_ptr<DirCache> dirCache);
+
+    void RemeberMtime(const InodeAttr& attr);
+
+    bool GetMtime(Ino ino, TimeSpec* time);
+
+    void UpdateDirEntryAttr(Ino ino, const InodeAttr& attr);
+
+ private:
+    friend class AttrWatcherGuard;
+
+ private:
+    RWLock rwlock_;
+    std::shared_ptr<LRUType> modifiedAt_;
+    std::shared_ptr<OpenFiles> openFiles_;
+    std::shared_ptr<DirCache> dirCache_;
+};
+
+/*
+ * each attribute reply to kernel, the watcher will:
+ *    before reply:
+ *      1) set attibute length if the corresponding file is opened
+ *    after reply:
+ *     1) remeber attribute modified time
+ *     2) write back attribute to dir entry cache if |writeBack| it true
+ */
+struct AttrWatcherGuard {
+ public:
+    AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
+                     InodeAttr* attr,
+                     bool writeBack);
+
+    ~AttrWatcherGuard();
+
+ private:
+    std::shared_ptr<AttrWatcher> watcher;
+    InodeAttr* attr;
+    bool writeBack;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_

curvefs/src/client/filesystem/defer_sync.cpp

@@ -0,0 +1,83 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/src/client/filesystem/defer_sync.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+DeferSync::DeferSync(DeferSyncOption option)
+    : option_(option),
+      mutex_(),
+      running_(false),
+      thread_(),
+      sleeper_(),
+      inodes_() {
+}
+
+void DeferSync::Start() {
+    if (!running_.exchange(true)) {
+        thread_ = std::thread(&DeferSync::SyncTask, this);
+        LOG(INFO) << "Defer sync thread start success";
+    }
+}
+
+void DeferSync::Stop() {
+    if (running_.exchange(false)) {
+        LOG(INFO) << "Stop defer sync thread...";
+        sleeper_.interrupt();
+        thread_.join();
+        LOG(INFO) << "Defer sync thread stopped";
+    }
+}
+
+void DeferSync::SyncTask() {
+    std::vector<std::shared_ptr<InodeWrapper>> inodes;
+    for ( ;; ) {
+        bool running = sleeper_.wait_for(std::chrono::seconds(option_.delay));
+
+        {
+            LockGuard lk(mutex_);
+            inodes.swap(inodes_);
+        }
+        for (const auto& inode : inodes) {
+            UniqueLock lck(inode->GetUniqueLock());
+            inode->Async(nullptr, true);
+        }
+        inodes.clear();
+
+        if (!running) {
+            break;
+        }
+    }
+}
+
+void DeferSync::Push(const std::shared_ptr<InodeWrapper>& inode) {
+    LockGuard lk(mutex_);
+    inodes_.emplace_back(inode);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/defer_sync.h

@@ -0,0 +1,69 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
+
+#include <atomic>
+#include <thread>
+
+#include "src/common/interruptible_sleeper.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::DeferSyncOption;
+
+using ::curve::common::Mutex;
+using ::curve::common::LockGuard;
+using ::curve::common::InterruptibleSleeper;
+
+class DeferSync {
+ public:
+    explicit DeferSync(DeferSyncOption option);
+
+    void Start();
+
+    void Stop();
+
+    void Push(const std::shared_ptr<InodeWrapper>& inode);
+
+ private:
+    void SyncTask();
+
+ private:
+    DeferSyncOption option_;
+    Mutex mutex_;
+    std::atomic<bool> running_;
+    std::thread thread_;
+    InterruptibleSleeper sleeper_;
+    std::vector<std::shared_ptr<InodeWrapper>> inodes_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_

curvefs/src/client/filesystem/dir_cache.cpp

@@ -0,0 +1,187 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/src/client/filesystem/dir_cache.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+DirEntryList::DirEntryList()
+    : rwlock_(),
+      mtime_(),
+      entries_(),
+      attrs_() {}
+
+size_t DirEntryList::Size() {
+    ReadLockGuard lk(rwlock_);
+    return entries_.size();
+}
+
+void DirEntryList::Add(const DirEntry& dirEntry) {
+    WriteLockGuard lk(rwlock_);
+    entries_.push_back(std::move(dirEntry));
+    attrs_[dirEntry.ino] = &entries_.back();
+}
+
+bool DirEntryList::Get(Ino ino, DirEntry* dirEntry) {
+    ReadLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+    *dirEntry = *iter->second;
+    return true;
+}
+
+bool DirEntryList::UpdateAttr(Ino ino, const InodeAttr& attr) {
+    WriteLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+
+    DirEntry* dirEntry = iter->second;
+    dirEntry->attr = std::move(attr);
+    return true;
+}
+
+bool DirEntryList::UpdateLength(Ino ino, const InodeAttr& open) {
+    WriteLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+
+    InodeAttr* attr = &iter->second->attr;
+    attr->set_length(open.length());
+    attr->set_mtime(open.mtime());
+    attr->set_mtime_ns(open.mtime_ns());
+    if (AttrCtime(open) > AttrCtime(*attr)) {
+        attr->set_ctime(open.ctime());
+        attr->set_ctime_ns(open.ctime_ns());
+    }
+    return true;
+}
+
+void DirEntryList::Iterate(IterateHandler handler) {
+    ReadLockGuard lk(rwlock_);
+    for (auto iter = entries_.begin(); iter != entries_.end(); iter++) {
+        handler(&(*iter));
+    }
+}
+
+void DirEntryList::Clear() {
+    WriteLockGuard lk(rwlock_);
+    entries_.clear();
+    attrs_.clear();
+}
+
+void DirEntryList::SetMtime(TimeSpec mtime) {
+    WriteLockGuard lk(rwlock_);
+    mtime_ = mtime;
+}
+
+TimeSpec DirEntryList::GetMtime() {
+    ReadLockGuard lk(rwlock_);
+    return mtime_;
+}
+
+DirCache::DirCache(DirCacheOption option)
+    : rwlock_(),
+      nentries_(0),
+      option_(option) {
+    lru_ = std::make_shared<LRUType>(0);  // control size by ourself
+    mq_ = std::make_shared<MessageQueueType>("dircache", 10000);
+    mq_->Subscribe([&](const std::shared_ptr<DirEntryList>& entries){
+        entries->Clear();
+    });
+
+    LOG(INFO) << "Using directory cache, capacity = " << option_.lruSize;
+}
+
+void DirCache::Start() {
+    mq_->Start();
+}
+
+void DirCache::Stop() {
+    WriteLockGuard lk(rwlock_);
+    Evit(option_.lruSize);
+    mq_->Stop();
+}
+
+void DirCache::Delete(Ino parent,
+                      std::shared_ptr<DirEntryList> entries,
+                      bool evit) {
+    nentries_ -= entries->Size();
+    mq_->Publish(entries);
+    lru_->Remove(parent);
+
+    LOG(INFO) << "Delete directory cache (evit=" << evit << "): "
+              << "parent = " << parent
+              << ", mtime = " << entries->GetMtime()
+              << ", size = " << entries->Size()
+              << ", nentries = " << nentries_;
+}
+
+void DirCache::Evit(size_t size) {
+    Ino parent;
+    std::shared_ptr<DirEntryList> entries;
+    while (nentries_ + size >= option_.lruSize) {
+        bool yes = lru_->GetLast(&parent, &entries);
+        if (!yes) {
+            break;
+        }
+        Delete(parent, entries, true);
+    }
+}
+
+void DirCache::Put(Ino parent, std::shared_ptr<DirEntryList> entries) {
+    WriteLockGuard lk(rwlock_);
+    Evit(entries->Size());
+    lru_->Put(parent, entries);
+    nentries_ += entries->Size();
+
+    LOG(INFO) << "Insert directory cache: parent = " << parent
+              << ", mtime = " << entries->GetMtime()
+              << ", size = " << entries->Size()
+              << ", nentries = " << nentries_;
+}
+
+bool DirCache::Get(Ino parent, std::shared_ptr<DirEntryList>* entries) {
+    ReadLockGuard lk(rwlock_);
+    return lru_->Get(parent, entries);
+}
+
+void DirCache::Drop(Ino parent) {
+    WriteLockGuard lk(rwlock_);
+    std::shared_ptr<DirEntryList> entries;
+    bool yes = lru_->Get(parent, &entries);
+    if (yes) {
+        Delete(parent, entries, false);
+    }
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/dir_cache.h

@@ -0,0 +1,113 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
+
+#include <map>
+#include <list>
+
+#include "src/common/lru_cache.h"
+#include "src/common/concurrent/concurrent.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/message_queue.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::DirCacheOption;
+
+class DirEntryList {
+ public:
+    using IterateHandler = std::function<void(DirEntry* dirEntry)>;
+
+ public:
+    DirEntryList();
+
+    size_t Size();
+
+    void Add(const DirEntry& dirEntry);
+
+    void Iterate(IterateHandler handler);
+
+    bool Get(Ino ino, DirEntry* dirEntry);
+
+    bool UpdateAttr(Ino ino, const InodeAttr& attr);
+
+    bool UpdateLength(Ino ino, const InodeAttr& open);
+
+    void Clear();
+
+    void SetMtime(TimeSpec mtime);
+
+    TimeSpec GetMtime();
+
+ private:
+    RWLock rwlock_;
+    TimeSpec mtime_;
+    std::list<DirEntry> entries_;
+    std::map<Ino, DirEntry*> attrs_;
+};
+
+class DirCache {
+ public:
+    using LRUType = LRUCache<Ino, std::shared_ptr<DirEntryList>>;
+    using MessageType = std::shared_ptr<DirEntryList>;
+    using MessageQueueType = MessageQueue<MessageType>;
+
+ public:
+    explicit DirCache(DirCacheOption option);
+
+    void Start();
+
+    void Stop();
+
+    void Put(Ino parent, std::shared_ptr<DirEntryList> entries);
+
+    bool Get(Ino parent, std::shared_ptr<DirEntryList>* entries);
+
+    void Drop(Ino parent);
+
+ private:
+    void Delete(Ino parent, std::shared_ptr<DirEntryList> entries, bool evit);
+
+    void Evit(size_t size);
+
+ private:
+    RWLock rwlock_;
+    size_t nentries_;
+    DirCacheOption option_;
+    std::shared_ptr<LRUType> lru_;
+    std::shared_ptr<MessageQueueType> mq_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_

curvefs/src/client/filesystem/error.cpp

@@ -0,0 +1,105 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+#include <ostream>
+
+#include "curvefs/src/client/filesystem/error.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+static const std::map<CURVEFS_ERROR, std::pair<int, std::string>> errors = {
+    { CURVEFS_ERROR::OK, { 0, "OK"} },
+    { CURVEFS_ERROR::INTERNAL,  { EIO, "internal error" } },
+    { CURVEFS_ERROR::UNKNOWN,  { -1, "unknown" } },
+    { CURVEFS_ERROR::EXISTS,  { EEXIST, "inode or dentry already exist" } },
+    { CURVEFS_ERROR::NOTEXIST, { ENOENT, "inode or dentry not exist" } },
+    { CURVEFS_ERROR::NO_SPACE,  { ENOSPC, "no space to alloc" } },
+    { CURVEFS_ERROR::BAD_FD,  { EBADF, "bad file number" } },
+    { CURVEFS_ERROR::INVALIDPARAM , { EINVAL , "invalid argument" } },
+    { CURVEFS_ERROR::NOPERMISSION,  { EACCES, "permission denied" } },
+    { CURVEFS_ERROR::NOTEMPTY, { ENOTEMPTY, "directory not empty" } },
+    { CURVEFS_ERROR::NOFLUSH, { -1, "no flush" } },
+    { CURVEFS_ERROR::NOTSUPPORT, { EOPNOTSUPP, "operation not supported" } },
+    { CURVEFS_ERROR::NAMETOOLONG, { ENAMETOOLONG, "file name too long" } },
+    { CURVEFS_ERROR::MOUNT_POINT_EXIST, { -1, "mount point already exist" } },
+    { CURVEFS_ERROR::MOUNT_FAILED, { -1, "mount failed" } },
+    { CURVEFS_ERROR::OUT_OF_RANGE, { ERANGE, "out of range" } },
+    { CURVEFS_ERROR::NODATA, { ENODATA, "no data available" } },
+    { CURVEFS_ERROR::IO_ERROR, { EIO, "I/O error" } },
+    { CURVEFS_ERROR::STALE, { ESTALE, "stale file handler" } },
+    { CURVEFS_ERROR::NOSYS, { ENOSYS, "invalid system call" } },
+};
+
+std::string StrErr(CURVEFS_ERROR code) {
+    auto it = errors.find(code);
+    if (it != errors.end()) {
+        return it->second.second;
+    }
+    return "unknown";
+}
+
+int SysErr(CURVEFS_ERROR code) {
+    int syscode = -1;
+    auto it = errors.find(code);
+    if (it != errors.end()) {
+        syscode = it->second.first;
+    }
+    return (syscode == -1) ? EIO : syscode;
+}
+
+std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
+    os << static_cast<int>(code) << "[" << [code]() {
+        auto it = errors.find(code);
+        if (it != errors.end()) {
+            return it->second.second;
+        }
+
+        return std::string{"Unknown"};
+    }() << "]";
+
+    return os;
+}
+
+CURVEFS_ERROR ToFSError(MetaStatusCode code) {
+    static std::map<MetaStatusCode, CURVEFS_ERROR> errs = {
+        { MetaStatusCode::OK, CURVEFS_ERROR::OK },
+        { MetaStatusCode::NOT_FOUND, CURVEFS_ERROR::NOTEXIST },
+        { MetaStatusCode::PARAM_ERROR, CURVEFS_ERROR::INVALIDPARAM },
+        { MetaStatusCode::INODE_EXIST, CURVEFS_ERROR::EXISTS },
+        { MetaStatusCode::DENTRY_EXIST, CURVEFS_ERROR::EXISTS },
+        { MetaStatusCode::SYM_LINK_EMPTY, CURVEFS_ERROR::INTERNAL },
+        { MetaStatusCode::RPC_ERROR, CURVEFS_ERROR::INTERNAL },
+    };
+
+    auto it = errs.find(code);
+    if (it != errs.end()) {
+        return it->second;
+    }
+    return CURVEFS_ERROR::UNKNOWN;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/error.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2021 NetEase Inc.
+ *  Copyright (c) 2023 NetEase Inc.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -14,22 +14,26 @@
  *  limitations under the License.
  */
 
-
 /*
- * Project: curve
- * Created Date: Thur May 27 2021
- * Author: xuchaojie
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
  */
 
-#ifndef CURVEFS_SRC_CLIENT_ERROR_CODE_H_
-#define CURVEFS_SRC_CLIENT_ERROR_CODE_H_
+#include <map>
+#include <iostream>
 
 #include "curvefs/proto/metaserver.pb.h"
 
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
+
 namespace curvefs {
 namespace client {
+namespace filesystem {
+
+using ::curvefs::metaserver::MetaStatusCode;
 
-// notice : the error code should be negative.
 enum class CURVEFS_ERROR {
     OK = 0,
     INTERNAL = -1,
@@ -49,15 +53,20 @@ enum class CURVEFS_ERROR {
     OUT_OF_RANGE = -15,
     NODATA = -16,
     IO_ERROR = -17,
+    STALE = -18,
+    NOSYS = -19,
 };
 
+std::string StrErr(CURVEFS_ERROR code);
+
+int SysErr(CURVEFS_ERROR code);
 
 std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code);
 
-CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
-    ::curvefs::metaserver::MetaStatusCode code);
+CURVEFS_ERROR ToFSError(MetaStatusCode code);
 
+}  // namespace filesystem
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_ERROR_CODE_H_
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_

curvefs/src/client/filesystem/filesystem.cpp

@@ -0,0 +1,339 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-08
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+
+#include "curvefs/src/client/filesystem/filesystem.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+FileSystem::FileSystem(FileSystemOption option, ExternalMember member)
+    : option_(option), member(member) {
+    deferSync_ = std::make_shared<DeferSync>(option.deferSyncOption);
+    negative_ = std::make_shared<LookupCache>(option.lookupCacheOption);
+    dirCache_ = std::make_shared<DirCache>(option.dirCacheOption);
+    openFiles_ = std::make_shared<OpenFiles>(option_.openFileOption,
+                                             deferSync_, dirCache_);
+    attrWatcher_ = std::make_shared<AttrWatcher>(option_.attrWatcherOption,
+                                                 openFiles_, dirCache_);
+    handlerManager_ = std::make_shared<HandlerManager>();
+    rpc_ = std::make_shared<RPCClient>(option.rpcOption, member);
+}
+
+void FileSystem::Run() {
+    deferSync_->Start();
+    dirCache_->Start();
+}
+
+void FileSystem::Destory() {
+    openFiles_->CloseAll();
+    deferSync_->Stop();
+    dirCache_->Stop();
+}
+
+void FileSystem::Attr2Stat(InodeAttr* attr, struct stat* stat) {
+    std::memset(stat, 0, sizeof(struct stat));
+    stat->st_ino = attr->inodeid();
+    stat->st_mode = attr->mode();
+    stat->st_nlink = attr->nlink();
+    stat->st_uid = attr->uid();
+    stat->st_gid = attr->gid();
+    stat->st_size = attr->length();
+    stat->st_rdev = attr->rdev();
+    stat->st_atim.tv_sec = attr->atime(); // atime
+    stat->st_atim.tv_nsec = attr->atime_ns();
+    stat->st_mtim.tv_sec = attr->mtime();
+    stat->st_mtim.tv_nsec = attr->mtime_ns();
+    stat->st_ctim.tv_sec = attr->ctime();
+    stat->st_ctim.tv_nsec = attr->ctime_ns();
+    stat->st_blksize = option_.blockSize;;
+    stat->st_blocks = 0;
+    if (IsS3File(*attr) || IsDir(*attr)) {
+        stat->st_blocks = (attr->length() + 511) / 512;
+    }
+}
+
+void FileSystem::Entry2Param(EntryOut* entryOut,
+                             fuse_entry_param* e) {
+    std::memset(e, 0, sizeof(fuse_entry_param));
+    e->ino = entryOut->attr.inodeid();
+    e->generation = 0;
+    Attr2Stat(&entryOut->attr, &e->attr);
+    e->entry_timeout = entryOut->entryTimeout;
+    e->attr_timeout = entryOut->attrTimeout;
+}
+
+void FileSystem::SetEntryTimeout(EntryOut* entryOut) {
+    auto option = option_.kernelCacheOption;
+    if (IsDir(entryOut->attr)) {
+        entryOut->entryTimeout = option.dirEntryTimeout;
+        entryOut->attrTimeout = option.dirAttrTimeout;
+    } else {
+        entryOut->entryTimeout = option.entryTimeout;
+        entryOut->attrTimeout = option.attrTimeout;
+    }
+}
+
+void FileSystem::SetAttrTimeout(AttrOut* attrOut) {
+    auto option = option_.kernelCacheOption;
+    if (IsDir(attrOut->attr)) {
+        attrOut->attrTimeout = option.dirAttrTimeout;
+    } else {
+        attrOut->attrTimeout = option.attrTimeout;
+    }
+}
+
+// fuse reply*
+void FileSystem::ReplyError(Request req, CURVEFS_ERROR code) {
+    fuse_reply_err(req, SysErr(code));
+}
+
+void FileSystem::ReplyEntry(Request req,
+                            EntryOut* entryOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr, true);
+    fuse_entry_param e;
+    SetEntryTimeout(entryOut);
+    Entry2Param(entryOut, &e);
+    fuse_reply_entry(req, &e);
+}
+
+void FileSystem::ReplyAttr(Request req,
+                           AttrOut* attrOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &attrOut->attr, true);
+    struct stat stat;
+    SetAttrTimeout(attrOut);
+    Attr2Stat(&attrOut->attr, &stat);
+    fuse_reply_attr(req, &stat, attrOut->attrTimeout);
+}
+
+void FileSystem::ReplyReadlink(Request req, const std::string& link) {
+    fuse_reply_readlink(req, link.c_str());
+}
+
+void FileSystem::ReplyOpen(Request req, FileInfo *fi) {
+    fuse_reply_open(req, fi);
+}
+
+void FileSystem::ReplyData(Request req,
+                           struct fuse_bufvec *bufv,
+		                   enum fuse_buf_copy_flags flags) {
+    fuse_reply_data(req, bufv, flags);
+}
+
+void FileSystem::ReplyWrite(Request req, size_t count) {
+    fuse_reply_write(req, count);
+}
+
+void FileSystem::ReplyBuffer(Request req, const char *buf, size_t size) {
+    fuse_reply_buf(req, buf, size);
+}
+
+void FileSystem::ReplyStatfs(Request req, const struct statvfs *stbuf) {
+    fuse_reply_statfs(req, stbuf);
+}
+
+void FileSystem::ReplyXattr(Request req, size_t size) {
+    fuse_reply_xattr(req, size);
+}
+
+void FileSystem::ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi) {
+    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr, true);
+    fuse_entry_param e;
+    SetEntryTimeout(entryOut);
+    Entry2Param(entryOut, &e);
+    fuse_reply_create(req, &e, fi);
+}
+
+void FileSystem::AddDirEntry(Request req,
+                             DirBufferHead* buffer,
+                             DirEntry* dirEntry) {
+    struct stat stat;
+    std::memset(&stat, 0, sizeof(stat));
+    stat.st_ino = dirEntry->ino;
+
+    // add a directory entry to the buffer
+    size_t oldsize = buffer->size;
+    const char* name = dirEntry->name.c_str();
+    buffer->size += fuse_add_direntry(req, NULL, 0, name, NULL, 0);
+    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
+    fuse_add_direntry(req,
+                      buffer->p + oldsize,  // char* buf
+                      buffer->size - oldsize,  // size_t bufisze
+                      name, &stat, buffer->size);
+}
+
+void FileSystem::AddDirEntryPlus(Request req,
+                                 DirBufferHead* buffer,
+                                 DirEntry* dirEntry) {
+    AttrWatcherGuard watcher(attrWatcher_, &dirEntry->attr, false);
+    struct fuse_entry_param e;
+    EntryOut entryOut(dirEntry->attr);
+    SetEntryTimeout(&entryOut);
+    Entry2Param(&entryOut, &e);
+
+    // add a directory entry to the buffer with the attributes
+    size_t oldsize = buffer->size;
+    const char* name = dirEntry->name.c_str();
+    buffer->size += fuse_add_direntry_plus(req, NULL, 0, name, NULL, 0);
+    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
+    fuse_add_direntry_plus(req,
+                           buffer->p + oldsize,  // char* buf
+                           buffer->size - oldsize,  // size_t bufisze
+                           name, &e, buffer->size);
+}
+
+// handler*
+std::shared_ptr<FileHandler> FileSystem::NewHandler() {
+    return handlerManager_->NewHandler();
+}
+
+std::shared_ptr<FileHandler> FileSystem::FindHandler(uint64_t fh) {
+    return handlerManager_->FindHandler(fh);
+}
+
+void FileSystem::ReleaseHandler(uint64_t fh) {
+    return handlerManager_->ReleaseHandler(fh);
+}
+
+FileSystemMember FileSystem::BorrowMember() {
+    return FileSystemMember(deferSync_, openFiles_);
+}
+
+// fuse request*
+CURVEFS_ERROR FileSystem::Lookup(Request req,
+                                 Ino parent,
+                                 const std::string& name,
+                                 EntryOut* entryOut) {
+    if (name.size() > option_.maxNameLength) {
+        return CURVEFS_ERROR::NAMETOOLONG;
+    }
+
+    bool yes = negative_->Get(parent, name);
+    if (yes) {
+        return CURVEFS_ERROR::NOTEXIST;
+    }
+
+    auto rc = rpc_->Lookup(parent, name, entryOut);
+    if (rc == CURVEFS_ERROR::NOTEXIST) {
+        negative_->Put(parent, name);
+    }
+    return rc;
+}
+
+CURVEFS_ERROR FileSystem::GetAttr(Request req, Ino ino, AttrOut* attrOut) {
+    InodeAttr attr;
+    auto rc = rpc_->GetAttr(ino, &attr);
+    if (rc == CURVEFS_ERROR::OK) {
+        *attrOut = AttrOut(attr);
+    }
+    return rc;
+}
+
+CURVEFS_ERROR FileSystem::OpenDir(Request req, Ino ino, FileInfo* fi) {
+    InodeAttr attr;
+    CURVEFS_ERROR rc = rpc_->GetAttr(ino, &attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    // revalidate directory cache
+    std::shared_ptr<DirEntryList> entries;
+    bool yes = dirCache_->Get(ino, &entries);
+    if (yes) {
+        if (entries->GetMtime() != AttrMtime(attr)) {
+            dirCache_->Drop(ino);
+        }
+    }
+
+    auto handler = NewHandler();
+    handler->mtime = AttrMtime(attr);
+    fi->fh = handler->fh;
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::ReadDir(Request req,
+                                  Ino ino,
+                                  FileInfo* fi,
+                                  std::shared_ptr<DirEntryList>* entries) {
+    bool yes = dirCache_->Get(ino, entries);
+    if (yes) {
+        return CURVEFS_ERROR::OK;
+    }
+
+    CURVEFS_ERROR rc = rpc_->ReadDir(ino, entries);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    (*entries)->SetMtime(FindHandler(fi->fh)->mtime);
+    dirCache_->Put(ino, *entries);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::ReleaseDir(Request req, Ino ino, FileInfo* fi) {
+    ReleaseHandler(fi->fh);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::Open(Request req, Ino ino, FileInfo* fi) {
+    std::shared_ptr<InodeWrapper> inode;
+    bool yes = openFiles_->IsOpened(ino, &inode);
+    if (yes) {
+        openFiles_->Open(ino, inode);
+        // fi->keep_cache = 1;
+        return CURVEFS_ERROR::OK;
+    }
+
+    CURVEFS_ERROR rc = rpc_->Open(ino, &inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    TimeSpec mtime;
+    yes = attrWatcher_->GetMtime(ino, &mtime);
+    if (!yes) {
+        // It is rare which only arise when attribute evited for attr-watcher.
+        LOG(WARNING) << "open(" << ino << "): stale file handler"
+                     << ": attribute not found in wacther";
+        return CURVEFS_ERROR::STALE;
+    } else if(mtime != InodeMtime(inode)) {
+        LOG(WARNING) << "open(" << ino << "): stale file handler"
+                     << ", cache(" << mtime << ") vs remote("
+                     << InodeMtime(inode) << ")";
+        return CURVEFS_ERROR::STALE;
+    }
+
+    openFiles_->Open(ino, inode);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::Release(Request req, Ino ino) {
+    openFiles_->Close(ino);
+    return CURVEFS_ERROR::OK;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/filesystem.h

@@ -0,0 +1,152 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-08
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
+
+#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/package.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/lookup_cache.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+#include "curvefs/src/client/filesystem/openfile.h"
+#include "curvefs/src/client/filesystem/attr_watcher.h"
+#include "curvefs/src/client/filesystem/rpc_client.h"
+#include "curvefs/src/client/filesystem/defer_sync.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::FileSystemOption;
+
+struct FileSystemMember {
+    FileSystemMember(std::shared_ptr<DeferSync> deferSync,
+                     std::shared_ptr<OpenFiles> openFiles)
+        : deferSync(deferSync),
+          openFiles(openFiles) {}
+
+    std::shared_ptr<DeferSync> deferSync;
+    std::shared_ptr<OpenFiles> openFiles;
+};
+
+class FileSystem {
+ public:
+    FileSystem(struct FileSystemOption option, ExternalMember member);
+
+    void Run();
+
+    void Destory();
+
+    // fuse request
+    CURVEFS_ERROR Lookup(Request req,
+                         Ino parent,
+                         const std::string& name,
+                         EntryOut* entryOut);
+
+    CURVEFS_ERROR GetAttr(Request req, Ino ino, AttrOut* attrOut);
+
+    CURVEFS_ERROR OpenDir(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR ReadDir(Request req,
+                          Ino ino,
+                          FileInfo* fi,
+                          std::shared_ptr<DirEntryList>* entries);
+
+    CURVEFS_ERROR ReleaseDir(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR Open(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR Release(Request req, Ino ino);
+
+    // fuse reply: we control all replies to vfs layer in same entrance.
+    void ReplyError(Request req, CURVEFS_ERROR code);
+
+    void ReplyEntry(Request req, EntryOut* entryOut);
+
+    void ReplyAttr(Request req, AttrOut* attrOut);
+
+    void ReplyReadlink(Request req, const std::string& link);
+
+    void ReplyOpen(Request req, FileInfo *fi);
+
+    void ReplyData(Request req,
+                   struct fuse_bufvec *bufv,
+		           enum fuse_buf_copy_flags flags);
+
+    void ReplyWrite(Request req, size_t count);
+
+    void ReplyBuffer(Request req, const char *buf, size_t size);
+
+    void ReplyStatfs(Request req, const struct statvfs *stbuf);
+
+    void ReplyXattr(Request req, size_t size);
+
+    void ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi);
+
+    void AddDirEntry(Request req,
+                     DirBufferHead* buffer,
+                     DirEntry* dirEntry);
+
+    void AddDirEntryPlus(Request req,
+                         DirBufferHead* buffer,
+                         DirEntry* dirEntry);
+
+    // utility: file handler
+    std::shared_ptr<FileHandler> NewHandler();
+
+    std::shared_ptr<FileHandler> FindHandler(uint64_t fh);
+
+    void ReleaseHandler(uint64_t fh);
+
+    // utility: others
+    FileSystemMember BorrowMember();
+
+ private:
+    // utility: convert to system type.
+    void Attr2Stat(InodeAttr* attr, struct stat* stat);
+
+    void Entry2Param(EntryOut* entryOut, fuse_entry_param* e);
+
+    // utility: set entry/attribute timeout
+    void SetEntryTimeout(EntryOut* entryOut);
+
+    void SetAttrTimeout(AttrOut* attrOut);
+
+ private:
+    FileSystemOption option_;
+    ExternalMember member;
+    std::shared_ptr<DeferSync> deferSync_;
+    std::shared_ptr<LookupCache> negative_;
+    std::shared_ptr<DirCache> dirCache_;
+    std::shared_ptr<OpenFiles> openFiles_;
+    std::shared_ptr<AttrWatcher> attrWatcher_;
+    std::shared_ptr<HandlerManager> handlerManager_;
+    std::shared_ptr<RPCClient> rpc_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_

curvefs/src/client/filesystem/lookup_cache.cpp

@@ -0,0 +1,76 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-31
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <glog/logging.h>
+
+#include <ctime>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/filesystem/utils.h"
+#include "curvefs/src/client/filesystem/lookup_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+LookupCache::LookupCache(LookupCacheOption option)
+    : enable_(option.negativeTimeout > 0),
+      rwlock_(),
+      option_(option) {
+    lru_ = std::make_shared<LRUType>(option.lruSize);
+}
+
+std::string LookupCache::Entry2Key(Ino parent, const std::string& name) {
+    return absl::StrFormat("%d:%s", parent, name);
+}
+
+bool LookupCache::Get(Ino parent, const std::string& name) {
+    if (!enable_) {
+        return false;
+    }
+
+    ReadLockGuard lk(rwlock_);
+    std::string key = Entry2Key(parent, name);
+    TimeSpec expireTime;
+    bool yes = lru_->Get(key, &expireTime);
+    if (!yes) {
+        return false;
+    } else if (Now() > expireTime) {
+        return false;
+    }
+    return true;
+}
+
+void LookupCache::Put(Ino parent, const std::string& name) {
+    if (!enable_) {
+        return;
+    }
+
+    WriteLockGuard lk(rwlock_);
+    auto key = Entry2Key(parent, name);
+    TimeSpec expireTime = Now() + TimeSpec(option_.negativeTimeout, 0);
+    lru_->Put(key, expireTime);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/lookup_cache.h

@@ -0,0 +1,67 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-31
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::LookupCacheOption;
+
+// memory cache for lookup result, now we only support cache negative result,
+// other positive entry will be cached in kernel.
+class LookupCache {
+ public:
+    using LRUType = LRUCache<std::string, TimeSpec>;
+
+ public:
+    explicit LookupCache(LookupCacheOption option);
+
+    bool Get(Ino parent, const std::string& name);
+
+    void Put(Ino parent, const std::string& name);
+
+ private:
+    std::string Entry2Key(Ino parent, const std::string& name);
+
+ private:
+    bool enable_;
+    RWLock rwlock_;
+    LookupCacheOption option_;
+    std::shared_ptr<LRUType> lru_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_

curvefs/src/client/filesystem/message_queue.h

@@ -0,0 +1,114 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
+
+#include <glog/logging.h>
+
+#include <string>
+#include <atomic>
+#include <thread>
+
+#include "src/common/concurrent/task_queue.h"
+#include "curvefs/src/common/threading.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::TaskQueue;
+using ::curvefs::common::SetThreadName;
+
+template <typename MessageT>
+class MessageQueue {
+ public:
+    using MessageHandler = std::function<void(const MessageT& message)>;
+
+ public:
+    MessageQueue(const std::string& name, size_t bufferSize)
+        : name_(name),
+          running_(false),
+          thread_(),
+          handler_(),
+          queue_(bufferSize) {}
+
+    void Start() {
+        if (running_.exchange(true)) {
+            return;
+        }
+
+        thread_ = std::thread(&MessageQueue::Consumer, this);
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread start success";
+    }
+
+    void Stop() {
+        if (!running_.exchange(false)) {
+            return;
+        }
+
+        auto wakeup = []() {};
+        queue_.Push(wakeup);
+
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread stoping...";
+
+        thread_.join();
+
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread stopped";
+    }
+
+    void Publish(MessageT message) {
+        if (handler_ != nullptr) {
+            queue_.Push([this, message](){
+                this->handler_(message);
+            });
+        }
+    }
+
+    void Subscribe(MessageHandler handler) {
+        handler_ = handler;
+    }
+
+ private:
+    void Consumer() {
+        SetThreadName(name_.c_str());
+        while (running_.load(std::memory_order_relaxed)) {
+            queue_.Pop()();
+        }
+    }
+
+ private:
+    std::string name_;
+    std::atomic<bool> running_;
+    std::thread thread_;
+    MessageHandler handler_;
+    TaskQueue queue_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_

curvefs/src/client/filesystem/meta.cpp

@@ -0,0 +1,129 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <sys/stat.h>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+HandlerManager::HandlerManager()
+    : mutex_(),
+      dirBuffer_(std::make_shared<DirBuffer>()),
+      handlers_() {}
+
+HandlerManager::~HandlerManager() {
+    dirBuffer_->DirBufferFreeAll();
+}
+
+std::shared_ptr<FileHandler> HandlerManager::NewHandler() {
+    UniqueLock lk(mutex_);
+    auto handler = std::make_shared<FileHandler>();
+    handler->fh = dirBuffer_->DirBufferNew();
+    handler->buffer = dirBuffer_->DirBufferGet(handler->fh);
+    handler->padding = false;
+    handlers_.emplace(handler->fh, handler);
+    return handler;
+}
+
+std::shared_ptr<FileHandler> HandlerManager::FindHandler(uint64_t fh) {
+    UniqueLock lk(mutex_);
+    auto iter = handlers_.find(fh);
+    if (iter == handlers_.end()) {
+        return nullptr;
+    }
+    return iter->second;
+}
+
+void HandlerManager::ReleaseHandler(uint64_t fh) {
+    UniqueLock lk(mutex_);
+    dirBuffer_->DirBufferRelease(fh);
+    handlers_.erase(fh);
+}
+
+std::string StrMode(uint16_t mode) {
+    static std::map<uint16_t, char> type2char = {
+	    { S_IFSOCK, 's' },
+	    { S_IFLNK, 'l' },
+	    { S_IFREG, '-' },
+	    { S_IFBLK, 'b' },
+	    { S_IFDIR, 'd' },
+	    { S_IFCHR, 'c' },
+	    { S_IFIFO, 'f' },
+	    { 0, '?' },
+    };
+
+    std::string s("?rwxrwxrwx");
+	s[0] = type2char[mode & (S_IFMT & 0xffff)];
+	if (mode & S_ISUID) {
+		s[3] = 's';
+	}
+	if (mode & S_ISGID) {
+		s[6] = 's';
+	}
+	if (mode & S_ISVTX) {
+		s[9] = 't';
+	}
+
+    for (auto i = 0; i < 9; i++) {
+		if ((mode & (1 << i)) == 0) {
+			if ((s[9-i] == 's') || (s[9-i] == 't')) {
+				s[9-i] &= 0xDF;
+			} else {
+				s[9-i] = '-';
+			}
+		}
+	}
+    return s;
+}
+
+namespace {
+
+std::string Attr2Str(const InodeAttr& attr) {
+    if (!attr.IsInitialized()) {
+        return "";
+    }
+
+    std::string smode;
+    return absl::StrFormat(" (%d,[%s:0%06o,%d,%d,%d,%d,%d,%d,%d])",
+        attr.inodeid(), StrMode(attr.mode()).c_str(), attr.mode(), attr.nlink(),
+        attr.uid(), attr.gid(),
+        attr.atime(), attr.mtime(), attr.ctime(),
+        attr.length());
+}
+
+}  // namespace
+
+std::string StrEntry(EntryOut entryOut) {
+    return Attr2Str(entryOut.attr);
+}
+
+std::string StrAttr(AttrOut attrOut) {
+    return Attr2Str(attrOut.attr);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/meta.h

@@ -0,0 +1,161 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
+
+#include <iostream>
+
+#include "src/common/concurrent/concurrent.h"
+#include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/src/client/fuse_common.h"
+#include "curvefs/src/client/dir_buffer.h"
+#include "curvefs/src/client/inode_wrapper.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::Mutex;
+using ::curve::common::RWLock;
+using ::curve::common::UniqueLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::metaserver::XAttr;
+using ::curvefs::metaserver::Dentry;
+using ::curvefs::metaserver::InodeAttr;
+using ::curvefs::metaserver::FsFileType;
+using ::curvefs::client::DirBufferHead;
+using ::curvefs::client::InodeWrapper;
+
+using Ino = fuse_ino_t;
+using Request = fuse_req_t;
+using FileInfo = struct fuse_file_info;
+
+struct EntryOut {
+    EntryOut() = default;
+
+    EntryOut(InodeAttr attr) : attr(attr) {}
+
+    InodeAttr attr;
+    double entryTimeout;
+    double attrTimeout;
+};
+
+struct AttrOut {
+    AttrOut() = default;
+
+    AttrOut(InodeAttr attr) : attr(attr) {}
+
+    InodeAttr attr;
+    double attrTimeout;
+};
+
+struct DirEntry {
+    DirEntry() = default;
+
+    DirEntry(Ino ino, const std::string& name, InodeAttr attr)
+        : ino(ino), name(name), attr(attr) {}
+
+    Ino ino;
+    std::string name;
+    InodeAttr attr;
+};
+
+struct TimeSpec {
+    TimeSpec() : seconds(0), nanoSeconds(0) {}
+
+    TimeSpec(uint64_t seconds, uint32_t nanoSeconds)
+        : seconds(seconds), nanoSeconds(nanoSeconds) {}
+
+    TimeSpec(const TimeSpec& time)
+        : seconds(time.seconds), nanoSeconds(time.nanoSeconds) {}
+
+    TimeSpec& operator=(const TimeSpec& time) = default;
+
+    TimeSpec operator+(const TimeSpec& time) const {
+        return TimeSpec(seconds + time.seconds, nanoSeconds + time.nanoSeconds);
+    }
+
+    uint64_t seconds;
+    uint32_t nanoSeconds;
+};
+
+struct FileHandler {
+    uint64_t fh;
+    DirBufferHead* buffer;
+    TimeSpec mtime;
+    bool padding;  // padding buffer
+};
+
+class HandlerManager {
+ public:
+    HandlerManager();
+
+    ~HandlerManager();
+
+    std::shared_ptr<FileHandler> NewHandler();
+
+    std::shared_ptr<FileHandler> FindHandler(uint64_t id);
+
+    void ReleaseHandler(uint64_t id);
+
+ private:
+    Mutex mutex_;
+    std::shared_ptr<DirBuffer> dirBuffer_;
+    std::map<uint64_t, std::shared_ptr<FileHandler>> handlers_;
+};
+
+inline bool operator==(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds == rhs.seconds) &&
+        (lhs.nanoSeconds == rhs.nanoSeconds);
+}
+
+inline bool operator!=(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return !(lhs == rhs);
+}
+
+inline bool operator<(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds < rhs.seconds) ||
+        (lhs.seconds == rhs.seconds && lhs.nanoSeconds < rhs.nanoSeconds);
+}
+
+inline bool operator>(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds > rhs.seconds) ||
+        (lhs.seconds == rhs.seconds && lhs.nanoSeconds > rhs.nanoSeconds);
+}
+
+inline std::ostream &operator<<(std::ostream &os, const TimeSpec& time) {
+    return os << time.seconds << "." << time.nanoSeconds;
+}
+
+std::string StrMode(uint16_t mode);
+
+std::string StrEntry(EntryOut entryOut);
+
+std::string StrAttr(AttrOut attrOut);
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_

curvefs/src/client/filesystem/openfile.cpp

@@ -0,0 +1,165 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/src/client/filesystem/openfile.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+OpenFiles::OpenFiles(OpenFileOption option,
+                     std::shared_ptr<DeferSync> deferSync,
+                     std::shared_ptr<DirCache> dirCache)
+    : rwlock_(),
+      option_(option),
+      deferSync_(deferSync),
+      dirCache_(dirCache),
+      files_(std::make_shared<LRUType>(option.lruSize)) {
+}
+
+void OpenFiles::UpdateDirEntryLength(Ino ino,
+                                     const std::shared_ptr<OpenFile>& file) {
+    InodeAttr attr;
+    std::shared_ptr<DirEntryList> entries;
+    file->inode->GetInodeAttr(&attr);
+    for (const auto parent : attr.parent()) {
+        bool yes = dirCache_->Get(parent, &entries);
+        if (!yes) {
+            continue;
+        }
+
+        entries->UpdateLength(ino, attr);
+
+        VLOG(1) << "Write back file length to dir entry cache: ino = " << ino
+                << ", length = " << attr.length()
+                << ", mtime = " << AttrMtime(attr)
+                << ", ctime = " << AttrCtime(attr);
+    }
+}
+
+/*
+ * Delete(...) does:
+ *   1) write back file length to dir entry cache
+ *   2) publish to message queue which will flush file to server with async
+ *   3) delete file from lru cache
+ */
+void OpenFiles::Delete(Ino ino,
+                       const std::shared_ptr<OpenFile>& file,
+                       bool flush) {
+    UpdateDirEntryLength(ino, file);  // write back file length to dir cache
+    if (flush) {
+        deferSync_->Push(file->inode);
+    }
+    files_->Remove(ino);
+
+    VLOG(1) << "Delete open file cache: ino = " << ino
+            << ", refs = " << file->refs
+            << ", mtime = " << InodeMtime(file->inode);
+}
+
+void OpenFiles::Evit(size_t size) {
+    Ino ino;
+    std::shared_ptr<OpenFile> file;
+    while (files_->Size() + size >= option_.lruSize) {
+        bool yes = files_->GetLast(&ino, &file);
+        if (!yes) {
+            break;
+        }
+        Delete(ino, file, true);
+    }
+}
+
+void OpenFiles::Open(Ino ino, std::shared_ptr<InodeWrapper> inode) {
+    WriteLockGuard lk(rwlock_);
+
+    Evit(1);
+
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (yes) {
+        file->refs++;
+        return;
+    }
+
+    file = std::make_shared<OpenFile>(inode);
+    file->refs++;
+    files_->Put(ino, file);
+
+    VLOG(1) << "Insert open file cache: ino = " << ino
+            << ", refs = " << file->refs
+            << ", mtime = " << InodeMtime(file->inode);
+}
+
+bool OpenFiles::IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
+    ReadLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return false;
+    }
+    *inode = file->inode;
+    return true;
+}
+
+void OpenFiles::Close(Ino ino) {
+    WriteLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return;
+    }
+
+    if (file->refs > 0) {
+        file->refs--;
+    }
+
+    if (file->refs == 0) {
+        Delete(ino, file, false);  // file already flushed before close
+    }
+}
+
+/*
+ * CloseAll() does:
+ *   1) flush all file to server and delete from LRU cache
+ *   2) stop message queue
+ */
+void OpenFiles::CloseAll() {
+    WriteLockGuard lk(rwlock_);
+    Evit(option_.lruSize);
+}
+
+bool OpenFiles::GetFileAttr(Ino ino, InodeAttr* attr) {
+    ReadLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return false;
+    }
+
+    file->inode->GetInodeAttr(attr);
+    return true;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/openfile.h

@@ -0,0 +1,89 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/inode_wrapper.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/defer_sync.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::InodeWrapper;
+using ::curvefs::client::common::OpenFileOption;
+
+struct OpenFile {
+    OpenFile(std::shared_ptr<InodeWrapper> inode)
+        : inode(inode), refs(0) {}
+
+    std::shared_ptr<InodeWrapper> inode;
+    uint64_t refs;
+};
+
+class OpenFiles {
+ public:
+    using LRUType = LRUCache<Ino, std::shared_ptr<OpenFile>>;
+
+ public:
+    explicit OpenFiles(OpenFileOption option,
+                       std::shared_ptr<DeferSync> deferSync,
+                       std::shared_ptr<DirCache> dirCache);
+
+    void Open(Ino ino, std::shared_ptr<InodeWrapper> inode);
+
+    bool IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode);
+
+    void Close(Ino ino);
+
+    void CloseAll();
+
+    bool GetFileAttr(Ino ino, InodeAttr* inode);
+
+ private:
+    void UpdateDirEntryLength(Ino ino,
+                              const std::shared_ptr<OpenFile>& file);
+
+    void Delete(Ino ino, const std::shared_ptr<OpenFile>& file, bool flush);
+
+    void Evit(size_t size);
+
+ private:
+    RWLock rwlock_;
+    OpenFileOption option_;
+    std::shared_ptr<DeferSync> deferSync_;
+    std::shared_ptr<DirCache> dirCache_;
+    std::shared_ptr<LRUType> files_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_

curvefs/src/client/filesystem/package.h

@@ -0,0 +1,51 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
+
+#include "curvefs/src/client/dentry_cache_manager.h"
+#include "curvefs/src/client/inode_cache_manager.h"
+#include "curvefs/src/client/xattr_manager.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+struct ExternalMember {  // external member depended by FileSystem
+    ExternalMember(std::shared_ptr<InodeCacheManager> inodeManager,
+                   std::shared_ptr<DentryCacheManager> dentryManager,
+                   std::shared_ptr<XattrManager> xattrManager)
+        : inodeManager(inodeManager),
+          dentryManager(dentryManager),
+          xattrManager(xattrManager) {}
+
+    std::shared_ptr<InodeCacheManager> inodeManager;
+    std::shared_ptr<DentryCacheManager> dentryManager;
+    std::shared_ptr<XattrManager> xattrManager;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_

curvefs/src/client/filesystem/rpc_client.cpp

@@ -0,0 +1,121 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+#include <set>
+
+#include "curvefs/src/client/filesystem/rpc_client.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+RPCClient::RPCClient(RPCOption option,
+                     ExternalMember member)
+    : option_(option),
+      inodeManager_(member.inodeManager),
+      dentryManager_(member.dentryManager) {}
+
+CURVEFS_ERROR RPCClient::GetAttr(Ino ino, InodeAttr* attr) {
+    CURVEFS_ERROR rc = inodeManager_->GetInodeAttr(ino, attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(getattr::GetInodeAttr) failed, retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+CURVEFS_ERROR RPCClient::Lookup(Ino parent,
+                                const std::string& name,
+                                EntryOut* entryOut) {
+    Dentry dentry;
+    CURVEFS_ERROR rc = dentryManager_->GetDentry(parent, name, &dentry);
+    if (rc != CURVEFS_ERROR::OK) {
+        if (rc != CURVEFS_ERROR::NOTEXIST) {
+            LOG(WARNING) << "rpc(lookup::GetDentry) failed, retCode = " << rc
+                         << ", parent = " << parent << ", name = " << name;
+        }
+        return rc;
+    }
+
+    Ino ino = dentry.inodeid();
+    rc = inodeManager_->GetInodeAttr(ino, &entryOut->attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(lookup::GetInodeAttr) failed, retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+CURVEFS_ERROR RPCClient::ReadDir(Ino ino, std::shared_ptr<DirEntryList>* entries) {
+    uint32_t limit = option_.listDentryLimit;
+
+    std::list<Dentry> dentries;
+    CURVEFS_ERROR rc = dentryManager_->ListDentry(ino, &dentries, limit);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(readdir::ListDentry) failed, retCode = " << rc
+                   << ", ino = " << ino;
+        return rc;
+    }
+
+    std::set<uint64_t> inos;
+    std::map<uint64_t, InodeAttr> attrs;
+    std::for_each(dentries.begin(), dentries.end(), [&](Dentry& dentry){
+        inos.emplace(dentry.inodeid());
+    });
+    rc = inodeManager_->BatchGetInodeAttrAsync(ino, &inos, &attrs);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(readdir::BatchGetInodeAttrAsync) failed"
+                   << ", retCode = " << rc << ", ino = " << ino;
+        return rc;
+    }
+
+    DirEntry dirEntry;
+    for (const auto& dentry : dentries) {
+        Ino ino = dentry.inodeid();
+        auto iter = attrs.find(ino);
+        if (iter == attrs.end()) {
+            LOG(WARNING) << "rpc(readdir::BatchGetInodeAttrAsync) "
+                         << "missing attribute, ino = " << ino;
+            continue;
+        }
+
+        dirEntry.ino = ino;
+        dirEntry.name = std::move(dentry.name());
+        dirEntry.attr = std::move(iter->second);
+        (*entries)->Add(dirEntry);
+    }
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR RPCClient::Open(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
+    CURVEFS_ERROR rc = inodeManager_->GetInode(ino, *inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(open/GetInode) failed" << ", retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/rpc_client.h

@@ -0,0 +1,62 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
+
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/package.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::RPCOption;
+
+class RPCClient {
+ public:
+    RPCClient(RPCOption option,
+              ExternalMember member);
+
+    CURVEFS_ERROR GetAttr(Ino ino, InodeAttr* attr);
+
+    CURVEFS_ERROR Lookup(Ino parent,
+                         const std::string& name,
+                         EntryOut* entryOut);
+
+    CURVEFS_ERROR ReadDir(Ino ino, std::shared_ptr<DirEntryList>* entries);
+
+    CURVEFS_ERROR Open(Ino ino, std::shared_ptr<InodeWrapper>* inode);
+
+ private:
+    RPCOption option_;
+    std::shared_ptr<InodeCacheManager> inodeManager_;
+    std::shared_ptr<DentryCacheManager> dentryManager_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_

curvefs/src/client/filesystem/utils.cpp

@@ -0,0 +1,69 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <glog/logging.h>
+
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+bool IsDir(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_DIRECTORY;
+}
+
+bool IsS3File(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_S3;
+}
+
+bool IsVolmeFile(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_FILE;
+}
+
+bool IsSymLink(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_SYM_LINK;
+}
+
+TimeSpec AttrMtime(const InodeAttr& attr) {
+    return TimeSpec(attr.mtime(), attr.mtime_ns());
+}
+
+TimeSpec AttrCtime(const InodeAttr& attr) {
+    return TimeSpec(attr.ctime(), attr.ctime_ns());
+}
+
+TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode) {
+    InodeAttr attr;
+    inode->GetInodeAttr(&attr);
+    return AttrMtime(attr);
+}
+
+TimeSpec Now() {
+    struct timespec now;
+    clock_gettime(CLOCK_REALTIME, &now);
+    return TimeSpec(now.tv_sec, now.tv_nsec);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/utils.h

@@ -0,0 +1,57 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
+
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/package.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+// directory
+bool IsDir(const InodeAttr& attr);
+
+// file which data is stored in s3
+bool IsS3File(const InodeAttr& attr);
+
+// file which data is stored in volume
+bool IsVolmeFile(const InodeAttr& attr);
+
+// symbol link
+bool IsSymLink(const InodeAttr& attr);
+
+struct TimeSpec AttrMtime(const InodeAttr& attr);
+
+struct TimeSpec AttrCtime(const InodeAttr& attr);
+
+struct TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode);
+
+struct TimeSpec Now();
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_

curvefs/src/client/fuse_client.cpp

@@ -26,6 +26,7 @@
 #include <list>
 #include <algorithm>
 #include <cstring>
+#include <memory>
 #include <string>
 #include <vector>
 #include <set>
@@ -34,10 +35,11 @@
 
 #include "curvefs/proto/mds.pb.h"
 #include "curvefs/src/client/common/common.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/fuse_common.h"
 #include "curvefs/src/client/client_operator.h"
 #include "curvefs/src/client/inode_wrapper.h"
+#include "curvefs/src/client/warmup/warmup_manager.h"
 #include "curvefs/src/client/xattr_manager.h"
 #include "curvefs/src/common/define.h"
 #include "src/common/net_common.h"
@@ -51,8 +53,11 @@ using ::curvefs::common::S3Info;
 using ::curvefs::common::Volume;
 using ::curvefs::mds::topology::PartitionTxId;
 using ::curvefs::mds::FSStatusCode_Name;
-using ::curvefs::client::common::MAXXATTRLENGTH;
-using ::curvefs::client::common::FileHandle;
+using ::curvefs::client::common::MAX_XATTR_NAME_LENGTH;
+using ::curvefs::client::common::MAX_XATTR_VALUE_LENGTH;
+using ::curvefs::client::filesystem::ExternalMember;
+using ::curvefs::client::filesystem::DirEntry;
+using ::curvefs::client::filesystem::DirEntryList;
 
 #define RETURN_IF_UNSUCCESS(action)                                            \
     do {                                                                       \
@@ -114,6 +119,13 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
     curve::client::ClientDummyServerInfo::GetInstance().SetPort(listenPort);
     curve::client::ClientDummyServerInfo::GetInstance().SetIP(localIp);
 
+    {
+        ExternalMember member(inodeManager_,
+                              dentryManager_,
+                              xattrManager_);
+        fs_ = std::make_shared<FileSystem>(option_.fileSystemOption, member);
+    }
+
     MetaStatusCode ret2 =
         metaClient_->Init(option.excutorOpt, option.excutorInternalOpt,
                           metaCache, channelManager);
@@ -121,228 +133,35 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
         return CURVEFS_ERROR::INTERNAL;
     }
 
-    CURVEFS_ERROR ret3 =
-        inodeManager_->Init(option.iCacheLruSize, option.enableICacheMetrics,
-                            option.flushPeriodSec, option.refreshDataOption);
-    if (ret3 != CURVEFS_ERROR::OK) {
-        return ret3;
-    }
-    ret3 =
-        dentryManager_->Init(option.dCacheLruSize, option.enableDCacheMetrics);
-    if (ret3 != CURVEFS_ERROR::OK) {
-        return ret3;
-    }
-    warmUpFile_.exist = false;
-    bgCmdStop_.store(false, std::memory_order_release);
-    bgCmdTaskThread_ = Thread(&FuseClient::WarmUpTask, this);
-    taskFetchMetaPool_.Start(WARMUP_THREADS);
-    return ret3;
-}
-
-void FuseClient::WarmUpTask() {
-    // TODO(hzwuhongsong): Maybe we can start the warmup thread after mount
-    while (!mounted_.load(std::memory_order_acquire)) {
-        usleep(WARMUP_CHECKINTERVAL_US);
-        VLOG(6) << "wait mount success.";
-        continue;
-    }
-    while (!bgCmdStop_.load(std::memory_order_acquire)) {
-        std::list<std::string> readAheadPaths;
-        WaitWarmUp();
-        while (hasWarmTask()) {
-            std::string warmUpTask;
-            GetwarmTask(&warmUpTask);
-            VLOG(9) << "warmup task is: " << warmUpTask;
-            std::string pDelimiter = "/";
-            char* pToken = nullptr;
-            char* pSave = nullptr;
-            pToken = strtok_r(const_cast<char*>(warmUpTask.c_str()),
-              const_cast<char*>(pDelimiter.c_str()), &pSave);
-            if (nullptr == pToken) {
-                VLOG(6) << "warmUpTask nullptr";
-                continue;
-            }
-            Dentry dentry;
-            CURVEFS_ERROR ret = dentryManager_->GetDentry(
-              fsInfo_->rootinodeid(), pToken, &dentry);
-            if (ret != CURVEFS_ERROR::OK) {
-                LOG(WARNING) << "FetchDentry error: " << ret
-                             << ", name: " << warmUpTask;
-                return;
-            }
-            if (FsFileType::TYPE_S3 != dentry.type()) {
-                LOG(WARNING) << "not a file: " << warmUpTask
-                        << "type is: " << dentry.type();
-                return;
-            }
-            fuse_ino_t  ino = dentry.inodeid();
-            std::shared_ptr<InodeWrapper> inodeWrapper;
-            ret = inodeManager_->GetInode(ino, inodeWrapper);
-            if (ret != CURVEFS_ERROR::OK) {
-                LOG(ERROR) << "inodeManager get inode fail, ret = "
-                           << ret << ", inodeid = " << ino;
-                return;
-            }
-            uint64_t len = inodeWrapper->GetLength();
-            VLOG(9) << "ino is: " << ino << ", len is: " << len;
-            WarmUpFileContext_t warmUpFile{ino, len, true};
-            SetWarmUpFile(warmUpFile);
+    {  // init inode manager
+        auto member = fs_->BorrowMember();
+        CURVEFS_ERROR rc = inodeManager_->Init(option.refreshDataOption,
+                                               member.openFiles,
+                                               member.deferSync);
+        if (rc != CURVEFS_ERROR::OK) {
+            return rc;
         }
     }
-}
 
-void FuseClient::FetchDentryEnqueue(std::string file) {
-    VLOG(9) << "FetchDentryEnqueue start: " << file;
-    auto task = [this, file]() {
-        LookPath(file);
-    };
-    taskFetchMetaPool_.Enqueue(task);
-}
+    if (warmupManager_ != nullptr) {
+        warmupManager_->Init(option);
+        warmupManager_->SetFsInfo(fsInfo_);
+    }
 
-void FuseClient::LookPath(std::string file) {
-    VLOG(9) << "LookPath start: " << file;
-    std::vector<std::string> splitPath;
-    // remove enter, newline, blank
-    std::string blanks("\r\n ");
-    file.erase(0, file.find_first_not_of(blanks));
-    file.erase(file.find_last_not_of(blanks) + 1);
-    if (file.empty()) {
-        VLOG(9) << "empty path";
-        return;
-    }
-    bool isRoot = false;
-    if (file == "/") {
-        splitPath.push_back(file);
-        isRoot = true;
-    } else {
-        splitStr(file, "/", &splitPath);
-    }
-    VLOG(6) << "splitPath size is: " << splitPath.size();
-    if (splitPath.size() == 1 && isRoot) {
-        VLOG(9) << "i am root";
-        FetchChildDentryEnqueue(fsInfo_->rootinodeid());
-        return;
-    } else if (splitPath.size() == 1) {
-        VLOG(9) << "parent is root: " << fsInfo_->rootinodeid()
-                << ", path is: " << splitPath[0];
-        this->FetchDentry(fsInfo_->rootinodeid(), splitPath[0]);
-        return;
-    } else if (splitPath.size() > 1) {  // travel path
-        VLOG(9) << "traverse path start: " << splitPath.size();
-        std::string lastName = splitPath.back();
-        splitPath.pop_back();
-        fuse_ino_t ino = fsInfo_->rootinodeid();
-        for (auto iter : splitPath) {
-            VLOG(9) << "traverse path: " << iter
-                    << "ino is: " << ino;
-            Dentry dentry;
-            std::string pathName = iter;
-            CURVEFS_ERROR ret = dentryManager_->GetDentry(
-              ino, pathName, &dentry);
-            if (ret != CURVEFS_ERROR::OK) {
-                if (ret != CURVEFS_ERROR::NOTEXIST) {
-                  LOG(WARNING) << "dentryManager_ get dentry fail, ret = "
-                    << ret << ", parent inodeid = " << ino
-                    << ", name = " << file;
-                }
-                VLOG(9) << "FetchDentry error: " << ret;
-                return;
-            }
-            ino = dentry.inodeid();
-        }
-        this->FetchDentry(ino, lastName);
-        VLOG(9) << "ino is: " << ino
-                << "lastname is: " << lastName;
-        return;
-    } else {
-        VLOG(3) << "unknown path";
-    }
-    return;
-}
-
-void FuseClient::FetchChildDentryEnqueue(fuse_ino_t  ino) {
-    auto task = [this, ino]() {
-       // reverse from root
-       this->FetchChildDentry(ino);
-    };
-    taskFetchMetaPool_.Enqueue(task);
-}
-
-void FuseClient::FetchChildDentry(fuse_ino_t ino) {
-    VLOG(9) << "FetchChildDentry start: " <<  ino;
-    std::list<Dentry> dentryList;
-    auto limit = option_.listDentryLimit;
-    CURVEFS_ERROR ret = dentryManager_->ListDentry(
-      ino, &dentryList, limit);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "dentryManager_ ListDentry fail, ret = " << ret
-                    << ", parent = " << ino;
-        return;
-    }
-    for (auto iter : dentryList) {
-        VLOG(9) << "FetchChildDentry: " <<  iter.name();
-        if (FsFileType::TYPE_S3 == iter.type()) {
-            std::unique_lock<std::mutex> lck(fetchMtx_);
-            readAheadFiles_.push_front(iter.inodeid());
-            VLOG(9) << "FetchChildDentry: " << iter.inodeid();;
-        } else if (FsFileType::TYPE_DIRECTORY == iter.type()) {
-            FetchChildDentryEnqueue(iter.inodeid());
-            VLOG(9) << "FetchChildDentry: " << iter.inodeid();
-        } else if (FsFileType::TYPE_SYM_LINK == iter.type()) {  // need todo
-        } else {
-            VLOG(9) << "unknown type";
-        }
-    }
-    return;
-}
-
-void FuseClient::FetchDentry(fuse_ino_t ino, std::string file) {
-    VLOG(9) << "FetchDentry start: " << file
-            << ", ino: " << ino;
-    Dentry dentry;
-    CURVEFS_ERROR ret = dentryManager_->GetDentry(ino, file, &dentry);
-    if (ret != CURVEFS_ERROR::OK) {
-        if (ret != CURVEFS_ERROR::NOTEXIST) {
-            LOG(WARNING) << "dentryManager_ get dentry fail, ret = " << ret
-                         << ", parent inodeid = " << ino
-                         << ", name = " << file;
-        }
-        VLOG(1) << "FetchDentry error: " << ret;
-        return;
-    }
-    if (FsFileType::TYPE_S3 == dentry.type()) {
-        std::unique_lock<std::mutex> lck(fetchMtx_);
-        readAheadFiles_.push_front(dentry.inodeid());
-        return;
-    } else if (FsFileType::TYPE_DIRECTORY == dentry.type()) {
-        FetchChildDentryEnqueue(dentry.inodeid());
-        VLOG(9) << "FetchDentry: " << dentry.inodeid();
-        return;
-
-    } else if (FsFileType::TYPE_SYM_LINK == dentry.type()) {
-    } else {
-        VLOG(3) << "unkown, file: " << file
-                << ", ino: " << ino;
-    }
-    VLOG(9) << "FetchDentry end: " << file
-            << ", ino: " << ino;
-    return;
+    return CURVEFS_ERROR::OK;
 }
 
 void FuseClient::UnInit() {
-    bgCmdStop_.store(true, std::memory_order_release);
-    WarmUpRun();
-    if (bgCmdTaskThread_.joinable()) {
-        bgCmdTaskThread_.join();
+    if (warmupManager_ != nullptr) {
+        warmupManager_->UnInit();
     }
-    taskFetchMetaPool_.Stop();
+
     delete mdsBase_;
     mdsBase_ = nullptr;
 }
 
 CURVEFS_ERROR FuseClient::Run() {
     if (isStop_.exchange(false)) {
-        inodeManager_->Run();
         return CURVEFS_ERROR::OK;
     }
     return CURVEFS_ERROR::INTERNAL;
@@ -350,7 +169,6 @@ CURVEFS_ERROR FuseClient::Run() {
 
 void FuseClient::Fini() {
     if (!isStop_.exchange(true)) {
-        inodeManager_->Stop();
         xattrManager_->Stop();
     }
 }
@@ -363,7 +181,7 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
                                                        : mOpts->mountPoint);
     std::string fsName = (mOpts->fsName == nullptr) ? "" : mOpts->fsName;
 
-    mountpoint_.set_cto(FLAGS_enableCto);
+    mountpoint_.set_cto(false);
 
     int retVal = SetHostPortInMountPoint(&mountpoint_);
     if (retVal < 0) {
@@ -389,6 +207,8 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
 
     fsMetric_ = std::make_shared<FSMetric>(fsName);
 
+    fs_->Run();
+
     // init fsname and mountpoint
     leaseExecutor_->SetFsName(fsName);
     leaseExecutor_->SetMountPoint(mountpoint_);
@@ -397,7 +217,9 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
     }
 
     init_ = true;
-    mounted_.store(true, std::memory_order_release);
+    if (warmupManager_ != nullptr) {
+        warmupManager_->SetMounted(true);
+    }
     return CURVEFS_ERROR::OK;
 }
 
@@ -407,7 +229,7 @@ void FuseClient::FuseOpDestroy(void *userdata) {
     }
 
     FlushAll();
-    dirBuf_->DirBufferFreeAll();
+    fs_->Destory();
 
     // stop lease before umount fs, otherwise, lease request after umount fs
     // will add a mountpoint entry.
@@ -439,80 +261,23 @@ void FuseClient::FuseOpDestroy(void *userdata) {
               << " success!";
 }
 
-void InodeAttr2ParamAttr(const InodeAttr &inodeAttr, struct stat *attr) {
-    attr->st_ino = inodeAttr.inodeid();
-    attr->st_mode = inodeAttr.mode();
-    attr->st_nlink = inodeAttr.nlink();
-    attr->st_uid = inodeAttr.uid();
-    attr->st_gid = inodeAttr.gid();
-    attr->st_size = inodeAttr.length();
-    attr->st_rdev = inodeAttr.rdev();
-    attr->st_atim.tv_sec = inodeAttr.atime();
-    attr->st_atim.tv_nsec = inodeAttr.atime_ns();
-    attr->st_mtim.tv_sec = inodeAttr.mtime();
-    attr->st_mtim.tv_nsec = inodeAttr.mtime_ns();
-    attr->st_ctim.tv_sec = inodeAttr.ctime();
-    attr->st_ctim.tv_nsec = inodeAttr.ctime_ns();
-    attr->st_blksize = kOptimalIOBlockSize;
-
-    switch (inodeAttr.type()) {
-        case metaserver::TYPE_S3:
-            attr->st_blocks = (inodeAttr.length() + 511) / 512;
-            break;
-        default:
-            attr->st_blocks = 0;
-            break;
+CURVEFS_ERROR FuseClient::FuseOpLookup(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       EntryOut* entryOut) {
+    CURVEFS_ERROR rc = fs_->Lookup(req, parent, name, entryOut);
+    if (rc != CURVEFS_ERROR::OK && rc != CURVEFS_ERROR::NOTEXIST) {
+        LOG(ERROR) << "Lookup() failed, retCode = " << rc
+                   << ", parent = " << parent << ", name = " << name;
     }
+    return rc;
 }
 
-void GetDentryParamFromInodeAttr(
-    const FuseClientOption &option,
-    const InodeAttr &inodeAttr,
-    fuse_entry_param *param) {
-    memset(param, 0, sizeof(fuse_entry_param));
-    param->ino = inodeAttr.inodeid();
-    param->generation = 0;
-    InodeAttr2ParamAttr(inodeAttr, &param->attr);
-    param->attr_timeout = option.attrTimeOut;
-    param->entry_timeout = option.entryTimeOut;
-}
-
-CURVEFS_ERROR FuseClient::FuseOpLookup(fuse_req_t req, fuse_ino_t parent,
-                                       const char *name, fuse_entry_param *e) {
-    VLOG(1) << "FuseOpLookup parent: " << parent
-            << ", name: " << name;
-    if (strlen(name) > option_.maxNameLength) {
-        return CURVEFS_ERROR::NAMETOOLONG;
-    }
-
-    Dentry dentry;
-    CURVEFS_ERROR ret = dentryManager_->GetDentry(parent, name, &dentry);
-    if (ret != CURVEFS_ERROR::OK) {
-        if (ret != CURVEFS_ERROR::NOTEXIST) {
-            LOG(WARNING) << "dentryManager_ get dentry fail, ret = " << ret
-                         << ", parent inodeid = " << parent
-                         << ", name = " << name;
-        }
-        return ret;
-    }
-
-    fuse_ino_t ino = dentry.inodeid();
-    InodeAttr attr;
-    ret = inodeManager_->GetInodeAttr(ino, &attr);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
-                   << ", inodeid = " << ino;
-        return ret;
-    }
-
-    GetDentryParamFromInodeAttr(option_, attr, e);
-    return ret;
-}
-
-CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
-                                     struct fuse_file_info *fi) {
-    VLOG(1) << "FuseOpOpen, ino: " << ino;
+CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req,
+                                          fuse_ino_t ino,
+                                          struct fuse_file_info* fi) {
     std::shared_ptr<InodeWrapper> inodeWrapper;
+    // alredy opened
     CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
@@ -560,14 +325,23 @@ CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
             return CURVEFS_ERROR::NOPERMISSION;
         }
     }
-    if (FLAGS_enableCto) {
-        inodeManager_->AddOpenedInode(ino);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     struct fuse_file_info* fi) {
+    CURVEFS_ERROR rc = fs_->Open(req, ino, fi);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "open(" << ino << ") failed, retCode = " << rc;
+        return rc;
     }
-    return ret;
+    return HandleOpenFlags(req, ino, fi);
 }
 
 CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
     fuse_ino_t parent, FsFileType type, NlinkChange nlink) {
+
     std::shared_ptr<InodeWrapper> parentInodeWrapper;
     auto ret = inodeManager_->GetInode(parent, parentInodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
@@ -583,16 +357,29 @@ CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
         if (FsFileType::TYPE_DIRECTORY == type) {
             parentInodeWrapper->UpdateNlinkLocked(nlink);
         }
+
+        if (option_.fileSystemOption.deferSyncOption.deferDirMtime) {
+            inodeManager_->ShipToFlush(parentInodeWrapper);
+        } else {
+            ret = parentInodeWrapper->SyncAttr();
+            if (ret != CURVEFS_ERROR::OK) {
+                return CURVEFS_ERROR::OK;
+            }
+        }
     }
-    inodeManager_->ShipToFlush(parentInodeWrapper);
+
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
-                                   const char *name, mode_t mode,
-                                   FsFileType type, dev_t rdev,
-                                   fuse_entry_param *e) {
-    if (strlen(name) > option_.maxNameLength) {
+CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req,
+                                   fuse_ino_t parent,
+                                   const char* name,
+                                   mode_t mode,
+                                   FsFileType type,
+                                   dev_t rdev,
+                                   bool internal,
+                                   std::shared_ptr<InodeWrapper>& inodeWrapper) {
+    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
     const struct fuse_ctx *ctx = fuse_req_ctx(req);
@@ -610,7 +397,6 @@ CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
     param.rdev = rdev;
     param.parent = parent;
 
-    std::shared_ptr<InodeWrapper> inodeWrapper;
     CURVEFS_ERROR ret = inodeManager_->CreateInode(param, inodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "inodeManager CreateInode fail, ret = " << ret
@@ -681,19 +467,26 @@ CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
         }
     }
 
-    InodeAttr attr;
-    inodeWrapper->GetInodeAttr(&attr);
-    GetDentryParamFromInodeAttr(option_, attr, e);
     return ret;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpMkDir(fuse_req_t req, fuse_ino_t parent,
-                                      const char *name, mode_t mode,
-                                      fuse_entry_param *e) {
+CURVEFS_ERROR FuseClient::FuseOpMkDir(fuse_req_t req,
+                                      fuse_ino_t parent,
+                                      const char *name,
+                                      mode_t mode,
+                                      EntryOut* entryOut) {
     VLOG(1) << "FuseOpMkDir, parent: " << parent << ", name: " << name
             << ", mode: " << mode;
-    return MakeNode(req, parent, name, S_IFDIR | mode,
-                    FsFileType::TYPE_DIRECTORY, 0, e);
+    bool internal = false;
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR rc = MakeNode(req, parent, name, S_IFDIR | mode,
+                                FsFileType::TYPE_DIRECTORY, 0, internal, inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    inode->GetInodeAttr(&entryOut->attr);
+    return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR FuseClient::FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
@@ -704,7 +497,7 @@ CURVEFS_ERROR FuseClient::FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
 
 CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
                                      const char *name, FsFileType type) {
-    if (strlen(name) > option_.maxNameLength) {
+    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
     Dentry dentry;
@@ -781,144 +574,91 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
                          << ", xattr = " << xattr.DebugString();
         }
     }
-
-    inodeManager_->ClearInodeCache(ino);
     return ret;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino,
-                                        struct fuse_file_info *fi) {
-    VLOG(1) << "FuseOpOpenDir ino = " << ino;
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
-                   << ", inodeid = " << ino;
-        return ret;
+CURVEFS_ERROR FuseClient::FuseOpOpenDir(fuse_req_t req,
+                                        fuse_ino_t ino,
+                                        struct fuse_file_info* fi) {
+    CURVEFS_ERROR rc = fs_->OpenDir(req, ino, fi);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "opendir() failed, retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+CURVEFS_ERROR FuseClient::FuseOpReadDir(fuse_req_t req,
+                                        fuse_ino_t ino,
+                                        size_t size,
+                                        off_t off,
+                                        struct fuse_file_info *fi,
+                                        char** bufferOut,
+                                        size_t* rSize,
+                                        bool plus) {
+    auto handler = fs_->FindHandler(fi->fh);
+    DirBufferHead* buffer = handler->buffer;
+    if (!handler->padding) {
+        auto entries = std::make_shared<DirEntryList>();
+        CURVEFS_ERROR rc = fs_->ReadDir(req, ino, fi, &entries);
+        if (rc != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "readdir() failed, retCode = " << rc
+                       << ", ino = " << ino << ", fh = " << fi->fh;
+            return rc;
+        }
+
+        entries->Iterate([&](DirEntry* dirEntry){
+            if (plus) {
+                fs_->AddDirEntryPlus(req, buffer, dirEntry);
+            } else {
+                fs_->AddDirEntry(req, buffer, dirEntry);
+            }
+        });
+        handler->padding = true;
+        //LOG(INFO) << "<<< padding";
     }
 
-    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-
-    uint64_t dindex = dirBuf_->DirBufferNew();
-    fi->fh = dindex;
-    VLOG(1) << "FuseOpOpenDir, ino: " << ino << ", dindex: " << dindex;
-    return ret;
-}
-
-CURVEFS_ERROR FuseClient::FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino,
-                                           struct fuse_file_info *fi) {
-    uint64_t dindex = fi->fh;
-    VLOG(1) << "FuseOpReleaseDir, ino: " << ino << ", dindex: " << dindex;
-    dirBuf_->DirBufferRelease(dindex);
-
-    // release inodeAttr cache
-    inodeManager_->ReleaseCache(ino);
+    if (off < buffer->size) {
+        *bufferOut = buffer->p + off;
+        *rSize = std::min(buffer->size - off, size);
+    } else {
+        *bufferOut = nullptr;
+        *rSize = 0;
+    }
     return CURVEFS_ERROR::OK;
 }
 
-static void dirbuf_add(fuse_req_t req, struct DirBufferHead *b,
-                       const Dentry &dentry,
-                       const FuseClientOption &option,
-                       bool cacheDir,
-                       InodeAttr *attr = nullptr) {
-    struct stat stbuf;
-    struct fuse_entry_param param;
-    size_t oldsize = b->size;
-    if (!cacheDir) {
-        b->size += fuse_add_direntry(req, NULL, 0, dentry.name().c_str(),
-                                     NULL, 0);
-        b->p = static_cast<char *>(realloc(b->p, b->size));
-        memset(&stbuf, 0, sizeof(stbuf));
-        stbuf.st_ino = dentry.inodeid();
-        fuse_add_direntry(req, b->p + oldsize, b->size - oldsize,
-                          dentry.name().c_str(), &stbuf, b->size);
-    } else {
-        b->size += fuse_add_direntry_plus(req, NULL, 0, dentry.name().c_str(),
-                                          NULL, 0);
-        b->p = static_cast<char *>(realloc(b->p, b->size));
-        GetDentryParamFromInodeAttr(option, *attr, &param);
-        fuse_add_direntry_plus(req, b->p + oldsize, b->size - oldsize,
-                               dentry.name().c_str(), &param, b->size);
+CURVEFS_ERROR FuseClient::FuseOpReleaseDir(fuse_req_t req,
+                                           fuse_ino_t ino,
+                                           struct fuse_file_info* fi) {
+    CURVEFS_ERROR rc = fs_->ReleaseDir(req, ino, fi);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "releasedir() failed, retCode = " << rc
+                   << ", ino = " << ino;
     }
-}
-
-CURVEFS_ERROR FuseClient::FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
-                                            size_t size, off_t off,
-                                            struct fuse_file_info *fi,
-                                            char **buffer, size_t *rSize,
-                                            bool cacheDir) {
-    VLOG(1) << "FuseOpReadDirPlus ino: " << ino << ", size: " << size
-            << ", off = " << off << ", cacheDir = " << cacheDir;
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
-                   << ", inodeid = " << ino;
-        return ret;
-    }
-
-    uint64_t dindex = fi->fh;
-    DirBufferHead *bufHead = dirBuf_->DirBufferGet(dindex);
-    if (!bufHead->wasRead) {
-        std::list<Dentry> dentryList;
-        std::set<uint64_t> inodeIds;
-        std::map<uint64_t, InodeAttr> inodeAttrMap;
-        auto limit = option_.listDentryLimit;
-        ret = dentryManager_->ListDentry(ino, &dentryList, limit);
-        if (ret != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "dentryManager_ ListDentry fail, ret = " << ret
-                       << ", parent = " << ino;
-            return ret;
-        }
-
-        if (!cacheDir) {
-            for (const auto &dentry : dentryList) {
-                dirbuf_add(req, bufHead, dentry, option_, cacheDir);
-            }
-        } else {
-            for (const auto &dentry : dentryList) {
-                inodeIds.emplace(dentry.inodeid());
-            }
-            VLOG(3) << "batch get inode size = " << inodeIds.size();
-            ret = inodeManager_->BatchGetInodeAttrAsync(ino, &inodeIds,
-                                                        &inodeAttrMap);
-            if (ret != CURVEFS_ERROR::OK) {
-                LOG(ERROR) << "BatchGetInodeAttr failed when FuseOpReadDir"
-                           << ", parentId = " << ino;
-                return ret;
-            }
-
-            for (const auto &dentry : dentryList) {
-                auto iter = inodeAttrMap.find(dentry.inodeid());
-                if (iter != inodeAttrMap.end()) {
-                    dirbuf_add(req, bufHead, dentry, option_,
-                               cacheDir, &iter->second);
-                } else {
-                    LOG(WARNING) << "BatchGetInodeAttr missing some inodes,"
-                                 << " inodeId = " << dentry.inodeid();
-                }
-            }
-        }
-        bufHead->wasRead = true;
-    }
-
-    if (off < bufHead->size) {
-        *buffer = bufHead->p + off;
-        *rSize = std::min(bufHead->size - off, size);
-    } else {
-        *buffer = nullptr;
-        *rSize = 0;
-    }
-    return ret;
+    return rc;
 }
 
 CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
                                        const char *name, fuse_ino_t newparent,
-                                       const char *newname) {
+                                       const char *newname,
+                                       unsigned int flags) {
     VLOG(1) << "FuseOpRename from (" << parent << ", " << name << ") to ("
             << newparent << ", " << newname << ")";
-    if (strlen(name) > option_.maxNameLength ||
-        strlen(newname) > option_.maxNameLength) {
+
+    // TODO(Wine93): the flag RENAME_EXCHANGE and RENAME_NOREPLACE
+    // is only used in linux interface renameat(), not required by posix,
+    // we can ignore it now
+    if (flags != 0) {
+        return CURVEFS_ERROR::INVALIDPARAM;
+    }
+
+    uint64_t maxNameLength = option_.fileSystemOption.maxNameLength;
+    if (strlen(name) > maxNameLength || strlen(newname) > maxNameLength) {
+        LOG(WARNING) << "FuseOpRename name too long, name = " << name
+                     << ", name len = " << strlen(name) << ", new name = "
+                     << newname << ", new name len = " << strlen(newname)
+                     << ", maxNameLength = " << maxNameLength;
         return CURVEFS_ERROR::NAMETOOLONG;
     }
 
@@ -956,26 +696,24 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
     return rc;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
+CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req,
+                                        fuse_ino_t ino,
                                         struct fuse_file_info *fi,
-                                        struct stat *attr) {
-    VLOG(1) << "FuseOpGetAttr ino = " << ino;
-    InodeAttr inodeAttr;
-    CURVEFS_ERROR ret =
-        inodeManager_->GetInodeAttr(ino, &inodeAttr);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
-                   << ", inodeid = " << ino;
-        return ret;
+                                        AttrOut* attrOut) {
+    CURVEFS_ERROR rc = fs_->GetAttr(req, ino, attrOut);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "getattr() fail, retCode = " << rc
+                   << ", ino = " << ino;
     }
-    InodeAttr2ParamAttr(inodeAttr, attr);
-    return ret;
+    return rc;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
-                                        struct stat *attr, int to_set,
-                                        struct fuse_file_info *fi,
-                                        struct stat *attrOut) {
+CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req,
+                                        fuse_ino_t ino,
+                                        struct stat* attr,
+                                        int to_set,
+                                        struct fuse_file_info* fi,
+                                        struct AttrOut* attrOut) {
     VLOG(1) << "FuseOpSetAttr to_set: " << to_set << ", ino: " << ino
             << ", attr: " << *attr;
     std::shared_ptr<InodeWrapper> inodeWrapper;
@@ -1032,9 +770,7 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
         if (ret != CURVEFS_ERROR::OK) {
             return ret;
         }
-        InodeAttr inodeAttr;
-        inodeWrapper->GetInodeAttrLocked(&inodeAttr);
-        InodeAttr2ParamAttr(inodeAttr, attrOut);
+        inodeWrapper->GetInodeAttrLocked(&attrOut->attr);
 
         if (enableSumInDir_ && changeSize != 0) {
             // update parent summary info
@@ -1059,22 +795,19 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-    InodeAttr inodeAttr;
-    inodeWrapper->GetInodeAttrLocked(&inodeAttr);
-    InodeAttr2ParamAttr(inodeAttr, attrOut);
+    inodeWrapper->GetInodeAttrLocked(&attrOut->attr);
     return ret;
 }
 
 CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
-                                         const char* name, void* value,
+                                         const char* name, std::string* value,
                                          size_t size) {
     VLOG(9) << "FuseOpGetXattr, ino: " << ino
             << ", name: " << name << ", size = " << size;
-    if (option_.disableXattr) {
-        return CURVEFS_ERROR::NOTSUPPORT;
+    if (option_.fileSystemOption.disableXattr) {
+        return CURVEFS_ERROR::NOSYS;
     }
 
-    std::string xValue;
     InodeAttr inodeAttr;
     CURVEFS_ERROR ret = inodeManager_->GetInodeAttr(ino, &inodeAttr);
     if (ret != CURVEFS_ERROR::OK) {
@@ -1083,17 +816,20 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
         return ret;
     }
 
-    ret = xattrManager_->GetXattr(name, &xValue, &inodeAttr, enableSumInDir_);
+    ret = xattrManager_->GetXattr(name, value, &inodeAttr, enableSumInDir_);
     if (CURVEFS_ERROR::OK != ret) {
         LOG(ERROR) << "xattrManager get xattr failed, name = " << name;
         return ret;
     }
 
     ret = CURVEFS_ERROR::NODATA;
-    if (xValue.length() > 0) {
-        if ((size == 0 && xValue.length() <= MAXXATTRLENGTH) ||
-            (size >= xValue.length() && xValue.length() <= MAXXATTRLENGTH)) {
-            memcpy(value, xValue.c_str(), xValue.length());
+    if (value->length() > 0) {
+        if ((size == 0 && value->length() <= MAX_XATTR_VALUE_LENGTH) ||
+            (size >= value->length() &&
+                value->length() <= MAX_XATTR_VALUE_LENGTH)) {
+            VLOG(1) << "FuseOpGetXattr name = " << name
+                    << ", length = " << value->length()
+                    << ", value = " << *value;
             ret = CURVEFS_ERROR::OK;
         } else {
             ret = CURVEFS_ERROR::OUT_OF_RANGE;
@@ -1105,15 +841,13 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseClient::FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
                                          const char* name, const char* value,
                                          size_t size, int flags) {
-    VLOG(1) << "FuseOpSetXattr ino: " << ino << ", name: " << name
-            << ", value: " << value;
-    if (option_.disableXattr) {
-        return CURVEFS_ERROR::NOTSUPPORT;
-    }
-
     std::string strname(name);
     std::string strvalue(value, size);
-    if (strname.length() > MAXXATTRLENGTH  || size > MAXXATTRLENGTH) {
+    VLOG(1) << "FuseOpSetXattr ino: " << ino << ", name: " << name
+            << ", size = " << size
+            << ", strvalue: " << strvalue;
+    if (strname.length() > MAX_XATTR_NAME_LENGTH  ||
+        size > MAX_XATTR_VALUE_LENGTH) {
         LOG(ERROR) << "xattr length is too long, name = " << name
                    << ", name length = " << strname.length()
                    << ", value length = " << size;
@@ -1188,10 +922,12 @@ CURVEFS_ERROR FuseClient::FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
     return CURVEFS_ERROR::OUT_OF_RANGE;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, const char *link,
-                                        fuse_ino_t parent, const char *name,
-                                        fuse_entry_param *e) {
-    if (strlen(name) > option_.maxNameLength) {
+CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req,
+                                        const char* link,
+                                        fuse_ino_t parent,
+                                        const char* name,
+                                        EntryOut* entryOut) {
+    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
     const struct fuse_ctx *ctx = fuse_req_ctx(req);
@@ -1260,17 +996,17 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, const char *link,
         }
     }
 
-    InodeAttr attr;
-    inodeWrapper->GetInodeAttr(&attr);
-    GetDentryParamFromInodeAttr(option_, attr, e);
+    inodeWrapper->GetInodeAttr(&entryOut->attr);
     return ret;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char *newname,
+CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     fuse_ino_t newparent,
+                                     const char* newname,
                                      FsFileType type,
-                                     fuse_entry_param *e) {
-    if (strlen(newname) > option_.maxNameLength) {
+                                     EntryOut* entryOut) {
+    if (strlen(newname) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
     std::shared_ptr<InodeWrapper> inodeWrapper;
@@ -1331,9 +1067,7 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
         }
     }
 
-    InodeAttr attr;
-    inodeWrapper->GetInodeAttr(&attr);
-    GetDentryParamFromInodeAttr(option_, attr, e);
+    inodeWrapper->GetInodeAttr(&entryOut->attr);
     return ret;
 }
 
@@ -1351,22 +1085,18 @@ CURVEFS_ERROR FuseClient::FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req, fuse_ino_t ino,
+CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req,
+                                        fuse_ino_t ino,
                                         struct fuse_file_info *fi) {
-    VLOG(1) << "FuseOpRelease, ino: " << ino;
-    if (FLAGS_enableCto) {
-        inodeManager_->RemoveOpenedInode(ino);
+    CURVEFS_ERROR rc = fs_->Release(req, ino);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "release() failed, ino = " << ino;
     }
-    return CURVEFS_ERROR::OK;
+    return rc;
 }
 
-void FuseClient::FlushInode() { inodeManager_->FlushInodeOnce(); }
-
-void FuseClient::FlushInodeAll() { inodeManager_->FlushAll(); }
-
 void FuseClient::FlushAll() {
     FlushData();
-    FlushInodeAll();
 }
 
 }  // namespace client

curvefs/src/client/fuse_client.h

@@ -52,10 +52,11 @@
 #include "curvefs/src/client/client_operator.h"
 #include "curvefs/src/client/lease/lease_excutor.h"
 #include "curvefs/src/client/xattr_manager.h"
+#include "curvefs/src/client/warmup/warmup_manager.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/filesystem.h"
 
 #define DirectIOAlignment 512
-#define WARMUP_CHECKINTERVAL_US 1000*1000
-#define WARMUP_THREADS 10
 
 using ::curve::common::Atomic;
 using ::curve::common::InterruptibleSleeper;
@@ -67,23 +68,27 @@ using ::curvefs::client::metric::FSMetric;
 namespace curvefs {
 namespace client {
 
+
+namespace warmup {
+class WarmupManager;
+}
+
 using common::FuseClientOption;
 using rpcclient::MDSBaseClient;
 using rpcclient::MdsClient;
 using rpcclient::MdsClientImpl;
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
+using ::curvefs::client::filesystem::FileSystem;
+using ::curvefs::client::filesystem::EntryOut;
+using ::curvefs::client::filesystem::AttrOut;
 
 using curvefs::common::is_aligned;
 
 const uint32_t kMaxHostNameLength = 255u;
 
 using mds::Mountpoint;
-typedef struct WarmUpFileContext {
-    fuse_ino_t inode;
-    uint64_t fileLen;
-    bool exist;
-} WarmUpFileContext_t;
+
 class FuseClient {
  public:
     FuseClient()
@@ -91,31 +96,30 @@ class FuseClient {
         metaClient_(std::make_shared<MetaServerClientImpl>()),
         inodeManager_(std::make_shared<InodeCacheManagerImpl>(metaClient_)),
         dentryManager_(std::make_shared<DentryCacheManagerImpl>(metaClient_)),
-        dirBuf_(std::make_shared<DirBuffer>()),
         fsInfo_(nullptr),
         mdsBase_(nullptr),
         isStop_(true),
         init_(false),
-        mounted_(false),
-        enableSumInDir_(false) {}
+        enableSumInDir_(false),
+        warmupManager_(nullptr) {}
 
     virtual ~FuseClient() {}
 
     FuseClient(const std::shared_ptr<MdsClient> &mdsClient,
         const std::shared_ptr<MetaServerClient> &metaClient,
         const std::shared_ptr<InodeCacheManager> &inodeManager,
-        const std::shared_ptr<DentryCacheManager> &dentryManager)
+        const std::shared_ptr<DentryCacheManager> &dentryManager,
+        const std::shared_ptr<warmup::WarmupManager> &warmupManager)
           : mdsClient_(mdsClient),
             metaClient_(metaClient),
             inodeManager_(inodeManager),
             dentryManager_(dentryManager),
-            dirBuf_(std::make_shared<DirBuffer>()),
             fsInfo_(nullptr),
             mdsBase_(nullptr),
             isStop_(true),
             init_(false),
-            mounted_(false),
-            enableSumInDir_(false) {}
+            enableSumInDir_(false),
+            warmupManager_(warmupManager) {}
 
     virtual CURVEFS_ERROR Init(const FuseClientOption &option);
 
@@ -140,24 +144,34 @@ class FuseClient {
                                      struct fuse_file_info* fi, char* buffer,
                                      size_t* rSize) = 0;
 
-    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       EntryOut* entryOut);
 
-    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req,
+                                     fuse_ino_t ino,
                                      struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, mode_t mode,
+    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       mode_t mode,
                                        struct fuse_file_info* fi,
-                                       fuse_entry_param* e) = 0;
+                                       EntryOut* entryOut) = 0;
 
-    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-                                      const char* name, mode_t mode, dev_t rdev,
-                                      fuse_entry_param* e) = 0;
+    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
+                                      fuse_ino_t parent,
+                                      const char* name,
+                                      mode_t mode,
+                                      dev_t rdev,
+                                      EntryOut* entryOut) = 0;
 
-    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req, fuse_ino_t parent,
-                                      const char* name, mode_t mode,
-                                      fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req,
+                                      fuse_ino_t parent,
+                                      const char* name,
+                                      mode_t mode,
+                                      EntryOut* entryOut);
 
     virtual CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                        const char* name) = 0;
@@ -165,33 +179,41 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
                                       const char* name);
 
-    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req,
+                                        fuse_ino_t ino,
                                         struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req,
+                                           fuse_ino_t ino,
                                            struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
-                                            size_t size, off_t off,
-                                            struct fuse_file_info* fi,
-                                            char** buffer, size_t* rSize,
-                                            bool cacheDir);
+    virtual CURVEFS_ERROR FuseOpReadDir(fuse_req_t req,
+                                        fuse_ino_t ino,
+                                        size_t size,
+                                        off_t off,
+                                        struct fuse_file_info* fi,
+                                        char** bufferOut,
+                                        size_t* rSize,
+                                        bool plus);
 
-    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, fuse_ino_t newparent,
-                                       const char* newname);
+    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       fuse_ino_t newparent,
+                                       const char* newname,
+                                       unsigned int flags);
 
     virtual CURVEFS_ERROR FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info* fi,
-                                        struct stat* attr);
+                                        struct AttrOut* out);
 
     virtual CURVEFS_ERROR FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct stat* attr, int to_set,
                                         struct fuse_file_info* fi,
-                                        struct stat* attrOut);
+                                        struct AttrOut* out);
 
     virtual CURVEFS_ERROR FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
-                                         const char* name, void* value,
+                                         const char* name, std::string* value,
                                          size_t size);
 
     virtual CURVEFS_ERROR FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
@@ -201,13 +223,17 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                             char *value, size_t size, size_t *realSize);
 
-    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req, const char* link,
-                                        fuse_ino_t parent, const char* name,
-                                        fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req,
+                                        const char* link,
+                                        fuse_ino_t parent,
+                                        const char* name,
+                                        EntryOut* entryOut);
 
-    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char* newname,
-                                     fuse_entry_param* e) = 0;
+    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     fuse_ino_t newparent,
+                                     const char* newname,
+                                     EntryOut* entryOut) = 0;
 
     virtual CURVEFS_ERROR FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
                                          std::string* linkStr);
@@ -236,31 +262,30 @@ class FuseClient {
         stbuf->f_fsid = fsInfo_->fsid();
 
         stbuf->f_flag = 0;
-        stbuf->f_namemax = option_.maxNameLength;
+        stbuf->f_namemax = option_.fileSystemOption.maxNameLength;
         return CURVEFS_ERROR::OK;
     }
 
+    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
+
     void SetFsInfo(const std::shared_ptr<FsInfo>& fsInfo) {
         fsInfo_ = fsInfo;
         init_ = true;
     }
 
     void SetMounted(bool mounted) {
-        mounted_ = mounted;
-    }
-
-    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable> &
-      GetTaskFetchPool() {
-        return taskFetchMetaPool_;
+        if (warmupManager_ != nullptr) {
+            warmupManager_->SetMounted(mounted);
+        }
     }
 
     std::shared_ptr<FsInfo> GetFsInfo() {
         return fsInfo_;
     }
 
-    virtual void FlushInode();
-
-    virtual void FlushInodeAll();
+    std::shared_ptr<FileSystem> GetFileSystem() {
+        return fs_;
+    }
 
     virtual void FlushAll();
 
@@ -268,47 +293,55 @@ class FuseClient {
     void SetEnableSumInDir(bool enable) {
         enableSumInDir_ = enable;
     }
-    std::list<fuse_ino_t>& GetReadAheadFiles() {
-        std::unique_lock<std::mutex> lck(fetchMtx_);
-        return readAheadFiles_;
+
+    bool EnableXattrSumming() {
+        return enableSumInDir_;
     }
 
-    void GetWarmUpFile(WarmUpFileContext_t* warmUpFile) {
-        std::unique_lock<std::mutex> lck(warmUpFileMtx_);
-        *warmUpFile = std::move(warmUpFile_);
-        warmUpFile_.exist = false;
-        return;
-    }
-    void SetWarmUpFile(WarmUpFileContext_t warmUpFile) {
-        std::unique_lock<std::mutex> lck(warmUpFileMtx_);
-        warmUpFile_ = warmUpFile;
-        warmUpFile_.exist = true;
-    }
-    bool hasWarmUpTask() {
-        std::unique_lock<std::mutex> lck(warmUpFileMtx_);
-        return warmUpFile_.exist;
+    bool PutWarmFilelistTask(fuse_ino_t key) {
+        if (fsInfo_->fstype() == FSType::TYPE_S3) {
+            return warmupManager_->AddWarmupFilelist(key);
+        }  // only support s3
+        return true;
     }
 
-    void FetchDentryEnqueue(std::string file);
+    bool PutWarmFileTask(fuse_ino_t key, const std::string& path) {
+        if (fsInfo_->fstype() == FSType::TYPE_S3) {
+            return warmupManager_->AddWarmupFile(key, path);
+        }  // only support s3
+        return true;
+    }
 
-    void PutWarmTask(const std::string& warmUpTask) {
-        std::unique_lock<std::mutex> lck(warmUpTaskMtx_);
-        warmUpTasks_.push_back(warmUpTask);
-        WarmUpRun();
+    bool GetWarmupProgress(fuse_ino_t key, warmup::WarmupProgress *progress) {
+        if (fsInfo_->fstype() == FSType::TYPE_S3) {
+            return warmupManager_->QueryWarmupProgress(key, progress);
+        }
+        return false;
     }
 
  protected:
-    CURVEFS_ERROR MakeNode(fuse_req_t req, fuse_ino_t parent, const char* name,
-                           mode_t mode, FsFileType type, dev_t rdev,
-                           fuse_entry_param* e);
+    CURVEFS_ERROR MakeNode(fuse_req_t req,
+                           fuse_ino_t parent,
+                           const char* name,
+                           mode_t mode,
+                           FsFileType type,
+                           dev_t rdev,
+                           bool internal,
+                           std::shared_ptr<InodeWrapper>& InodeWrapper);
 
     CURVEFS_ERROR RemoveNode(fuse_req_t req, fuse_ino_t parent,
                              const char* name, FsFileType type);
 
-    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                             fuse_ino_t newparent, const char* newname,
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                             fuse_ino_t ino,
+                             fuse_ino_t newparent,
+                             const char* newname,
                              FsFileType type,
-                             fuse_entry_param* e);
+                             EntryOut* entryOut);
+
+    CURVEFS_ERROR HandleOpenFlags(fuse_req_t req,
+                                  fuse_ino_t ino,
+                                  struct fuse_file_info* fi);
 
     int SetHostPortInMountPoint(Mountpoint* out) {
         char hostname[kMaxHostNameLength];
@@ -323,54 +356,24 @@ class FuseClient {
         return 0;
     }
 
-    void splitStr(const std::string& srcStr, const std::string& delimiter,
-      std::vector<std::string>* splitPath) {
-        char* pToken = nullptr;
-        char* pSave = nullptr;
-        pToken = strtok_r(const_cast<char*>(srcStr.c_str()),
-          const_cast<char*>(delimiter.c_str()), &pSave);
-        if (nullptr == pToken) {
-            VLOG(6) << "del lookpath end";
-            return;
-        }
-        splitPath->push_back(pToken);
-        while (true) {
-            pToken = strtok_r(NULL, const_cast<char*>(
-              delimiter.c_str()), &pSave);
-            if (nullptr == pToken) {
-                VLOG(6) << "del lookpath end";
-                break;
-            }
-            VLOG(9) << "del pToken is:" << pToken
-                    << "pSave:" << pSave;
-            splitPath->push_back(pToken);
-        }
-    }
 
  private:
-    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
-
     virtual void FlushData() = 0;
 
     CURVEFS_ERROR UpdateParentMCTimeAndNlink(
         fuse_ino_t parent, FsFileType type,  NlinkChange nlink);
 
-    void WarmUpTask();
+    std::string GenerateNewRecycleName(fuse_ino_t ino,
+            fuse_ino_t parent, const char* name) {
+        std::string newName(name);
+        newName = std::to_string(parent) + "_" + std::to_string(ino)
+                    + "_" + newName;
+        if (newName.length() > option_.fileSystemOption.maxNameLength) {
+            newName = newName.substr(0, option_.fileSystemOption.maxNameLength);
+        }
 
-    void WarmUpRun() {
-        std::lock_guard<std::mutex> lk(mtx_);
-        runned_ = true;
-        cond_.notify_one();
+        return newName;
     }
-    void WaitWarmUp() {
-        std::unique_lock<std::mutex> lk(mtx_);
-        cond_.wait(lk, [this]() { return runned_; });
-        runned_ = false;
-    }
-    std::mutex mtx_;
-    std::condition_variable cond_;
-    bool runned_ = false;
-
 
  protected:
     // mds client
@@ -390,9 +393,6 @@ class FuseClient {
 
     std::shared_ptr<LeaseExecutor> leaseExecutor_;
 
-    // dir buffer
-    std::shared_ptr<DirBuffer> dirBuf_;
-
     // filesystem info
     std::shared_ptr<FsInfo> fsInfo_;
 
@@ -401,8 +401,6 @@ class FuseClient {
     // init flags
     bool init_;
 
-    std::atomic<bool>  mounted_;
-
     // enable record summary info in dir inode xattr
     bool enableSumInDir_;
 
@@ -410,46 +408,17 @@ class FuseClient {
 
     Mountpoint mountpoint_;
 
+    // warmup manager
+    std::shared_ptr<warmup::WarmupManager> warmupManager_;
+
+    std::shared_ptr<FileSystem> fs_;
+
  private:
     MDSBaseClient* mdsBase_;
 
     Atomic<bool> isStop_;
 
     curve::common::Mutex renameMutex_;
-
-    Thread bgCmdTaskThread_;
-    std::atomic<bool> bgCmdStop_;
-    std::mutex cmdMtx_;
-
-    void FetchChildDentryEnqueue(fuse_ino_t  ino);
-    void FetchChildDentry(fuse_ino_t ino);
-    void FetchDentry(fuse_ino_t ino, std::string file);
-    void LookPath(std::string file);
-    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
-        taskFetchMetaPool_;
-
-    // need warmup files
-    std::list<fuse_ino_t> readAheadFiles_;
-    std::mutex fetchMtx_;
-
-    //  one warmup file provided by the user
-    WarmUpFileContext_t warmUpFile_;
-    std::mutex warmUpFileMtx_;
-
-    std::list<std::string> warmUpTasks_;  // todo: need size control ?
-    std::mutex warmUpTaskMtx_;
-
-    void GetwarmTask(std::string *warmUpTask) {
-        std::unique_lock<std::mutex> lck(warmUpTaskMtx_);
-        if (warmUpTasks_.empty())
-            return;
-        *warmUpTask = std::move(warmUpTasks_.front());
-        warmUpTasks_.pop_front();
-    }
-    bool hasWarmTask() {
-        std::unique_lock<std::mutex> lck(warmUpTaskMtx_);
-        return !warmUpTasks_.empty();
-    }
 };
 
 }  // namespace client

curvefs/src/client/fuse_s3_client.cpp

@@ -48,7 +48,6 @@ using curvefs::mds::topology::MemcacheServerInfo;
 
 CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
     FuseClientOption opt(option);
-    initbgFetchThread_ = false;
 
     CURVEFS_ERROR ret = FuseClient::Init(opt);
     if (ret != CURVEFS_ERROR::OK) {
@@ -71,7 +70,7 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
     auto fsCacheManager = std::make_shared<FsCacheManager>(
         dynamic_cast<S3ClientAdaptorImpl *>(s3Adaptor_.get()),
         opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte,
-        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte);
+        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte, kvClientManager_);
     if (opt.s3Opt.s3ClientAdaptorOpt.diskCacheOpt.diskCacheType !=
         DiskCacheType::Disable) {
         auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
@@ -85,17 +84,12 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
             diskCacheManager, s3DiskCacheClient);
         ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
                                inodeManager_, mdsClient_, fsCacheManager,
-                               diskCacheManagerImpl, true);
+                               diskCacheManagerImpl, kvClientManager_, true);
     } else {
         ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
                                inodeManager_, mdsClient_, fsCacheManager,
-                               nullptr, true);
+                               nullptr, kvClientManager_, true);
     }
-
-    bgFetchStop_.store(false, std::memory_order_release);
-    bgFetchThread_ = Thread(&FuseS3Client::BackGroundFetch, this);
-    initbgFetchThread_ = true;
-    GetTaskFetchPool();
     return ret;
 }
 
@@ -118,264 +112,21 @@ bool FuseS3Client::InitKVCache(const KVClientManagerOpt &opt) {
         return false;
     }
 
-    g_kvClientManager = new KVClientManager();
-    if (!g_kvClientManager->Init(opt, memcacheClient)) {
+    kvClientManager_ = std::make_shared<KVClientManager>();
+    if (!kvClientManager_->Init(opt, memcacheClient)) {
         LOG(ERROR) << "FLAGS_supportKVcache = " << FLAGS_supportKVcache
                    << ", but init kvClientManager fail";
         return false;
     }
 
+    if (warmupManager_ != nullptr) {
+        warmupManager_->SetKVClientManager(kvClientManager_);
+    }
+
     return true;
 }
 
-void FuseS3Client::GetWarmUpFileList(const WarmUpFileContext_t&warmUpFile,
-  std::vector<std::string>& warmUpFilelist) {
-        struct fuse_file_info fi{};
-        fi.flags &= ~O_DIRECT;
-        size_t rSize = 0;
-        std::unique_ptr<char[]> data(new char[warmUpFile.fileLen+1]);
-        std::memset(data.get(), 0, warmUpFile.fileLen);
-        data[warmUpFile.fileLen] = '\n';
-        FuseOpRead(nullptr, warmUpFile.inode,
-            warmUpFile.fileLen, 0, &fi, data.get(), &rSize);
-        std::string file = data.get();
-        VLOG(9) << "file is: " << file;
-        // remove enter, newline, blank
-        std::string blanks("\r\n ");
-        file.erase(0, file.find_first_not_of(blanks));
-        file.erase(file.find_last_not_of(blanks) + 1);
-        VLOG(9) << "after del file is: " << file;
-        splitStr(file, "\n", &warmUpFilelist);
-}
-
-void FuseS3Client::BackGroundFetch() {
-    while (!bgFetchStop_.load(std::memory_order_acquire)) {
-        LOG_EVERY_N(WARNING, 100)
-            << "fetch thread start.";
-        if (hasWarmUpTask()) {  // new warmup task
-            WarmUpFileContext_t warmUpFile;
-            GetWarmUpFile(&warmUpFile);
-            VLOG(9) << " len is: " << warmUpFile.fileLen
-                << "ino is: " << warmUpFile.inode;
-
-            std::vector<std::string> warmUpFilelist;
-            GetWarmUpFileList(warmUpFile, warmUpFilelist);
-            for (auto filePath : warmUpFilelist) {
-                FetchDentryEnqueue(filePath);
-            }
-        }
-        {   // file need warmup
-            std::list<fuse_ino_t> readAheadFiles;
-            readAheadFiles.swap(GetReadAheadFiles());
-            for (auto iter : readAheadFiles) {
-                VLOG(9) << "BackGroundFetch: " << iter;
-                fetchDataEnqueue(iter);
-            }
-        }
-        LOG_EVERY_N(WARNING, 100)
-            << "fetch thread end.";
-        usleep(WARMUP_CHECKINTERVAL_US);
-    }
-    return;
-}
-
-void FuseS3Client::fetchDataEnqueue(fuse_ino_t ino) {
-    VLOG(9) << "fetchDataEnqueue start: " << ino;
-    auto task = [this, ino]() {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
-        if (ret != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
-                    << ", inodeid = " << ino;
-            return;
-        }
-        google::protobuf::Map<uint64_t, S3ChunkInfoList> *s3ChunkInfoMap
-          = nullptr;
-        {
-            ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-            s3ChunkInfoMap = inodeWrapper->GetChunkInfoMap();
-        }
-        if (nullptr == s3ChunkInfoMap ||
-          s3ChunkInfoMap->empty()) {
-            return;
-        }
-        travelChunks(ino, s3ChunkInfoMap);
-    };
-    GetTaskFetchPool().Enqueue(task);
-}
-
-// travel and download all objs belong to the chunk
-void FuseS3Client::travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo,
-  std::list<std::pair<std::string, uint64_t>>* prefetchObjs) {
-    uint64_t blockSize = s3Adaptor_->GetBlockSize();
-    uint64_t chunkSize = s3Adaptor_->GetChunkSize();
-    uint64_t offset, len, chunkid, compaction;
-    for (size_t i = 0; i < chunkInfo.s3chunks_size(); i++) {
-        auto chunkinfo = chunkInfo.mutable_s3chunks(i);
-        auto fsId = fsInfo_->fsid();
-        chunkid = chunkinfo->chunkid();
-        compaction = chunkinfo->compaction();
-        offset = chunkinfo->offset();
-        len = chunkinfo->len();
-        // the offset in the chunk
-        uint64_t chunkPos = offset % chunkSize;
-        // the offset in the block
-        uint64_t blockPos = chunkPos % blockSize;
-        // the first blockIndex
-        uint64_t blockIndexBegin = chunkPos / blockSize;
-
-        if (len < blockSize) {  // just one block
-            auto objectName = curvefs::common::s3util::GenObjName(
-                chunkid, blockIndexBegin, compaction, fsId, ino);
-            prefetchObjs->push_back(std::make_pair(objectName, len));
-        } else {
-            // the offset in the block
-            uint64_t blockPos = chunkPos % blockSize;
-
-            // firstly, let's get the size in the first block
-            // then, subtract the length in the first block
-            // to obtain the remaining length
-            // lastly, We need to judge the last block is full or not
-            uint64_t firstBlockSize = (blockPos != 0) ?
-              blockSize - blockPos : blockSize;
-            uint64_t leftSize = len - firstBlockSize;
-            uint32_t blockCounts = (leftSize % blockSize == 0) ?
-              (leftSize / blockSize + 1) : (leftSize / blockSize + 1 + 1);
-            // so we can get the last blockIndex
-            // because the bolck Index is cumulative
-            uint64_t blockIndexEnd = blockIndexBegin + blockCounts - 1;
-
-            // the size of the last block
-            uint64_t lastBlockSize = leftSize % blockSize;
-            // whether the first block or the last block is full or not
-            bool firstBlockFull = (blockPos == 0) ? true : false;
-            bool lastBlockFull = (lastBlockSize == 0) ? true : false;
-            // the start and end block Index that need travel
-            uint64_t travelStartIndex, travelEndIndex;
-            // if the block is full, the size is needed download
-            // of the obj is blockSize. Otherwise, the value is special.
-            if (!firstBlockFull) {
-                travelStartIndex = blockIndexBegin + 1;
-                auto objectName = curvefs::common::s3util::GenObjName(
-                  chunkid, blockIndexBegin, compaction, fsId, ino);
-                prefetchObjs->push_back(std::make_pair(
-                  objectName, firstBlockSize));
-            } else {
-                travelStartIndex = blockIndexBegin;
-            }
-            if (!lastBlockFull) {
-                // block index is greater than or equal to 0
-                travelEndIndex = (blockIndexEnd == blockIndexBegin) ?
-                  blockIndexEnd : blockIndexEnd - 1;
-                auto objectName = curvefs::common::s3util::GenObjName(
-                  chunkid, blockIndexEnd, compaction, fsId, ino);
-                // there is no need to care about the order
-                // in which objects are downloaded
-                prefetchObjs->push_back(
-                  std::make_pair(objectName, lastBlockSize));
-            } else {
-                travelEndIndex = blockIndexEnd;
-            }
-            VLOG(9) << "travel obj, ino: " << ino
-                << ", chunkid: " << chunkid
-                << ", blockCounts: " << blockCounts
-                << ", compaction: " << compaction
-                << ", blockSize: " << blockSize
-                << ", chunkSize: " << chunkSize
-                << ", offset: "  << offset
-                << ", blockIndexBegin: " << blockIndexBegin
-                << ", blockIndexEnd: " << blockIndexEnd
-                << ", len: " << len
-                << ", firstBlockSize: " << firstBlockSize
-                << ", lastBlockSize: " << lastBlockSize
-                << ", blockPos: " << blockPos
-                << ", chunkPos: " << chunkPos;
-            for (auto blockIndex = travelStartIndex;
-              blockIndex <= travelEndIndex ; blockIndex++) {
-                auto objectName = curvefs::common::s3util::GenObjName(
-                    chunkid, blockIndex, compaction, fsId, ino);
-                prefetchObjs->push_back(std::make_pair(objectName, blockSize));
-            }
-        }
-    }
-}
-
-// TODO(hzwuhongsong): These logics are very similar to other place,
-// try to merge it
-void FuseS3Client::WarmUpAllObjs(
-  const std::list<std::pair<std::string, uint64_t>> &prefetchObjs) {
-    std::atomic<uint64_t> pendingReq(0);
-    curve::common::CountDownEvent cond(1);
-    // callback function
-    GetObjectAsyncCallBack cb =
-        [&](const S3Adapter *adapter,
-            const std::shared_ptr<GetObjectAsyncContext> &context) {
-            if (context->retCode == 0) {
-                VLOG(9) << "Get Object success: " << context->key;
-                int ret = s3Adaptor_->GetDiskCacheManager()->WriteReadDirect(
-                  context->key, context->buf, context->len);
-                if (ret < 0) {
-                    LOG_EVERY_SECOND(INFO) <<
-                    "write read directly failed, key: " << context->key;
-                }
-                if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == 1) {
-                    VLOG(6) << "pendingReq is over";
-                    cond.Signal();
-                }
-                delete []context->buf;
-                return;
-            }
-            // todo: retry
-            LOG(WARNING) << "Get Object failed, key: " << context->key
-                         << ", offset: " << context->offset;
-            s3Adaptor_->GetS3Client()->DownloadAsync(context);
-    };
-
-    pendingReq.fetch_add(prefetchObjs.size(), std::memory_order_seq_cst);
-    if (pendingReq.load(std::memory_order_seq_cst)) {
-        VLOG(9) << "wait for pendingReq";
-        for (auto iter : prefetchObjs) {
-            VLOG(9) << "download start: " << iter.first;
-            std::string name = iter.first;
-            uint64_t readLen = iter.second;
-            if (s3Adaptor_->GetDiskCacheManager()->IsCached(name)) {
-                pendingReq.fetch_sub(1);
-                continue;
-            }
-            char *cacheS3 = new char[readLen];
-            memset(cacheS3, 0, readLen);
-            auto context = std::make_shared<GetObjectAsyncContext>();
-            context->key = name;
-            context->buf = cacheS3;
-            context->offset = 0;
-            context->len = readLen;
-            context->cb = cb;
-            s3Adaptor_->GetS3Client()->DownloadAsync(context);
-        }
-        if (pendingReq.load())
-            cond.Wait();
-    }
-}
-
-void FuseS3Client::travelChunks(fuse_ino_t ino, google::protobuf::Map<uint64_t,
-    S3ChunkInfoList> *s3ChunkInfoMap) {
-    VLOG(9) << "travel chunk start: " << ino
-            << ", size: " << s3ChunkInfoMap->size();
-    std::list<std::pair<std::string, uint64_t>> prefetchObjs;
-    for (auto &iter : *s3ChunkInfoMap) {
-        VLOG(9) << "travel chunk: " << iter.first;
-        travelChunk(ino, iter.second, &prefetchObjs);
-    }
-    WarmUpAllObjs(prefetchObjs);
-    VLOG(9) << "travel chunks end";
-    return;
-}
-
 void FuseS3Client::UnInit() {
-    bgFetchStop_.store(true, std::memory_order_release);
-    if (initbgFetchThread_) {
-        bgFetchThread_.join();
-    }
     FuseClient::UnInit();
     s3Adaptor_->Stop();
     curve::common::S3Adapter::Shutdown();
@@ -517,32 +268,56 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseS3Client::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                          const char *name, mode_t mode,
                                          struct fuse_file_info *fi,
-                                         fuse_entry_param *e) {
+                                         EntryOut* entryOut) {
     VLOG(1) << "FuseOpCreate, parent: " << parent << ", name: " << name
             << ", mode: " << mode;
+
+    std::shared_ptr<InodeWrapper> inode;
     CURVEFS_ERROR ret =
-        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, e);
+        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, false, inode);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-    return FuseOpOpen(req, e->ino, fi);
+
+    auto openFiles = fs_->BorrowMember().openFiles;
+    openFiles->Open(inode->GetInodeId(), inode);
+
+    inode->GetInodeAttr(&entryOut->attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-                                        const char *name, mode_t mode,
-                                        dev_t rdev, fuse_entry_param *e) {
+CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req,
+                                        fuse_ino_t parent,
+                                        const char* name,
+                                        mode_t mode,
+                                        dev_t rdev,
+                                        EntryOut* entryOut) {
     VLOG(1) << "FuseOpMkNod, parent: " << parent << ", name: " << name
             << ", mode: " << mode << ", rdev: " << rdev;
-    return MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, rdev, e);
+
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR rc = MakeNode(req, parent, name, mode,
+                                FsFileType::TYPE_S3, rdev, false,
+                                inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    InodeAttr attr;
+    inode->GetInodeAttr(&attr);
+    *entryOut = EntryOut(attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char *newname,
-                                     fuse_entry_param *e) {
+CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req,
+                                       fuse_ino_t ino,
+                                       fuse_ino_t newparent,
+                                       const char* newname,
+                                       EntryOut* entryOut) {
     VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
             << ", newname: " << newname;
     return FuseClient::FuseOpLink(
-        req, ino, newparent, newname, FsFileType::TYPE_S3, e);
+        req, ino, newparent, newname, FsFileType::TYPE_S3, entryOut);
 }
 
 CURVEFS_ERROR FuseS3Client::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,

curvefs/src/client/fuse_s3_client.h

@@ -32,6 +32,7 @@
 
 #include "curvefs/src/client/fuse_client.h"
 #include "curvefs/src/client/s3/client_s3_cache_manager.h"
+#include "curvefs/src/client/warmup/warmup_manager.h"
 #include "src/common/s3_adapter.h"
 
 namespace curvefs {
@@ -39,18 +40,34 @@ namespace client {
 
 using curve::common::GetObjectAsyncContext;
 using curve::common::GetObjectAsyncCallBack;
+namespace warmup {
+class WarmupManager;
+class WarmupManagerS3Impl;
+}  // namespace warmup
+
 
 class FuseS3Client : public FuseClient {
  public:
     FuseS3Client()
-        : FuseClient(), s3Adaptor_(std::make_shared<S3ClientAdaptorImpl>()) {}
+        : FuseClient(), s3Adaptor_(std::make_shared<S3ClientAdaptorImpl>()) {
+        auto readFunc = [this](fuse_req_t req, fuse_ino_t ino, size_t size,
+                               off_t off, struct fuse_file_info *fi,
+                               char *buffer, size_t *rSize) {
+            return FuseOpRead(req, ino, size, off, fi, buffer, rSize);
+        };
+        warmupManager_ = std::make_shared<warmup::WarmupManagerS3Impl>(
+            metaClient_, inodeManager_, dentryManager_, fsInfo_, readFunc,
+            s3Adaptor_, nullptr);
+    }
 
     FuseS3Client(const std::shared_ptr<MdsClient> &mdsClient,
                  const std::shared_ptr<MetaServerClient> &metaClient,
                  const std::shared_ptr<InodeCacheManager> &inodeManager,
                  const std::shared_ptr<DentryCacheManager> &dentryManager,
-                 const std::shared_ptr<S3ClientAdaptor> &s3Adaptor)
-        : FuseClient(mdsClient, metaClient, inodeManager, dentryManager),
+                 const std::shared_ptr<S3ClientAdaptor> &s3Adaptor,
+                 const std::shared_ptr<warmup::WarmupManager> &warmupManager)
+        : FuseClient(mdsClient, metaClient, inodeManager, dentryManager,
+                     warmupManager),
           s3Adaptor_(s3Adaptor) {}
 
     CURVEFS_ERROR Init(const FuseClientOption &option) override;
@@ -70,17 +87,25 @@ class FuseS3Client : public FuseClient {
         char *buffer,
         size_t *rSize) override;
 
-    CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, struct fuse_file_info *fi,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
+                               fuse_ino_t parent,
+                               const char* name,
+                               mode_t mode,
+                               struct fuse_file_info *fi,
+                               EntryOut* entryOut) override;
 
-    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, dev_t rdev,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
+                              fuse_ino_t parent,
+                              const char* name,
+                              mode_t mode,
+                              dev_t rdev,
+                              EntryOut* entryOut) override;
 
-    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-        fuse_ino_t newparent, const char *newname,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                             fuse_ino_t ino,
+                             fuse_ino_t newparent,
+                             const char *newname,
+                             EntryOut* entryOut) override;
 
     CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
         const char *name) override;
@@ -91,37 +116,17 @@ class FuseS3Client : public FuseClient {
     CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
         struct fuse_file_info *fi) override;
 
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
+
  private:
     bool InitKVCache(const KVClientManagerOpt &opt);
 
-    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
-
     void FlushData() override;
-    // get the warmUp filelist
-    void GetWarmUpFileList(const WarmUpFileContext_t&,
-      std::vector<std::string>&);
-    void BackGroundFetch();
-    // put the file needed warmup to queue,
-    // then can downlaod the objs belong to it
-    void fetchDataEnqueue(fuse_ino_t ino);
-    // travel all chunks
-    void travelChunks(fuse_ino_t ino, google::protobuf::Map<uint64_t,
-      S3ChunkInfoList> *s3ChunkInfoMap);
-    // travel and download all objs belong to the chunk
-    void travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo,
-      std::list<std::pair<std::string, uint64_t>>* prefetchObjs);
-    // warmup all the prefetchObjs
-    void WarmUpAllObjs(const std::list<
-      std::pair<std::string, uint64_t>> &prefetchObjs);
 
  private:
     // s3 adaptor
     std::shared_ptr<S3ClientAdaptor> s3Adaptor_;
-
-    bool initbgFetchThread_;
-    Thread bgFetchThread_;
-    std::atomic<bool> bgFetchStop_;
-    std::mutex fetchMtx_;
+    std::shared_ptr<KVClientManager> kvClientManager_;
 };
 
 

curvefs/src/client/fuse_volume_client.cpp

@@ -32,7 +32,7 @@
 #include "absl/cleanup/cleanup.h"
 #include "absl/memory/memory.h"
 #include "curvefs/proto/mds.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/volume/default_volume_storage.h"
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/src/volume/common.h"
@@ -233,33 +233,57 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req,
 CURVEFS_ERROR FuseVolumeClient::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                              const char *name, mode_t mode,
                                              struct fuse_file_info *fi,
-                                             fuse_entry_param *e) {
+                                             EntryOut* entryOut) {
     VLOG(3) << "FuseOpCreate, parent: " << parent
               << ", name: " << name
               << ", mode: " << mode;
-    CURVEFS_ERROR ret =
-        MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, 0, e);
+
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR ret = MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, 0, false,
+                                 inode);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-    return FuseOpOpen(req, e->ino, fi);
+
+    auto openFiles = fs_->BorrowMember().openFiles;
+    openFiles->Open(inode->GetInodeId(), inode);
+
+    inode->GetInodeAttr(&entryOut->attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-                                            const char *name, mode_t mode,
-                                            dev_t rdev, fuse_entry_param *e) {
+CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req,
+                                            fuse_ino_t parent,
+                                            const char* name,
+                                            mode_t mode,
+                                            dev_t rdev,
+                                            EntryOut* entryOut) {
     VLOG(3) << "FuseOpMkNod, parent: " << parent << ", name: " << name
             << ", mode: " << mode << ", rdev: " << rdev;
-    return MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, rdev, e);
+
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR rc = MakeNode(req, parent, name, mode,
+                                FsFileType::TYPE_FILE, rdev,
+                                false, inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    InodeAttr attr;
+    inode->GetInodeAttr(&attr);
+    *entryOut = EntryOut(attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseVolumeClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char *newname,
-                                     fuse_entry_param *e) {
+CURVEFS_ERROR FuseVolumeClient::FuseOpLink(fuse_req_t req,
+                                           fuse_ino_t ino,
+                                           fuse_ino_t newparent,
+                                           const char* newname,
+                                           EntryOut* entryOut) {
     VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
             << ", newname: " << newname;
     return FuseClient::FuseOpLink(
-        req, ino, newparent, newname, FsFileType::TYPE_FILE, e);
+        req, ino, newparent, newname, FsFileType::TYPE_FILE, entryOut);
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,

curvefs/src/client/fuse_volume_client.h

@@ -52,7 +52,8 @@ class FuseVolumeClient : public FuseClient {
         const std::shared_ptr<InodeCacheManager> &inodeManager,
         const std::shared_ptr<DentryCacheManager> &dentryManager,
         const std::shared_ptr<BlockDeviceClient> &blockDeviceClient)
-        : FuseClient(mdsClient, metaClient, inodeManager, dentryManager),
+        : FuseClient(mdsClient, metaClient, inodeManager, dentryManager,
+                     nullptr),
           blockDeviceClient_(blockDeviceClient) {}
 
     CURVEFS_ERROR Init(const FuseClientOption &option) override;
@@ -71,17 +72,25 @@ class FuseVolumeClient : public FuseClient {
             struct fuse_file_info *fi,
             char *buffer,
             size_t *rSize) override;
-    CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, struct fuse_file_info *fi,
-        fuse_entry_param *e) override;
 
-    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, dev_t rdev,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
+                               fuse_ino_t parent,
+                               const char* name,
+                               mode_t mode,
+                               struct fuse_file_info* fi,
+                               EntryOut* entryOut) override;
+
+    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
+                              fuse_ino_t parent,
+                              const char* name,
+                              mode_t mode,
+                              dev_t rdev,
+                              EntryOut* entryOut) override;
 
     CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-        fuse_ino_t newparent, const char *newname,
-        fuse_entry_param *e) override;
+                             fuse_ino_t newparent,
+                             const char *newname,
+                             EntryOut* entryOut) override;
 
     CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                const char *name) override;
@@ -92,13 +101,13 @@ class FuseVolumeClient : public FuseClient {
     CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                               struct fuse_file_info *fi) override;
 
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
+
     void SetSpaceManagerForTesting(SpaceManager *manager);
 
     void SetVolumeStorageForTesting(VolumeStorage *storage);
 
  private:
-    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
-
     void FlushData() override;
 
  private:

curvefs/src/client/inode_cache_manager.cpp

@@ -29,7 +29,7 @@
 #include <memory>
 #include <utility>
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/inode_wrapper.h"
 
 using ::curvefs::metaserver::Inode;
@@ -46,36 +46,11 @@ DECLARE_bool(enableCto);
 namespace curvefs {
 namespace client {
 
+using ::curvefs::client::filesystem::ToFSError;
+
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
 using curvefs::client::common::FLAGS_enableCto;
 
-class TrimICacheAsyncDone : public MetaServerClientDone {
- public:
-    explicit TrimICacheAsyncDone(
-        const std::shared_ptr<InodeWrapper> &inodeWrapper,
-        const std::shared_ptr<InodeCacheManagerImpl> &inodeCacheManager)
-        : inodeWrapper_(inodeWrapper), inodeCacheManager_(inodeCacheManager) {}
-
-    void Run() override {
-        std::unique_ptr<TrimICacheAsyncDone> self_guard(this);
-        MetaStatusCode ret = GetStatusCode();
-        if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
-            LOG(ERROR) << "metaClient_ UpdateInode failed, "
-                       << "MetaStatusCode: " << ret
-                       << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
-                       << ", inodeid: " << inodeWrapper_->GetInodeId();
-            inodeWrapper_->MarkInodeError();
-        }
-        VLOG(9) << "Trim inode " << inodeWrapper_->GetInodeId()
-                << " async success.";
-        inodeCacheManager_->RemoveICache(inodeWrapper_);
-    };
-
- private:
-    std::shared_ptr<InodeWrapper> inodeWrapper_;
-    std::shared_ptr<InodeCacheManagerImpl> inodeCacheManager_;
-};
-
 #define GET_INODE_REMOTE(FSID, INODEID, OUT, STREAMING)                        \
     MetaStatusCode ret = metaClient_->GetInode(FSID, INODEID, OUT, STREAMING); \
     if (ret != MetaStatusCode::OK) {                                           \
@@ -83,17 +58,7 @@ class TrimICacheAsyncDone : public MetaServerClientDone {
             << "metaClient_ GetInode failed, MetaStatusCode = " << ret         \
             << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)          \
             << ", inodeid = " << INODEID;                                      \
-        return MetaStatusCodeToCurvefsErrCode(ret);                            \
-    }
-
-#define PUT_INODE_CACHE(INODEID, INODEWRAPPER)                                 \
-    std::shared_ptr<InodeWrapper> eliminatedOne;                               \
-    bool eliminated = iCache_->Put(INODEID, INODEWRAPPER, &eliminatedOne);     \
-    if (eliminated) {                                                          \
-        VLOG(3) << "GetInode eliminate one inode, ino: "                       \
-                << eliminatedOne->GetInodeId()                                 \
-                << ", iCache does not evict inodes via put interface ";        \
-        assert(0);                                                             \
+        return ToFSError(ret);                            \
     }
 
 #define REFRESH_DATA_REMOTE(OUT, STREAMING)                                    \
@@ -106,15 +71,8 @@ CURVEFS_ERROR
 InodeCacheManagerImpl::GetInode(uint64_t inodeId,
                                 std::shared_ptr<InodeWrapper> &out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    // get inode from cache
-    bool ok = iCache_->Get(inodeId, &out);
-    if (ok && NeedUseCahce(inodeId, out->IsDirty())) {
-        curve::common::UniqueLock lgGuard = out->GetUniqueLock();
-        if (out->GetType() == FsFileType::TYPE_FILE) {
-            return CURVEFS_ERROR::OK;
-        }
-
-        REFRESH_DATA_REMOTE(out, out->NeedRefreshData());
+    bool yes = openFiles_->IsOpened(inodeId, &out);
+    if (yes) {
         return CURVEFS_ERROR::OK;
     }
 
@@ -129,24 +87,12 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId,
     // refresh data
     REFRESH_DATA_REMOTE(out, streaming);
 
-    // put to cache
-    PUT_INODE_CACHE(inodeId, out);
-
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
                                                   InodeAttr *out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    // 1. find in icache
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    bool ok = iCache_->Get(inodeId, &inodeWrapper);
-    if (ok && NeedUseCahce(inodeId, inodeWrapper->IsDirty())) {
-        inodeWrapper->GetInodeAttr(out);
-        return CURVEFS_ERROR::OK;
-    }
-
-    // 2. get form metaserver
     std::set<uint64_t> inodeIds;
     std::list<InodeAttr> attrs;
     inodeIds.emplace(inodeId);
@@ -156,7 +102,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
         LOG(ERROR) << "metaClient BatchGetInodeAttr failed"
                    << ", inodeId = " << inodeId << ", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
 
     if (attrs.size() != 1) {
@@ -173,21 +119,6 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
     std::set<uint64_t> *inodeIds,
     std::list<InodeAttr> *attrs) {
-    // get some inode attr in icache
-    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        NameLockGuard lock(nameLock_, std::to_string(*iter));
-        bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
-            InodeAttr tmpAttr;
-            inodeWrapper->GetInodeAttr(&tmpAttr);
-            attrs->emplace_back(std::move(tmpAttr));
-            iter = inodeIds->erase(iter);
-            continue;
-        }
-        ++iter;
-    }
-
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
@@ -199,7 +130,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
     }
-    return MetaStatusCodeToCurvefsErrCode(ret);
+    return ToFSError(ret);
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
@@ -207,26 +138,6 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
     std::set<uint64_t> *inodeIds,
     std::map<uint64_t, InodeAttr> *attrs) {
     NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
-    std::map<uint64_t, InodeAttr> cachedAttr;
-    bool cache  = iAttrCache_->Get(parentId, &cachedAttr);
-
-    // get some inode attr in icache
-    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        NameLockGuard lock(nameLock_, std::to_string(*iter));
-        bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
-            InodeAttr tmpAttr;
-            inodeWrapper->GetInodeAttr(&tmpAttr);
-            attrs->emplace(*iter, std::move(tmpAttr));
-            iter = inodeIds->erase(iter);
-        } else if (cache && cachedAttr.find(*iter) != cachedAttr.end()) {
-            attrs->emplace(*iter, cachedAttr[*iter]);
-            iter = inodeIds->erase(iter);
-        } else {
-            ++iter;
-        }
-    }
 
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
@@ -238,12 +149,12 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
         return CURVEFS_ERROR::NOTEXIST;
     }
 
+    ::curve::common::Mutex mutex;
     std::shared_ptr<CountDownEvent> cond =
         std::make_shared<CountDownEvent>(inodeGroups.size());
     for (const auto& it : inodeGroups) {
         VLOG(3) << "BatchGetInodeAttrAsync Send " << it.size();
-        auto* done = new BatchGetInodeAttrAsyncDone(shared_from_this(),
-                                                    cond, parentId);
+        auto* done = new BatchGetInodeAttrAsyncDone(attrs, &mutex, cond);
         MetaStatusCode ret = metaClient_->BatchGetInodeAttrAsync(fsId_, it,
                                                                  done);
         if (MetaStatusCode::OK != ret) {
@@ -256,30 +167,12 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
 
     // wait for all sudrequest finished
     cond->Wait();
-
-    bool ok  = iAttrCache_->Get(parentId, attrs);
-    if (!ok) {
-        LOG(WARNING) << "get attrs form iAttrCache_ failed.";
-    }
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
     std::set<uint64_t> *inodeIds,
     std::list<XAttr> *xattr) {
-    // get some inode in icache
-    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        NameLockGuard lock(nameLock_, std::to_string(*iter));
-        bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
-            xattr->emplace_back(inodeWrapper->GetXattr());
-            iter = inodeIds->erase(iter);
-        } else {
-            ++iter;
-        }
-    }
-
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
@@ -290,7 +183,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
     }
-    return MetaStatusCodeToCurvefsErrCode(ret);
+    return ToFSError(ret);
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
@@ -301,159 +194,29 @@ CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
     if (ret != MetaStatusCode::OK) {
         LOG(ERROR) << "metaClient_ CreateInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
-    uint64_t inodeid = inode.inodeid();
     out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_,
         s3ChunkInfoMetric_, option_.maxDataSize,
         option_.refreshDataIntervalSec);
-
-    std::shared_ptr<InodeWrapper> eliminatedOne;
-    bool eliminated = false;
-    {
-        NameLockGuard lock(nameLock_, std::to_string(inodeid));
-        eliminated = iCache_->Put(inodeid, out, &eliminatedOne);
-    }
-    if (eliminated) {
-        /* iCache does not evict inodes via put interface */
-        assert(0);
-    }
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::DeleteInode(uint64_t inodeId) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    iCache_->Remove(inodeId);
     MetaStatusCode ret = metaClient_->DeleteInode(fsId_, inodeId);
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeId = " << inodeId;
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
-
-    curve::common::LockGuard lg2(dirtyMapMutex_);
-    dirtyMap_.erase(inodeId);
     return CURVEFS_ERROR::OK;
 }
 
-void InodeCacheManagerImpl::AddInodeAttrs(
-    uint64_t parentId, const RepeatedPtrField<InodeAttr>& inodeAttrs) {
-    iAttrCache_->Set(parentId, inodeAttrs);
-}
-
-void InodeCacheManagerImpl::ClearInodeCache(uint64_t inodeId) {
-    {
-        NameLockGuard lock(nameLock_, std::to_string(inodeId));
-        iCache_->Remove(inodeId);
-    }
-    curve::common::LockGuard lg2(dirtyMapMutex_);
-    dirtyMap_.erase(inodeId);
-}
-
 void InodeCacheManagerImpl::ShipToFlush(
-    const std::shared_ptr<InodeWrapper> &inodeWrapper) {
-    curve::common::LockGuard lg(dirtyMapMutex_);
-    dirtyMap_.emplace(inodeWrapper->GetInodeId(), inodeWrapper);
-}
-
-void InodeCacheManagerImpl::FlushAll() {
-    while (!dirtyMap_.empty()) {
-        FlushInodeOnce();
-    }
-}
-
-void InodeCacheManagerImpl::FlushInodeOnce() {
-    std::map<uint64_t, std::shared_ptr<InodeWrapper>> temp_;
-    {
-        curve::common::LockGuard lg(dirtyMapMutex_);
-        temp_.swap(dirtyMap_);
-    }
-    for (auto it = temp_.begin(); it != temp_.end(); it++) {
-        curve::common::UniqueLock ulk = it->second->GetUniqueLock();
-        it->second->Async(nullptr, true);
-    }
-}
-
-void InodeCacheManagerImpl::ReleaseCache(uint64_t parentId) {
-    NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
-    iAttrCache_->Release(parentId);
-}
-
-void InodeCacheManagerImpl::FlushInodeBackground() {
-    LOG(INFO) << "flush thread is start.";
-    while (!isStop_.load()) {
-        if (iCache_->Size() > maxCacheSize_) {
-            TrimIcache(iCache_->Size() - maxCacheSize_);
-        }
-        FlushInodeOnce();
-        sleeper_.wait_for(std::chrono::seconds(flushPeriodSec_));
-    }
-    LOG(INFO) << "flush thread is stop.";
-}
-
-namespace {
-// Wether an inode is dirty or not.
-// if |needLock| is true, we acquire the lock firstly and do the test,
-// otherwise, we assume the lock is already held.
-bool IsDirtyInode(InodeWrapper *ino, bool needLock) {
-    auto check = [ino]() {
-        return ino->IsDirty() || !ino->S3ChunkInfoEmptyNolock() ||
-               ino->GetMutableExtentCacheLocked()->HasDirtyExtents();
-    };
-
-    if (needLock) {
-        auto lk = ino->GetUniqueLock();
-        return check();
-    }
-
-    return check();
-}
-}  // namespace
-
-void InodeCacheManagerImpl::RemoveICache(
-    const std::shared_ptr<InodeWrapper> &inode) {
-    if (!IsDirtyInode(inode.get(), true)) {
-        uint64_t inodeId = inode->GetInodeId();
-        NameLockGuard lock(nameLock_, std::to_string(inodeId));
-        ::curve::common::UniqueLock lgGuard = inode->GetUniqueLock();
-        iCache_->Remove(inodeId);
-    }
-}
-
-void InodeCacheManagerImpl::TrimIcache(uint64_t trimSize) {
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    uint64_t inodeId;
-    VLOG(3) << "TrimIcache trimSize " << trimSize;
-    while (trimSize > 0) {
-        bool ok = iCache_->GetLast(&inodeId, &inodeWrapper);
-        if (ok) {
-            NameLockGuard lock(nameLock_, std::to_string(inodeId));
-            ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-            if (IsDirtyInode(inodeWrapper.get(), false)) {
-                VLOG(9) << "TrimIcache sync dirty inode " << inodeId;
-                dirtyMapMutex_.lock();
-                dirtyMap_.erase(inodeId);
-                dirtyMapMutex_.unlock();
-                auto *done =
-                    new TrimICacheAsyncDone(inodeWrapper, shared_from_this());
-                inodeWrapper->Async(done);
-            } else {
-                VLOG(9) << "TrimIcache remove inode " << inodeId
-                        << " from iCache";
-                iCache_->Remove(inodeId);
-            }
-            trimSize--;
-            // remove the attr of the inode in iattrcache
-            auto parents = inodeWrapper->GetParentLocked();
-            for (uint64_t parent : parents) {
-                iAttrCache_->Remove(parent, inodeId);
-            }
-        } else {
-            LOG(ERROR) << "icache size " << iCache_->Size();
-            assert(0);
-        }
-    }
+    const std::shared_ptr<InodeWrapper>& inode) {
+    deferSync_->Push(inode);
 }
 
 CURVEFS_ERROR
@@ -490,33 +253,5 @@ InodeCacheManagerImpl::RefreshData(std::shared_ptr<InodeWrapper> &inode,
     return rc;
 }
 
-void InodeCacheManagerImpl::AddOpenedInode(uint64_t inodeId) {
-    VLOG(1) << "AddOpenedInode inodeId: " <<  inodeId;
-    curve::common::LockGuard lg(openInodesMutex_);
-    openedInodes_.emplace(inodeId);
-}
-
-void InodeCacheManagerImpl::RemoveOpenedInode(uint64_t inodeId) {
-    VLOG(1) << "RemoveOpenedInode inodeId: " <<  inodeId;
-    curve::common::LockGuard lg(openInodesMutex_);
-    auto iter = openedInodes_.find(inodeId);
-    if (iter != openedInodes_.end()) {
-        openedInodes_.erase(iter);
-    }
-}
-
-bool InodeCacheManagerImpl::OpenInodeCached(uint64_t inodeId) {
-    curve::common::LockGuard lg(openInodesMutex_);
-    auto iter = openedInodes_.find(inodeId);
-    return iter != openedInodes_.end();
-}
-
-bool InodeCacheManagerImpl::NeedUseCahce(uint64_t inodeId, bool IsDirty) {
-    if (!FLAGS_enableCto || OpenInodeCached(inodeId) || IsDirty) {
-        return true;
-    }
-    return false;
-}
-
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/inode_cache_manager.h

@@ -38,12 +38,14 @@
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/interruptible_sleeper.h"
 #include "curvefs/src/client/inode_wrapper.h"
 #include "src/common/concurrent/name_lock.h"
 #include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/openfile.h"
+#include "curvefs/src/client/filesystem/defer_sync.h"
 
 using ::curve::common::LRUCache;
 using ::curve::common::CacheMetrics;
@@ -64,57 +66,8 @@ using rpcclient::BatchGetInodeAttrDone;
 using curve::common::CountDownEvent;
 using metric::S3ChunkInfoMetric;
 using common::RefreshDataOption;
-
-class InodeAttrCache {
- public:
-    InodeAttrCache() {}
-    ~InodeAttrCache() {}
-
-    bool Get(uint64_t parentId, std::map<uint64_t, InodeAttr> *imap) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        auto iter = iAttrCache_.find(parentId);
-        if (iter != iAttrCache_.end()) {
-            imap->insert(iter->second.begin(), iter->second.end());
-            return true;
-        }
-        return false;
-    }
-
-    void Set(uint64_t parentId, const RepeatedPtrField<InodeAttr>& inodeAttrs) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        VLOG(1) << "parentId = " << parentId
-                << ", iAttrCache set size = " << inodeAttrs.size();
-        auto& inner = iAttrCache_[parentId];
-        for (const auto &it : inodeAttrs) {
-            inner.emplace(it.inodeid(), it);
-        }
-    }
-
-    void Release(uint64_t parentId) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        auto size = iAttrCache_.size();
-        auto iter = iAttrCache_.find(parentId);
-        if (iter != iAttrCache_.end()) {
-            iAttrCache_.erase(iter);
-        }
-        VLOG(1) << "inodeId = " << parentId
-                << ", inode attr cache release, before = "
-                << size << ", after = " << iAttrCache_.size();
-    }
-
-    void Remove(uint64_t parentId, uint64_t inodeId) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        auto iter = iAttrCache_.find(parentId);
-        if (iter != iAttrCache_.end()) {
-            iter->second.erase(inodeId);
-        }
-    }
-
- private:
-    // inodeAttr cache; <parentId, <inodeId, inodeAttr>>
-    std::map<uint64_t, std::map<uint64_t, InodeAttr>> iAttrCache_;
-    curve::common::Mutex iAttrCacheMutex_;
-};
+using ::curvefs::client::filesystem::OpenFiles;
+using ::curvefs::client::filesystem::DeferSync;
 
 class InodeCacheManager {
  public:
@@ -126,13 +79,9 @@ class InodeCacheManager {
         fsId_ = fsId;
     }
 
-    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
-                               uint32_t flushPeriodSec,
-                               RefreshDataOption option) = 0;
-
-    virtual void Run() = 0;
-
-    virtual void Stop() = 0;
+    virtual CURVEFS_ERROR Init(RefreshDataOption option,
+                               std::shared_ptr<OpenFiles> openFiles,
+                               std::shared_ptr<DeferSync> deferSync) = 0;
 
     virtual CURVEFS_ERROR
     GetInode(uint64_t inodeId,
@@ -157,81 +106,34 @@ class InodeCacheManager {
 
     virtual CURVEFS_ERROR DeleteInode(uint64_t inodeId) = 0;
 
-    virtual void AddInodeAttrs(uint64_t parentId,
-        const RepeatedPtrField<InodeAttr>& inodeAttrs) = 0;
-
-    virtual void ClearInodeCache(uint64_t inodeId) = 0;
-
     virtual void ShipToFlush(
         const std::shared_ptr<InodeWrapper> &inodeWrapper) = 0;
 
-    virtual void FlushAll() = 0;
-
-    virtual void FlushInodeOnce() = 0;
-
-    virtual void ReleaseCache(uint64_t parentId) = 0;
-
-    virtual void AddOpenedInode(uint64_t inodeId) = 0;
-
-    virtual void RemoveOpenedInode(uint64_t inodeId) = 0;
-
  protected:
     uint32_t fsId_;
 };
 
 class InodeCacheManagerImpl : public InodeCacheManager,
     public std::enable_shared_from_this<InodeCacheManagerImpl> {
+
  public:
     InodeCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()),
-        iCache_(nullptr),
-        iAttrCache_(nullptr),
-        isStop_(true) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
 
     explicit InodeCacheManagerImpl(
         const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient),
-        iCache_(nullptr),
-        iAttrCache_(nullptr) {}
+      : metaClient_(metaClient) {}
 
-    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
-                       uint32_t flushPeriodSec,
-                       RefreshDataOption option) override {
-        if (enableCacheMetrics) {
-            iCache_ = std::make_shared<
-                LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0,
-                    std::make_shared<CacheMetrics>("icache"));
-        } else {
-            iCache_ = std::make_shared<
-                LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0);
-        }
-        maxCacheSize_ = cacheSize;
+    CURVEFS_ERROR Init(RefreshDataOption option,
+                       std::shared_ptr<OpenFiles> openFiles,
+                       std::shared_ptr<DeferSync> deferSync) override {
         option_ = option;
-        flushPeriodSec_ = flushPeriodSec;
-        iAttrCache_ = std::make_shared<InodeAttrCache>();
         s3ChunkInfoMetric_ = std::make_shared<S3ChunkInfoMetric>();
+        openFiles_ =  openFiles;
+        deferSync_ = deferSync;
         return CURVEFS_ERROR::OK;
     }
 
-    void Run() {
-        isStop_.exchange(false);
-        flushThread_ =
-            Thread(&InodeCacheManagerImpl::FlushInodeBackground, this);
-        LOG(INFO) << "Start inodeManager flush thread ok.";
-    }
-
-    void Stop() {
-        isStop_.exchange(true);
-        LOG(INFO) << "stop inodeManager flush thread ...";
-        sleeper_.interrupt();
-        flushThread_.join();
-    }
-
-    bool IsDirtyMapExist(uint64_t inodeId) {
-        curve::common::LockGuard lg(dirtyMapMutex_);
-        return dirtyMap_.count(inodeId) > 0;
-    }
-
     CURVEFS_ERROR GetInode(uint64_t inodeId,
                            std::shared_ptr<InodeWrapper> &out) override;
 
@@ -252,71 +154,37 @@ class InodeCacheManagerImpl : public InodeCacheManager,
 
     CURVEFS_ERROR DeleteInode(uint64_t inodeId) override;
 
-    void AddInodeAttrs(uint64_t parentId,
-        const RepeatedPtrField<InodeAttr>& inodeAttrs) override;
-
-    void ClearInodeCache(uint64_t inodeId) override;
-
     void ShipToFlush(
         const std::shared_ptr<InodeWrapper> &inodeWrapper) override;
 
-    void FlushAll() override;
-
-    void FlushInodeOnce() override;
-
-    void ReleaseCache(uint64_t parentId) override;
-
-    void RemoveICache(const std::shared_ptr<InodeWrapper> &inode);
-
-    void AddOpenedInode(uint64_t inodeId) override;
-
-    void RemoveOpenedInode(uint64_t inodeId) override;
-
-    bool NeedUseCahce(uint64_t inodeId, bool IsDirty);
-
  private:
-    virtual void FlushInodeBackground();
-    void TrimIcache(uint64_t trimSize);
     CURVEFS_ERROR RefreshData(std::shared_ptr<InodeWrapper> &inode,  // NOLINT
                               bool streaming = true);
-    bool OpenInodeCached(uint64_t inodeId);
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
-    std::shared_ptr<LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>> iCache_;
     std::shared_ptr<S3ChunkInfoMetric> s3ChunkInfoMetric_;
 
-    std::shared_ptr<InodeAttrCache> iAttrCache_;
+    std::shared_ptr<OpenFiles> openFiles_;
 
-    // dirty map, key is inodeid
-    std::map<uint64_t, std::shared_ptr<InodeWrapper>> dirtyMap_;
-    curve::common::Mutex dirtyMapMutex_;
-
-    // record opened inode
-    std::multiset<uint64_t> openedInodes_;
-    curve::common::Mutex openInodesMutex_;
+    std::shared_ptr<DeferSync> deferSync_;
 
     curve::common::GenericNameLock<Mutex> nameLock_;
 
     curve::common::GenericNameLock<Mutex> asyncNameLock_;
 
-    uint64_t maxCacheSize_;
     RefreshDataOption option_;
-    uint32_t flushPeriodSec_;
-    Thread flushThread_;
-    InterruptibleSleeper sleeper_;
-    Atomic<bool> isStop_;
 };
 
 class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {
  public:
-    BatchGetInodeAttrAsyncDone(
-        const std::shared_ptr<InodeCacheManager> &inodeCacheManager,
-        std::shared_ptr<CountDownEvent> cond,
-        uint64_t parentId):
-        inodeCacheManager_(inodeCacheManager),
-        cond_(cond),
-        parentId_(parentId) {}
+    BatchGetInodeAttrAsyncDone(std::map<uint64_t, InodeAttr>* attrs,
+                               ::curve::common::Mutex* mutex,
+                               std::shared_ptr<CountDownEvent> cond):
+        attrs_(attrs),
+        mutex_(mutex),
+        cond_(cond) {}
+
     ~BatchGetInodeAttrAsyncDone() {}
 
     void Run() override {
@@ -324,22 +192,25 @@ class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {
         MetaStatusCode ret = GetStatusCode();
         if (ret != MetaStatusCode::OK) {
             LOG(ERROR) << "BatchGetInodeAttrAsync failed, "
-                       << "parentId = " << parentId_
                        << ", MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret);
         } else {
             auto inodeAttrs = GetInodeAttrs();
             VLOG(3) << "BatchGetInodeAttrAsyncDone update inodeAttrCache"
                     << " size = " << inodeAttrs.size();
-            inodeCacheManager_->AddInodeAttrs(parentId_, inodeAttrs);
+
+            curve::common::LockGuard lk(*mutex_);
+            for (const auto& attr : inodeAttrs) {
+                attrs_->emplace(attr.inodeid(), attr);
+            }
         }
         cond_->Signal();
     };
 
  private:
-    std::shared_ptr<InodeCacheManager> inodeCacheManager_;
+    ::curve::common::Mutex* mutex_;
+    std::map<uint64_t, InodeAttr>* attrs_;
     std::shared_ptr<CountDownEvent> cond_;
-    uint64_t parentId_;
 };
 
 }  // namespace client

curvefs/src/client/inode_wrapper.cpp

@@ -33,7 +33,6 @@
 
 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/client/async_request_closure.h"
-#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
 #include "curvefs/src/client/xattr_manager.h"
@@ -47,6 +46,7 @@ namespace client {
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::DataIndices;
+using ::curvefs::client::filesystem::ToFSError;
 
 bvar::Adder<int64_t> g_alive_inode_count{"alive_inode_count"};
 
@@ -192,7 +192,7 @@ CURVEFS_ERROR InodeWrapper::SyncAttr(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
 
         dirty_ = false;
@@ -212,7 +212,7 @@ CURVEFS_ERROR InodeWrapper::SyncS3ChunkInfo(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
         ClearS3ChunkInfoAdd();
     }
@@ -294,7 +294,7 @@ CURVEFS_ERROR InodeWrapper::RefreshS3ChunkInfo() {
                    << "MetaStatusCode: " << ret
                    << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                    << ", inodeid: " << inode_.inodeid();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
     auto before = s3ChunkInfoSize_;
     inode_.mutable_s3chunkinfomap()->swap(s3ChunkInfoMap);
@@ -328,7 +328,7 @@ CURVEFS_ERROR InodeWrapper::Link(uint64_t parent) {
                    <<", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeid = " << inode_.inodeid();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
 
     dirty_ = false;
@@ -396,7 +396,7 @@ CURVEFS_ERROR InodeWrapper::UnLink(uint64_t parent) {
                        << ", MetaStatusCode = " << ret
                        << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                        << ", inodeid = " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
         dirty_ = false;
         dirtyAttr_.Clear();
@@ -427,7 +427,7 @@ CURVEFS_ERROR InodeWrapper::UpdateParent(
                    << ", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeid = " << inode_.inodeid();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
     dirty_ = false;
     dirtyAttr_.Clear();
@@ -457,7 +457,7 @@ CURVEFS_ERROR InodeWrapper::Sync(bool internal) {
 }
 
 void InodeWrapper::Async(MetaServerClientDone *done, bool internal) {
-    VLOG(3) << "async inode: " << inode_.ShortDebugString();
+    VLOG(9) << "async inode: " << inode_.ShortDebugString();
 
     switch (inode_.type()) {
         case FsFileType::TYPE_S3:
@@ -515,7 +515,7 @@ CURVEFS_ERROR InodeWrapper::SyncS3(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
         ClearS3ChunkInfoAdd();
         dirty_ = false;
@@ -597,7 +597,7 @@ CURVEFS_ERROR InodeWrapper::RefreshVolumeExtent() {
         LOG(ERROR) << "GetVolumeExtent failed, inodeid: " << inode_.inodeid();
     }
 
-    return MetaStatusCodeToCurvefsErrCode(st);
+    return ToFSError(st);
 }
 
 CURVEFS_ERROR InodeWrapper::RefreshNlink() {

curvefs/src/client/inode_wrapper.h

@@ -34,12 +34,12 @@
 
 #include "curvefs/src/common/define.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "src/common/concurrent/concurrent.h"
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/src/client/metric/client_metric.h"
 #include "src/common/timeutility.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 using ::curvefs::metaserver::Inode;
 using ::curvefs::metaserver::S3ChunkInfoList;
@@ -53,15 +53,13 @@ constexpr int kChangeTime = 1 << 1;
 constexpr int kModifyTime = 1 << 2;
 
 using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
 
 enum class InodeStatus {
     kNormal = 0,
     kError = -1,
 };
 
-// TODO(xuchaojie) : get from conf maybe?
-const uint32_t kOptimalIOBlockSize = 0x10000u;
-
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::MetaServerClientDone;

curvefs/src/client/kvclient/kvclient_manager.cpp

@@ -21,6 +21,7 @@
  */
 
 #include "curvefs/src/client/kvclient/kvclient_manager.h"
+#include <memory>
 #include "src/client/client_metric.h"
 #include "src/common/concurrent/count_down_event.h"
 
@@ -31,59 +32,48 @@ using curvefs::client::metric::KVClientMetric;
 namespace curvefs {
 namespace client {
 
-KVClientManager *g_kvClientManager = nullptr;
-KVClientMetric *g_kvClientMetric = nullptr;
-
-#define ONRETURN(TYPE, RES, KEY, ERRORLOG)                                     \
+#define ONRETURN(TYPE, RES)                                                    \
     if (RES) {                                                                 \
-        g_kvClientMetric->kvClient##TYPE.qps.count << 1;                       \
-        VLOG(9) << "##TYPE key = " << KEY << " OK";                            \
+        kvClientMetric_.kvClient##TYPE.qps.count << 1;                        \
     } else {                                                                   \
-        g_kvClientMetric->kvClient##TYPE.eps.count << 1;                       \
-        LOG(ERROR) << "##TYPE key = " << KEY << " error = " << ERRORLOG;       \
-    }
+        kvClientMetric_.kvClient##TYPE.eps.count << 1;                        \
+    }                                                                          \
 
 bool KVClientManager::Init(const KVClientManagerOpt &config,
                            const std::shared_ptr<KVClient> &kvclient) {
     client_ = kvclient;
-    g_kvClientMetric = new KVClientMetric();
     return threadPool_.Start(config.setThreadPooln) == 0;
 }
 
 void KVClientManager::Uninit() {
     client_->UnInit();
     threadPool_.Stop();
-    delete g_kvClientMetric;
 }
 
 void KVClientManager::Set(std::shared_ptr<SetKVCacheTask> task) {
     threadPool_.Enqueue([task, this]() {
-        LatencyGuard guard(&g_kvClientMetric->kvClientSet.latency);
+        LatencyGuard guard(&kvClientMetric_.kvClientSet.latency);
 
         std::string error_log;
         auto res =
             client_->Set(task->key, task->value, task->length, &error_log);
-        ONRETURN(Set, res, task->key, error_log);
+        ONRETURN(Set, res);
 
         task->done(task);
     });
 }
 
-bool KVClientManager::Get(std::shared_ptr<GetKvCacheContext> task) {
-    assert(nullptr != task->value);
-    return Get(task->key, task->value, task->offset, task->length);
-}
+void KVClientManager::Get(std::shared_ptr<GetKVCacheTask> task) {
+    threadPool_.Enqueue([task, this]() {
+        LatencyGuard guard(&kvClientMetric_.kvClientGet.latency);
 
+        std::string error_log;
+        task->res = client_->Get(task->key, task->value, task->offset,
+                                task->length, &error_log);
+        ONRETURN(Get, task->res);
 
-bool KVClientManager::Get(const std::string &key, char *value, uint64_t offset,
-                          uint64_t length) {
-    LatencyGuard guard(&g_kvClientMetric->kvClientGet.latency);
-
-    assert(nullptr != value);
-    std::string error_log;
-    auto res = client_->Get(key, value, offset, length, &error_log);
-    ONRETURN(Get, res, key, error_log);
-    return res;
+        task->done(task);
+    });
 }
 
 }  // namespace client

curvefs/src/client/kvclient/kvclient_manager.h

@@ -44,14 +44,14 @@ namespace client {
 
 class KVClientManager;
 class SetKVCacheTask;
+class GetKVCacheTask;
 using curve::common::TaskThreadPool;
 using curvefs::client::common::KVClientManagerOpt;
 
-extern KVClientManager *g_kvClientManager;
-extern KVClientMetric *g_kvClientMetric;
-
 typedef std::function<void(const std::shared_ptr<SetKVCacheTask> &)>
     SetKVCacheDone;
+typedef std::function<void(const std::shared_ptr<GetKVCacheTask> &)>
+    GetKVCacheDone;
 
 struct SetKVCacheTask {
     std::string key;
@@ -59,17 +59,23 @@ struct SetKVCacheTask {
     uint64_t length;
     SetKVCacheDone done;
     SetKVCacheTask() = default;
-    SetKVCacheTask(const std::string &k, const char *val, const uint64_t len)
-        : key(k), value(val), length(len) {}
+    SetKVCacheTask(
+        const std::string &k, const char *val, const uint64_t len,
+        SetKVCacheDone done = [](const std::shared_ptr<SetKVCacheTask> &) {})
+        : key(k), value(val), length(len), done(std::move(done)) {}
 };
 
-struct GetKvCacheContext {
+struct GetKVCacheTask {
     const std::string &key;
     char *value;
     uint64_t offset;
     uint64_t length;
-    GetKvCacheContext(const std::string &k, char *v, uint64_t off, uint64_t len)
-        : key(k), value(v), offset(off), length(len) {}
+    bool res;
+    GetKVCacheDone done;
+    GetKVCacheTask(const std::string &k, char *v, uint64_t off, uint64_t len)
+        : key(k), value(v), offset(off), length(len), res(false) {
+        done = [](const std::shared_ptr<GetKVCacheTask> &) {};
+    }
 };
 
 class KVClientManager {
@@ -87,10 +93,9 @@ class KVClientManager {
      */
     void Set(std::shared_ptr<SetKVCacheTask> task);
 
-    bool Get(const std::string &key, char *value, uint64_t offset,
-             uint64_t length);
+    void Get(std::shared_ptr<GetKVCacheTask> task);
 
-    bool Get(std::shared_ptr<GetKvCacheContext> task);
+    KVClientMetric *GetClientMetricForTesting() { return &kvClientMetric_; }
 
  private:
     void Uninit();
@@ -98,6 +103,7 @@ class KVClientManager {
  private:
     TaskThreadPool<bthread::Mutex, bthread::ConditionVariable> threadPool_;
     std::shared_ptr<KVClient> client_;
+    KVClientMetric kvClientMetric_;
 };
 
 }  // namespace client

curvefs/src/client/kvclient/memcache_client.h

@@ -92,12 +92,9 @@ class MemCachedClient : public KVClient {
                 return false;
             }
         }
-
         memcached_behavior_set(client_, MEMCACHED_BEHAVIOR_DISTRIBUTION,
                                MEMCACHED_DISTRIBUTION_CONSISTENT);
         memcached_behavior_set(client_, MEMCACHED_BEHAVIOR_RETRY_TIMEOUT, 5);
-        memcached_behavior_set(client_,
-                               MEMCACHED_BEHAVIOR_REMOVE_FAILED_SERVERS, 1);
 
         return PushServer();
     }
@@ -117,9 +114,11 @@ class MemCachedClient : public KVClient {
         auto res = memcached_set(tcli, key.c_str(), key.length(), value,
                                  value_len, 0, 0);
         if (MEMCACHED_SUCCESS == res) {
+            VLOG(9) << "Set key = " << key << " OK";
             return true;
         }
         *errorlog = ResError(res);
+        LOG(ERROR) << "Set key = " << key << " error = " << *errorlog;
         return false;
     }
 
@@ -135,13 +134,21 @@ class MemCachedClient : public KVClient {
         memcached_return_t ue;
         char *res = memcached_get(tcli, key.c_str(), key.length(),
                                   &value_length, &flags, &ue);
-        if (res != nullptr && value) {
+        if (MEMCACHED_SUCCESS == ue && res != nullptr && value &&
+            value_length >= length) {
+            VLOG(9) << "Get key = " << key << " OK";
             memcpy(value, res + offset, length);
             free(res);
             return true;
         }
 
         *errorlog = ResError(ue);
+        if (ue != MEMCACHED_NOTFOUND) {
+            LOG(ERROR) << "Get key = " << key << " error = " << *errorlog
+                       << ", get_value_len = " << value_length
+                       << ", expect_value_len = " << length;
+        }
+
         return false;
     }
 

curvefs/src/client/main.cpp

@@ -47,13 +47,11 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
     release : FuseOpRelease,
     fsync : FuseOpFsync,
     opendir : FuseOpOpenDir,
-    // TODO(wuhongsong): readdirplus is problematic,
-    // resulting in inconsistent metadata
-    // #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
-    // readdir : 0,
-    // #else
+    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    readdir : 0,
+    #else
     readdir : FuseOpReadDir,
-    // #endif
+    #endif
     releasedir : FuseOpReleaseDir,
     fsyncdir : 0,
     statfs : FuseOpStatFs,
@@ -77,11 +75,11 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
     flock : 0,
     fallocate : 0,
     #endif
-    // TODO(wuhongsong): The current implementation is problematic,
-    // resulting in inconsistent metadata
-    // #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    readdirplus : FuseOpReadDirPlus,
+    #else
     readdirplus : 0,
-    // #endif
+    #endif
     #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 4)
     copy_file_range : 0,
     #endif
@@ -218,8 +216,8 @@ int main(int argc, char *argv[]) {
 
     fuse_daemonize(opts.foreground);
 
-    if (InitGlog(mOpts.conf, argv[0]) < 0) {
-        printf("Init glog failed, confpath = %s\n", mOpts.conf);
+    if (InitLog(mOpts.conf, argv[0]) < 0) {
+        printf("Init log failed, confpath = %s\n", mOpts.conf);
     }
 
     ret = InitFuseClient(mOpts.conf, mOpts.fsName, mOpts.fsType, mOpts.mdsAddr);

curvefs/src/client/metric/client_metric.h

@@ -294,6 +294,16 @@ struct S3ChunkInfoMetric {
     S3ChunkInfoMetric() : s3ChunkInfoSize(prefix, "size") {}
 };
 
+struct WarmupManagerS3Metric {
+    const std::string prefix = "curvefs_warmup";
+    InterfaceMetric warmupS3Cached;
+    bvar::Adder<uint64_t> warmupS3CacheSize;
+
+    WarmupManagerS3Metric()
+        : warmupS3Cached(prefix, "s3_cached"),
+          warmupS3CacheSize(prefix, "s3_cache_size") {}
+};
+
 }  // namespace metric
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/s3/client_s3_adaptor.cpp

@@ -24,6 +24,7 @@
 #include <brpc/controller.h>
 #include <algorithm>
 #include <list>
+#include <utility>
 
 #include "absl/memory/memory.h"
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
@@ -39,6 +40,7 @@ S3ClientAdaptorImpl::Init(
     std::shared_ptr<MdsClient> mdsClient,
     std::shared_ptr<FsCacheManager> fsCacheManager,
     std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
+    std::shared_ptr<KVClientManager> kvClientManager,
     bool startBackGround) {
     pendingReq_ = 0;
     blockSize_ = option.blockSize;
@@ -66,6 +68,7 @@ S3ClientAdaptorImpl::Init(
     fsCacheManager_ = fsCacheManager;
     waitInterval_.Init(option.intervalSec * 1000);
     diskCacheManagerImpl_ = diskCacheManagerImpl;
+    kvClientManager_ = std::move(kvClientManager);
     if (HasDiskCache()) {
         diskCacheManagerImpl_ = diskCacheManagerImpl;
         if (diskCacheManagerImpl_->Init(option) < 0) {
@@ -372,7 +375,7 @@ CURVEFS_ERROR S3ClientAdaptorImpl::FlushAllCache(uint64_t inodeId) {
     }
 
     // force flush data in diskcache to s3
-    if (!g_kvClientManager && HasDiskCache()) {
+    if (!kvClientManager_ && HasDiskCache()) {
         VLOG(6) << "FlushAllCache, wait inodeId:" << inodeId
                 << "related chunk upload to s3";
         if (ClearDiskCache(inodeId) < 0) {

curvefs/src/client/s3/client_s3_adaptor.h

@@ -33,7 +33,7 @@
 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/inode_cache_manager.h"
 #include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/src/client/s3/client_s3.h"
@@ -71,6 +71,7 @@ class S3ClientAdaptor {
          std::shared_ptr<MdsClient> mdsClient,
          std::shared_ptr<FsCacheManager> fsCacheManager,
          std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
+         std::shared_ptr<KVClientManager> kvClientManager,
          bool startBackGround = false) = 0;
     /**
      * @brief write data to s3
@@ -128,6 +129,7 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor {
          std::shared_ptr<MdsClient> mdsClient,
          std::shared_ptr<FsCacheManager> fsCacheManager,
          std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
+         std::shared_ptr<KVClientManager> kvClientManager,
          bool startBackGround = false);
     /**
      * @brief write data to s3
@@ -274,6 +276,8 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor {
 
     TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
         taskPool_;
+
+    std::shared_ptr<KVClientManager> kvClientManager_ = nullptr;
 };
 
 }  // namespace client

curvefs/src/client/s3/client_s3_cache_manager.cpp

@@ -96,7 +96,7 @@ FsCacheManager::FindOrCreateFileCacheManager(uint64_t fsId, uint64_t inodeId) {
     }
 
     FileCacheManagerPtr fileCacheManager = std::make_shared<FileCacheManager>(
-        fsId, inodeId, s3ClientAdaptor_);
+        fsId, inodeId, s3ClientAdaptor_, kvClientManager_);
     auto ret = fileCacheManagerMap_.emplace(inodeId, fileCacheManager);
     g_s3MultiManagerMetric->fileManagerNum << 1;
     assert(ret.second);
@@ -293,7 +293,8 @@ FileCacheManager::FindOrCreateChunkCacheManager(uint64_t index) {
     }
 
     ChunkCacheManagerPtr chunkCacheManager =
-        std::make_shared<ChunkCacheManager>(index, s3ClientAdaptor_);
+        std::make_shared<ChunkCacheManager>(index, s3ClientAdaptor_,
+                                            kvClientManager_);
     auto ret = chunkCacheMap_.emplace(index, chunkCacheManager);
     g_s3MultiManagerMetric->chunkManagerNum << 1;
     assert(ret.second);
@@ -509,11 +510,20 @@ bool FileCacheManager::ReadKVRequestFromRemoteCache(const std::string &name,
                                                     char *databuf,
                                                     uint64_t offset,
                                                     uint64_t length) {
-    if (!g_kvClientManager) {
+    if (!kvClientManager_) {
         return false;
     }
 
-    return g_kvClientManager->Get(name, databuf, offset, length);
+    auto task = std::make_shared<GetKVCacheTask>(name, databuf, offset, length);
+    CountDownEvent event(1);
+    task->done = [&](const std::shared_ptr<GetKVCacheTask> &task) {
+        event.Signal();
+        return;
+    };
+    kvClientManager_->Get(task);
+    event.Wait();
+
+    return task->res;
 }
 
 bool FileCacheManager::ReadKVRequestFromS3(const std::string &name,
@@ -607,7 +617,7 @@ int FileCacheManager::ReadKVRequest(
             WriteLockGuard writeLockGuard(chunkCacheManager->rwLockChunk_);
             DataCachePtr dataCache = std::make_shared<DataCache>(
                 s3ClientAdaptor_, chunkCacheManager, chunkPos, req->len,
-                dataBuf + req->readOffset);
+                dataBuf + req->readOffset, kvClientManager_);
             chunkCacheManager->AddReadDataCache(dataCache);
         }
     }
@@ -1468,8 +1478,9 @@ DataCachePtr ChunkCacheManager::FindWriteableDataCache(
 void ChunkCacheManager::WriteNewDataCache(S3ClientAdaptorImpl *s3ClientAdaptor,
                                           uint32_t chunkPos, uint32_t len,
                                           const char *data) {
-    DataCachePtr dataCache = std::make_shared<DataCache>(
-        s3ClientAdaptor, this->shared_from_this(), chunkPos, len, data);
+    DataCachePtr dataCache =
+        std::make_shared<DataCache>(s3ClientAdaptor, this->shared_from_this(),
+                                    chunkPos, len, data, kvClientManager_);
     VLOG(9) << "WriteNewDataCache chunkPos:" << chunkPos << ", len:" << len
             << ", new len:" << dataCache->GetLen() << ",chunkIndex:" << index_;
     WriteLockGuard writeLockGuard(rwLockWrite_);
@@ -1724,7 +1735,8 @@ void ChunkCacheManager::AddWriteDataCacheForTest(DataCachePtr dataCache) {
 
 DataCache::DataCache(S3ClientAdaptorImpl *s3ClientAdaptor,
                      ChunkCacheManagerPtr chunkCacheManager, uint64_t chunkPos,
-                     uint64_t len, const char *data)
+                     uint64_t len, const char *data,
+              std::shared_ptr<KVClientManager> kvClientManager)
     : s3ClientAdaptor_(std::move(s3ClientAdaptor)),
       chunkCacheManager_(chunkCacheManager),
       status_(DataCacheStatus::Dirty), inReadCache_(false) {
@@ -1783,6 +1795,8 @@ DataCache::DataCache(S3ClientAdaptorImpl *s3ClientAdaptor,
     assert((actualLen_ % pageSize) == 0);
     assert((actualChunkPos_ % pageSize) == 0);
     createTime_ = ::curve::common::TimeUtility::GetTimeofDaySec();
+
+    kvClientManager_ = std::move(kvClientManager);
 }
 
 void DataCache::CopyBufToDataCache(uint64_t dataCachePos, uint64_t len,
@@ -2303,7 +2317,7 @@ CURVEFS_ERROR DataCache::PrepareFlushTasks(uint64_t inodeId,
         s3Tasks->emplace_back(context);
 
         // generate flush to kvcache task
-        if (g_kvClientManager) {
+        if (kvClientManager_) {
             auto task = std::make_shared<SetKVCacheTask>();
             task->key = objectName;
             task->value = data + (*writeOffset);
@@ -2389,11 +2403,11 @@ void DataCache::FlushTaskExecute(
             });
     }
     // kvtask execute
-    if (g_kvClientManager && kvPendingTaskCal.load()) {
+    if (kvClientManager_ && kvPendingTaskCal.load()) {
         std::for_each(kvCacheTasks.begin(), kvCacheTasks.end(),
                       [&](const std::shared_ptr<SetKVCacheTask> &task) {
                           task->done = kvdone;
-                          g_kvClientManager->Set(task);
+                          kvClientManager_->Set(task);
                       });
         kvTaskEnvent.Wait();
     }

curvefs/src/client/s3/client_s3_cache_manager.h

@@ -29,12 +29,12 @@
 #include <memory>
 #include <string>
 #include <unordered_map>
+#include <utility>
 #include <vector>
 #include <set>
-#include <utility>
 
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/s3/client_s3.h"
 #include "curvefs/src/client/common/common.h"
 #include "src/common/concurrent/concurrent.h"
@@ -139,7 +139,8 @@ class DataCache : public std::enable_shared_from_this<DataCache> {
  public:
     DataCache(S3ClientAdaptorImpl *s3ClientAdaptor,
               ChunkCacheManagerPtr chunkCacheManager, uint64_t chunkPos,
-              uint64_t len, const char *data);
+              uint64_t len, const char *data,
+              std::shared_ptr<KVClientManager> kvClientManager);
     virtual ~DataCache() {
         auto iter = dataMap_.begin();
         for (; iter != dataMap_.end(); iter++) {
@@ -236,6 +237,8 @@ class DataCache : public std::enable_shared_from_this<DataCache> {
     std::atomic<int> status_;
     std::atomic<bool> inReadCache_;
     std::map<uint64_t, PageDataMap> dataMap_;  // first is block index
+
+    std::shared_ptr<KVClientManager> kvClientManager_;
 };
 
 class S3ReadResponse {
@@ -255,9 +258,11 @@ class S3ReadResponse {
 class ChunkCacheManager
     : public std::enable_shared_from_this<ChunkCacheManager> {
  public:
-    ChunkCacheManager(uint64_t index, S3ClientAdaptorImpl *s3ClientAdaptor)
+    ChunkCacheManager(uint64_t index, S3ClientAdaptorImpl *s3ClientAdaptor,
+                      std::shared_ptr<KVClientManager> kvClientManager)
         : index_(index), s3ClientAdaptor_(s3ClientAdaptor),
-          flushingDataCache_(nullptr) {}
+          flushingDataCache_(nullptr),
+          kvClientManager_(std::move(kvClientManager)) {}
     virtual ~ChunkCacheManager() = default;
     void ReadChunk(uint64_t index, uint64_t chunkPos, uint64_t readLen,
                    char *dataBuf, uint64_t dataBufOffset,
@@ -322,13 +327,17 @@ class ChunkCacheManager
     curve::common::Mutex flushMtx_;
     DataCachePtr flushingDataCache_;
     curve::common::Mutex flushingDataCacheMtx_;
+
+    std::shared_ptr<KVClientManager> kvClientManager_;
 };
 
 class FileCacheManager {
  public:
     FileCacheManager(uint32_t fsid, uint64_t inode,
-                     S3ClientAdaptorImpl *s3ClientAdaptor)
-        : fsId_(fsid), inode_(inode), s3ClientAdaptor_(s3ClientAdaptor) {}
+                     S3ClientAdaptorImpl *s3ClientAdaptor,
+                     std::shared_ptr<KVClientManager> kvClientManager)
+        : fsId_(fsid), inode_(inode), s3ClientAdaptor_(s3ClientAdaptor),
+          kvClientManager_(std::move(kvClientManager)) {}
     FileCacheManager() {}
 
     ChunkCacheManagerPtr FindOrCreateChunkCacheManager(uint64_t index);
@@ -434,16 +443,20 @@ class FileCacheManager {
     S3ClientAdaptorImpl *s3ClientAdaptor_;
     curve::common::Mutex downloadMtx_;
     std::set<std::string> downloadingObj_;
+
+    std::shared_ptr<KVClientManager> kvClientManager_;
 };
 
 class FsCacheManager {
  public:
     FsCacheManager(S3ClientAdaptorImpl *s3ClientAdaptor,
-                   uint64_t readCacheMaxByte, uint64_t writeCacheMaxByte)
+                   uint64_t readCacheMaxByte, uint64_t writeCacheMaxByte,
+                   std::shared_ptr<KVClientManager> kvClientManager)
         : lruByte_(0), wDataCacheNum_(0), wDataCacheByte_(0),
           readCacheMaxByte_(readCacheMaxByte),
           writeCacheMaxByte_(writeCacheMaxByte),
-          s3ClientAdaptor_(s3ClientAdaptor), isWaiting_(false) {}
+          s3ClientAdaptor_(s3ClientAdaptor), isWaiting_(false),
+          kvClientManager_(std::move(kvClientManager)) {}
     FsCacheManager() {}
     virtual FileCacheManagerPtr FindFileCacheManager(uint64_t inodeId);
     virtual FileCacheManagerPtr FindOrCreateFileCacheManager(uint64_t fsId,
@@ -551,6 +564,8 @@ class FsCacheManager {
     std::condition_variable cond_;
 
     ReadCacheReleaseExecutor releaseReadCache_;
+
+    std::shared_ptr<KVClientManager> kvClientManager_;
 };
 
 }  // namespace client

curvefs/src/client/volume/default_volume_storage.cpp

@@ -30,7 +30,7 @@
 #include <vector>
 
 #include "absl/meta/type_traits.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/inode_cache_manager.h"
 #include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/src/client/volume/extent_cache.h"

curvefs/src/client/volume/default_volume_storage.h

@@ -31,12 +31,14 @@
 #include "curvefs/src/client/volume/volume_storage.h"
 #include "curvefs/src/volume/block_device_client.h"
 #include "curvefs/src/volume/space_manager.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 namespace curvefs {
 namespace client {
 
 using ::curvefs::volume::BlockDeviceClient;
 using ::curvefs::volume::SpaceManager;
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
 
 class InodeCacheManager;
 

curvefs/src/client/volume/volume_storage.h

@@ -28,11 +28,13 @@
 #include <cstddef>
 #include <cstdint>
 
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 namespace curvefs {
 namespace client {
 
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
+
 class VolumeStorage {
  public:
     virtual ~VolumeStorage() = default;

curvefs/src/client/warmup/warmup_manager.cpp

@@ -0,0 +1,703 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Created Date: 2023-01-31
+ * Author: chengyi01
+ */
+
+#include "curvefs/src/client/warmup/warmup_manager.h"
+
+#include <glog/logging.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <atomic>
+#include <deque>
+#include <list>
+#include <memory>
+#include <utility>
+
+#include "curvefs/src/client/inode_wrapper.h"
+#include "curvefs/src/client/kvclient/kvclient_manager.h"
+#include "curvefs/src/client/s3/client_s3_cache_manager.h"
+#include "curvefs/src/common/s3util.h"
+#include "src/common/concurrent/concurrent.h"
+#include "src/common/string_util.h"
+
+
+namespace curvefs {
+namespace client {
+namespace warmup {
+
+using curve::common::WriteLockGuard;
+
+#define WARMUP_CHECKINTERVAL_US (1000 * 1000)
+
+bool WarmupManagerS3Impl::AddWarmupFilelist(fuse_ino_t key) {
+    if (!mounted_.load(std::memory_order_acquire)) {
+        LOG(ERROR) << "not mounted";
+        return false;
+    }
+    // add warmup Progress
+    if (AddWarmupProcess(key)) {
+        VLOG(9) << "add warmup list task:" << key;
+        WriteLockGuard lock(warmupFilelistDequeMutex_);
+        auto iter = FindKeyWarmupFilelistLocked(key);
+        if (iter == warmupFilelistDeque_.end()) {
+            std::shared_ptr<InodeWrapper> inodeWrapper;
+            CURVEFS_ERROR ret = inodeManager_->GetInode(key, inodeWrapper);
+            if (ret != CURVEFS_ERROR::OK) {
+                LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+                           << ", inodeid = " << key;
+                return false;
+            }
+            uint64_t len = inodeWrapper->GetLength();
+            warmupFilelistDeque_.emplace_back(key, len);
+        }
+    }  // Skip already added
+    return true;
+}
+
+bool WarmupManagerS3Impl::AddWarmupFile(fuse_ino_t key,
+                                        const std::string &path) {
+    if (!mounted_.load(std::memory_order_acquire)) {
+        LOG(ERROR) << "not mounted";
+        return false;
+    }
+    // add warmup Progress
+    if (AddWarmupProcess(key)) {
+        VLOG(9) << "add warmup single task:" << key;
+        FetchDentryEnqueue(key, path);
+    }
+    return true;
+}
+
+void WarmupManagerS3Impl::UnInit() {
+    bgFetchStop_.store(true, std::memory_order_release);
+    if (initbgFetchThread_) {
+        bgFetchThread_.join();
+    }
+
+    for (auto &task : inode2FetchDentryPool_) {
+        task.second->Stop();
+    }
+    WriteLockGuard lockDentry(inode2FetchDentryPoolMutex_);
+    inode2FetchDentryPool_.clear();
+
+    for (auto &task : inode2FetchS3ObjectsPool_) {
+        task.second->Stop();
+    }
+    WriteLockGuard lockS3Objects(inode2FetchS3ObjectsPoolMutex_);
+    inode2FetchS3ObjectsPool_.clear();
+
+    WriteLockGuard lockInodes(warmupInodesDequeMutex_);
+    warmupInodesDeque_.clear();
+
+    WriteLockGuard lockFileList(warmupFilelistDequeMutex_);
+    warmupFilelistDeque_.clear();
+
+    WarmupManager::UnInit();
+}
+
+void WarmupManagerS3Impl::Init(const FuseClientOption &option) {
+    WarmupManager::Init(option);
+    bgFetchStop_.store(false, std::memory_order_release);
+    bgFetchThread_ = Thread(&WarmupManagerS3Impl::BackGroundFetch, this);
+    initbgFetchThread_ = true;
+}
+
+void WarmupManagerS3Impl::BackGroundFetch() {
+    while (!bgFetchStop_.load(std::memory_order_acquire)) {
+        usleep(WARMUP_CHECKINTERVAL_US);
+        ScanWarmupFilelist();
+        ScanWarmupInodes();
+        ScanCleanFetchS3ObjectsPool();
+        ScanCleanFetchDentryPool();
+        ScanCleanWarmupProgress();
+    }
+}
+
+void WarmupManagerS3Impl::GetWarmupList(const WarmupFilelist &filelist,
+                                        std::vector<std::string> *list) {
+    struct fuse_file_info fi {};
+    fi.flags &= ~O_DIRECT;
+    size_t rSize = 0;
+    std::unique_ptr<char[]> data(new char[filelist.GetFileLen() + 1]);
+    std::memset(data.get(), 0, filelist.GetFileLen());
+    data[filelist.GetFileLen()] = '\n';
+    fuseOpRead_(nullptr, filelist.GetKey(), filelist.GetFileLen(), 0, &fi,
+                data.get(), &rSize);
+    std::string file = data.get();
+    VLOG(9) << "file is: " << file;
+    // remove enter, newline, blank
+    std::string blanks("\r\n ");
+    file.erase(0, file.find_first_not_of(blanks));
+    file.erase(file.find_last_not_of(blanks) + 1);
+    VLOG(9) << "after del file is: " << file;
+    curve::common::AddSplitStringToResult(file, "\n", list);
+}
+
+void WarmupManagerS3Impl::FetchDentryEnqueue(fuse_ino_t key,
+                                             const std::string &file) {
+    VLOG(9) << "FetchDentryEnqueue start: " << key << " file: " << file;
+    auto task = [this, key, file]() { LookPath(key, file); };
+    AddFetchDentryTask(key, task);
+    VLOG(9) << "FetchDentryEnqueue end: " << key << " file: " << file;
+}
+
+void WarmupManagerS3Impl::LookPath(fuse_ino_t key, std::string file) {
+    VLOG(9) << "LookPath start key: " << key << " file: " << file;
+    std::vector<std::string> splitPath;
+    // remove enter, newline, blank
+    std::string blanks("\r\n ");
+    file.erase(0, file.find_first_not_of(blanks));
+    file.erase(file.find_last_not_of(blanks) + 1);
+    if (file.empty()) {
+        VLOG(9) << "empty path";
+        return;
+    }
+    bool isRoot = false;
+    if (file == "/") {
+        splitPath.push_back(file);
+        isRoot = true;
+    } else {
+        curve::common::AddSplitStringToResult(file, "/", &splitPath);
+    }
+    VLOG(6) << "splitPath size is: " << splitPath.size();
+    if (splitPath.size() == 1 && isRoot) {
+        VLOG(9) << "i am root";
+        auto task = [this, key]() {
+            FetchChildDentry(key, fsInfo_->rootinodeid());
+        };
+        AddFetchDentryTask(key, task);
+        return;
+    } else if (splitPath.size() == 1) {
+        VLOG(9) << "parent is root: " << fsInfo_->rootinodeid()
+                << ", path is: " << splitPath[0];
+        auto task = [this, key, splitPath]() {
+            FetchDentry(key, fsInfo_->rootinodeid(), splitPath[0]);
+        };
+        AddFetchDentryTask(key, task);
+        return;
+    } else if (splitPath.size() > 1) {  // travel path
+        VLOG(9) << "traverse path start: " << splitPath.size();
+        std::string lastName = splitPath.back();
+        splitPath.pop_back();
+        fuse_ino_t ino = fsInfo_->rootinodeid();
+        for (auto iter : splitPath) {
+            VLOG(9) << "traverse path: " << iter << "ino is: " << ino;
+            Dentry dentry;
+            std::string pathName = iter;
+            CURVEFS_ERROR ret =
+                dentryManager_->GetDentry(ino, pathName, &dentry);
+            if (ret != CURVEFS_ERROR::OK) {
+                if (ret != CURVEFS_ERROR::NOTEXIST) {
+                    LOG(WARNING)
+                        << "dentryManager_ get dentry fail, ret = " << ret
+                        << ", parent inodeid = " << ino << ", name = " << file;
+                }
+                VLOG(9) << "FetchDentry error: " << ret;
+                return;
+            }
+            ino = dentry.inodeid();
+        }
+        auto task = [this, key, ino, lastName]() {
+            FetchDentry(key, ino, lastName);
+        };
+        AddFetchDentryTask(key, task);
+        VLOG(9) << "ino is: " << ino << " lastname is: " << lastName;
+        return;
+    } else {
+        VLOG(3) << "unknown path";
+    }
+    VLOG(9) << "LookPath start end: " << key << " file: " << file;
+}
+
+void WarmupManagerS3Impl::FetchDentry(fuse_ino_t key, fuse_ino_t ino,
+                                      const std::string &file) {
+    VLOG(9) << "FetchDentry start: " << file << ", ino: " << ino
+            << " key: " << key;
+    Dentry dentry;
+    CURVEFS_ERROR ret = dentryManager_->GetDentry(ino, file, &dentry);
+    if (ret != CURVEFS_ERROR::OK) {
+        if (ret != CURVEFS_ERROR::NOTEXIST) {
+            LOG(WARNING) << "dentryManager_ get dentry fail, ret = " << ret
+                         << ", parent inodeid = " << ino << ", name = " << file;
+        } else {
+            LOG(ERROR) << "FetchDentry key: " << key << " file: " << file
+                       << " errorCode: " << ret;
+        }
+        return;
+    }
+    if (FsFileType::TYPE_S3 == dentry.type()) {
+        WriteLockGuard lock(warmupInodesDequeMutex_);
+        auto iterDeque = FindKeyWarmupInodesLocked(key);
+        if (iterDeque == warmupInodesDeque_.end()) {
+            warmupInodesDeque_.emplace_back(
+                key, std::set<fuse_ino_t>{dentry.inodeid()});
+        } else {
+            iterDeque->AddFileInode(dentry.inodeid());
+        }
+        return;
+    } else if (FsFileType::TYPE_DIRECTORY == dentry.type()) {
+        auto task = [this, key, dentry]() {
+            FetchChildDentry(key, dentry.inodeid());
+        };
+        AddFetchDentryTask(key, task);
+        VLOG(9) << "FetchDentry: " << dentry.inodeid();
+        return;
+
+    } else if (FsFileType::TYPE_SYM_LINK == dentry.type()) {
+        // skip links
+    } else {
+        VLOG(3) << "unkown, file: " << file << ", ino: " << ino;
+        return;
+    }
+    VLOG(9) << "FetchDentry end: " << file << ", ino: " << ino;
+}
+
+void WarmupManagerS3Impl::FetchChildDentry(fuse_ino_t key, fuse_ino_t ino) {
+    VLOG(9) << "FetchChildDentry start: key:" << key << " inode: " << ino;
+    std::list<Dentry> dentryList;
+    auto limit = option_.listDentryLimit;
+    CURVEFS_ERROR ret = dentryManager_->ListDentry(ino, &dentryList, limit);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "dentryManager_ ListDentry fail, ret = " << ret
+                   << ", parent = " << ino;
+        return;
+    }
+    for (const auto &dentry : dentryList) {
+        VLOG(9) << "FetchChildDentry: key:" << key
+                << " dentry: " << dentry.name();
+        if (FsFileType::TYPE_S3 == dentry.type()) {
+            WriteLockGuard lock(warmupInodesDequeMutex_);
+            auto iterDeque = FindKeyWarmupInodesLocked(key);
+            if (iterDeque == warmupInodesDeque_.end()) {
+                warmupInodesDeque_.emplace_back(
+                    key, std::set<fuse_ino_t>{dentry.inodeid()});
+            } else {
+                iterDeque->AddFileInode(dentry.inodeid());
+            }
+            VLOG(9) << "FetchChildDentry: " << dentry.inodeid();
+        } else if (FsFileType::TYPE_DIRECTORY == dentry.type()) {
+            auto task = [this, key, dentry]() {
+                FetchChildDentry(key, dentry.inodeid());
+            };
+            AddFetchDentryTask(key, task);
+            VLOG(9) << "FetchChildDentry: " << dentry.inodeid();
+        } else if (FsFileType::TYPE_SYM_LINK == dentry.type()) {  // need todo
+        } else {
+            VLOG(9) << "unknown type";
+        }
+    }
+    VLOG(9) << "FetchChildDentry end: key:" << key << " inode: " << ino;
+}
+
+void WarmupManagerS3Impl::FetchDataEnqueue(fuse_ino_t key, fuse_ino_t ino) {
+    VLOG(9) << "FetchDataEnqueue start: key:" << key << " inode: " << ino;
+    auto task = [key, ino, this]() {
+        std::shared_ptr<InodeWrapper> inodeWrapper;
+        CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
+        if (ret != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+                       << ", inodeid = " << ino;
+            return;
+        }
+        S3ChunkInfoMapType s3ChunkInfoMap;
+        {
+            ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+            s3ChunkInfoMap = *inodeWrapper->GetChunkInfoMap();
+        }
+        if (s3ChunkInfoMap.empty()) {
+            return;
+        }
+        TravelChunks(key, ino, s3ChunkInfoMap);
+    };
+    AddFetchS3objectsTask(key, task);
+    VLOG(9) << "FetchDataEnqueue end: key:" << key << " inode: " << ino;
+}
+
+void WarmupManagerS3Impl::TravelChunks(
+    fuse_ino_t key, fuse_ino_t ino, const S3ChunkInfoMapType &s3ChunkInfoMap) {
+    VLOG(9) << "travel chunk start: " << ino
+            << ", size: " << s3ChunkInfoMap.size();
+    for (auto const &infoIter : s3ChunkInfoMap) {
+        VLOG(9) << "travel chunk: " << infoIter.first;
+        std::list<std::pair<std::string, uint64_t>> prefetchObjs;
+        TravelChunk(ino, infoIter.second, &prefetchObjs);
+        {
+            ReadLockGuard lock(inode2ProgressMutex_);
+            auto iter = FindKeyWarmupProgressLocked(key);
+            if (iter != inode2Progress_.end()) {
+                iter->second.AddTotal(prefetchObjs.size());
+            } else {
+                LOG(ERROR) << "no such warmup progress: " << key;
+            }
+        }
+        auto task = [this, key, prefetchObjs]() {
+            WarmUpAllObjs(key, prefetchObjs);
+        };
+        AddFetchS3objectsTask(key, task);
+    }
+    VLOG(9) << "travel chunks end";
+}
+
+void WarmupManagerS3Impl::TravelChunk(fuse_ino_t ino,
+                                      const S3ChunkInfoList &chunkInfo,
+                                      ObjectListType *prefetchObjs) {
+    uint64_t blockSize = s3Adaptor_->GetBlockSize();
+    uint64_t chunkSize = s3Adaptor_->GetChunkSize();
+    uint64_t offset, len, chunkid, compaction;
+    for (size_t i = 0; i < chunkInfo.s3chunks_size(); i++) {
+        auto const &chunkinfo = chunkInfo.s3chunks(i);
+        auto fsId = fsInfo_->fsid();
+        chunkid = chunkinfo.chunkid();
+        compaction = chunkinfo.compaction();
+        offset = chunkinfo.offset();
+        len = chunkinfo.len();
+        // the offset in the chunk
+        uint64_t chunkPos = offset % chunkSize;
+        // the offset in the block
+        uint64_t blockPos = chunkPos % blockSize;
+        // the first blockIndex
+        uint64_t blockIndexBegin = chunkPos / blockSize;
+
+        if (len < blockSize) {  // just one block
+            auto objectName = curvefs::common::s3util::GenObjName(
+                chunkid, blockIndexBegin, compaction, fsId, ino);
+            prefetchObjs->push_back(std::make_pair(objectName, len));
+        } else {
+            // the offset in the block
+            uint64_t blockPos = chunkPos % blockSize;
+
+            // firstly, let's get the size in the first block
+            // then, subtract the length in the first block
+            // to obtain the remaining length
+            // lastly, We need to judge the last block is full or not
+            uint64_t firstBlockSize =
+                (blockPos != 0) ? blockSize - blockPos : blockSize;
+            uint64_t leftSize = len - firstBlockSize;
+            uint32_t blockCounts = (leftSize % blockSize == 0)
+                                       ? (leftSize / blockSize + 1)
+                                       : (leftSize / blockSize + 1 + 1);
+            // so we can get the last blockIndex
+            // because the bolck Index is cumulative
+            uint64_t blockIndexEnd = blockIndexBegin + blockCounts - 1;
+
+            // the size of the last block
+            uint64_t lastBlockSize = leftSize % blockSize;
+            // whether the first block or the last block is full or not
+            bool firstBlockFull = (blockPos == 0);
+            bool lastBlockFull = (lastBlockSize == 0);
+            // the start and end block Index that need travel
+            uint64_t travelStartIndex, travelEndIndex;
+            // if the block is full, the size is needed download
+            // of the obj is blockSize. Otherwise, the value is special.
+            if (!firstBlockFull) {
+                travelStartIndex = blockIndexBegin + 1;
+                auto objectName = curvefs::common::s3util::GenObjName(
+                    chunkid, blockIndexBegin, compaction, fsId, ino);
+                prefetchObjs->push_back(
+                    std::make_pair(objectName, firstBlockSize));
+            } else {
+                travelStartIndex = blockIndexBegin;
+            }
+            if (!lastBlockFull) {
+                // block index is greater than or equal to 0
+                travelEndIndex = (blockIndexEnd == blockIndexBegin)
+                                     ? blockIndexEnd
+                                     : blockIndexEnd - 1;
+                auto objectName = curvefs::common::s3util::GenObjName(
+                    chunkid, blockIndexEnd, compaction, fsId, ino);
+                // there is no need to care about the order
+                // in which objects are downloaded
+                prefetchObjs->push_back(
+                    std::make_pair(objectName, lastBlockSize));
+            } else {
+                travelEndIndex = blockIndexEnd;
+            }
+            VLOG(9) << "travel obj, ino: " << ino << ", chunkid: " << chunkid
+                    << ", blockCounts: " << blockCounts
+                    << ", compaction: " << compaction
+                    << ", blockSize: " << blockSize
+                    << ", chunkSize: " << chunkSize << ", offset: " << offset
+                    << ", blockIndexBegin: " << blockIndexBegin
+                    << ", blockIndexEnd: " << blockIndexEnd << ", len: " << len
+                    << ", firstBlockSize: " << firstBlockSize
+                    << ", lastBlockSize: " << lastBlockSize
+                    << ", blockPos: " << blockPos << ", chunkPos: " << chunkPos;
+            for (auto blockIndex = travelStartIndex;
+                 blockIndex <= travelEndIndex; blockIndex++) {
+                auto objectName = curvefs::common::s3util::GenObjName(
+                    chunkid, blockIndex, compaction, fsId, ino);
+                prefetchObjs->push_back(std::make_pair(objectName, blockSize));
+            }
+        }
+    }
+}
+
+// TODO(hzwuhongsong): These logics are very similar to other place,
+// try to merge it
+void WarmupManagerS3Impl::WarmUpAllObjs(
+    fuse_ino_t key,
+    const std::list<std::pair<std::string, uint64_t>> &prefetchObjs) {
+    std::atomic<uint64_t> pendingReq(0);
+    curve::common::CountDownEvent cond(1);
+    uint64_t start = butil::cpuwide_time_us();
+    // callback function
+    GetObjectAsyncCallBack cb =
+        [&](const S3Adapter *adapter,
+            const std::shared_ptr<GetObjectAsyncContext> &context) {
+            if (bgFetchStop_.load(std::memory_order_acquire)) {
+                VLOG(9) << "need stop warmup";
+                cond.Signal();
+                return;
+            }
+            if (context->retCode == 0) {
+                VLOG(9) << "Get Object success: " << context->key;
+                {
+                    // update progress
+                    ReadLockGuard lock(inode2ProgressMutex_);
+                    auto iter = FindKeyWarmupProgressLocked(key);
+                    if (iter != inode2Progress_.end()) {
+                        iter->second.FinishedPlusOne();
+                    } else {
+                        VLOG(9) << "no such warmup progress: " << key;
+                    }
+                }
+                PutObjectToCache(context->key, context->buf, context->len);
+                CollectMetrics(&warmupS3Metric_.warmupS3Cached, context->len,
+                               start);
+                warmupS3Metric_.warmupS3CacheSize << context->len;
+                if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == 1) {
+                    VLOG(6) << "pendingReq is over";
+                    cond.Signal();
+                }
+                delete[] context->buf;
+                return;
+            }
+            warmupS3Metric_.warmupS3Cached.eps.count << 1;
+            if (++context->retry >= option_.downloadMaxRetryTimes) {
+                if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == 1) {
+                    VLOG(6) << "pendingReq is over";
+                    cond.Signal();
+                }
+                VLOG(9) << "Up to max retry times, "
+                        << "download object failed, key: " << context->key;
+                delete[] context->buf;
+                return;
+            }
+
+            LOG(WARNING) << "Get Object failed, key: " << context->key
+                         << ", offset: " << context->offset;
+            s3Adaptor_->GetS3Client()->DownloadAsync(context);
+        };
+
+    pendingReq.fetch_add(prefetchObjs.size(), std::memory_order_seq_cst);
+    if (pendingReq.load(std::memory_order_seq_cst)) {
+        VLOG(9) << "wait for pendingReq";
+        for (auto iter : prefetchObjs) {
+            VLOG(9) << "download start: " << iter.first;
+            std::string name = iter.first;
+            uint64_t readLen = iter.second;
+            if (s3Adaptor_->GetDiskCacheManager()->IsCached(name)) {
+                pendingReq.fetch_sub(1);
+                continue;
+            }
+            char *cacheS3 = new char[readLen];
+            memset(cacheS3, 0, readLen);
+            auto context = std::make_shared<GetObjectAsyncContext>();
+            context->key = name;
+            context->buf = cacheS3;
+            context->offset = 0;
+            context->len = readLen;
+            context->cb = cb;
+            context->retry = 0;
+            s3Adaptor_->GetS3Client()->DownloadAsync(context);
+        }
+        if (pendingReq.load())
+            cond.Wait();
+    }
+}
+
+bool WarmupManagerS3Impl::ProgressDone(fuse_ino_t key) {
+    bool ret;
+    {
+        ReadLockGuard lockList(warmupFilelistDequeMutex_);
+        ret = FindKeyWarmupFilelistLocked(key) == warmupFilelistDeque_.end();
+    }
+
+    {
+        ReadLockGuard lockDentry(inode2FetchDentryPoolMutex_);
+        ret = ret && (FindKeyFetchDentryPoolLocked(key) ==
+                      inode2FetchDentryPool_.end());
+    }
+
+    {
+        ReadLockGuard lockInodes(warmupInodesDequeMutex_);
+        ret =
+            ret && (FindKeyWarmupInodesLocked(key) == warmupInodesDeque_.end());
+    }
+
+
+    {
+        ReadLockGuard lockS3Objects(inode2FetchS3ObjectsPoolMutex_);
+        ret = ret && (FindKeyFetchS3ObjectsPoolLocked(key) ==
+                      inode2FetchS3ObjectsPool_.end());
+    }
+    return ret;
+}
+
+void WarmupManagerS3Impl::ScanCleanFetchDentryPool() {
+    // clean inode2FetchDentryPool_
+    WriteLockGuard lock(inode2FetchDentryPoolMutex_);
+    for (auto iter = inode2FetchDentryPool_.begin();
+         iter != inode2FetchDentryPool_.end();) {
+        std::deque<WarmupInodes>::iterator iterInode;
+        if (iter->second->QueueSize() == 0) {
+            VLOG(9) << "remove FetchDentry task: " << iter->first;
+            iter->second->Stop();
+            iter = inode2FetchDentryPool_.erase(iter);
+        } else {
+            ++iter;
+        }
+    }
+}
+
+void WarmupManagerS3Impl::ScanCleanFetchS3ObjectsPool() {
+    // clean inode2FetchS3ObjectsPool_
+    WriteLockGuard lock(inode2FetchS3ObjectsPoolMutex_);
+    for (auto iter = inode2FetchS3ObjectsPool_.begin();
+         iter != inode2FetchS3ObjectsPool_.end();) {
+        if (iter->second->QueueSize() == 0) {
+            VLOG(9) << "remove FetchS3object task: " << iter->first;
+            iter->second->Stop();
+            iter = inode2FetchS3ObjectsPool_.erase(iter);
+        } else {
+            ++iter;
+        }
+    }
+}
+
+void WarmupManagerS3Impl::ScanCleanWarmupProgress() {
+    // clean done warmupProgress
+    ReadLockGuard lock(inode2ProgressMutex_);
+    for (auto iter = inode2Progress_.begin(); iter != inode2Progress_.end();) {
+        if (ProgressDone(iter->first)) {
+            VLOG(9) << "warmup key: " << iter->first << " done!";
+            iter = inode2Progress_.erase(iter);
+        } else {
+            ++iter;
+        }
+    }
+}
+
+void WarmupManagerS3Impl::ScanWarmupInodes() {
+    // file need warmup
+    WriteLockGuard lock(warmupInodesDequeMutex_);
+    if (!warmupInodesDeque_.empty()) {
+        WarmupInodes inodes = warmupInodesDeque_.front();
+        for (auto const &iter : inodes.GetReadAheadFiles()) {
+            VLOG(9) << "BackGroundFetch: key: " << inodes.GetKey()
+                    << " inode:" << iter;
+            FetchDataEnqueue(inodes.GetKey(), iter);
+        }
+        warmupInodesDeque_.pop_front();
+    }
+}
+
+void WarmupManagerS3Impl::ScanWarmupFilelist() {
+    // Use a write lock to ensure that all parsing tasks are added.
+    WriteLockGuard lock(warmupFilelistDequeMutex_);
+    if (!warmupFilelistDeque_.empty()) {
+        WarmupFilelist warmupFilelist = warmupFilelistDeque_.front();
+        VLOG(9) << "warmup ino: " << warmupFilelist.GetKey()
+                << " len is: " << warmupFilelist.GetFileLen();
+
+        std::vector<std::string> warmuplist;
+        GetWarmupList(warmupFilelist, &warmuplist);
+        for (auto filePath : warmuplist) {
+            FetchDentryEnqueue(warmupFilelist.GetKey(), filePath);
+        }
+        warmupFilelistDeque_.pop_front();
+    }
+}
+
+void WarmupManagerS3Impl::AddFetchDentryTask(fuse_ino_t key,
+                                             std::function<void()> task) {
+    VLOG(9) << "add fetchDentry task: " << key;
+    if (!bgFetchStop_.load(std::memory_order_acquire)) {
+        WriteLockGuard lock(inode2FetchDentryPoolMutex_);
+        auto iter = inode2FetchDentryPool_.find(key);
+        if (iter == inode2FetchDentryPool_.end()) {
+            std::unique_ptr<ThreadPool> tp = absl::make_unique<ThreadPool>();
+            tp->Start(option_.warmupThreadsNum);
+            iter = inode2FetchDentryPool_.emplace(key, std::move(tp)).first;
+        }
+        if (!iter->second->Enqueue(task)) {
+            LOG(ERROR) << "key:" << key
+                       << " fetch dentry thread pool has been stoped!";
+        }
+        VLOG(9) << "add fetchDentry task: " << key << " finished";
+    }
+}
+
+void WarmupManagerS3Impl::AddFetchS3objectsTask(fuse_ino_t key,
+                                                std::function<void()> task) {
+    VLOG(9) << "add fetchS3Objects task: " << key;
+    if (!bgFetchStop_.load(std::memory_order_acquire)) {
+        WriteLockGuard lock(inode2FetchS3ObjectsPoolMutex_);
+        auto iter = inode2FetchS3ObjectsPool_.find(key);
+        if (iter == inode2FetchS3ObjectsPool_.end()) {
+            std::unique_ptr<ThreadPool> tp = absl::make_unique<ThreadPool>();
+            tp->Start(option_.warmupThreadsNum);
+            iter = inode2FetchS3ObjectsPool_.emplace(key, std::move(tp)).first;
+        }
+        if (!iter->second->Enqueue(task)) {
+            LOG(ERROR) << "key:" << key
+                       << " fetch s3 objects thread pool has been stoped!";
+        }
+        VLOG(9) << "add fetchS3Objects task: " << key << " finished";
+    }
+}
+
+void WarmupManagerS3Impl::PutObjectToCache(const std::string &filename,
+                                           const char *data, uint64_t len) {
+    int ret =
+        s3Adaptor_->GetDiskCacheManager()->WriteReadDirect(filename, data, len);
+    if (ret < 0) {
+        LOG_EVERY_SECOND(INFO)
+            << "write read directly failed, key: " << filename;
+    }
+
+    if (kvClientManager_ != nullptr) {
+        kvClientManager_->Set(
+            std::make_shared<SetKVCacheTask>(filename, data, len));
+    }
+}
+
+void WarmupManager::CollectMetrics(InterfaceMetric *interface, int count,
+                                   uint64_t start) {
+    interface->bps.count << count;
+    interface->qps.count << 1;
+    interface->latency << (butil::cpuwide_time_us() - start);
+}
+
+}  // namespace warmup
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/warmup/warmup_manager.h

@@ -0,0 +1,439 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Created Date: 2023-01-31
+ * Author: chengyi01
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_WARMUP_WARMUP_MANAGER_H_
+#define CURVEFS_SRC_CLIENT_WARMUP_WARMUP_MANAGER_H_
+
+#include <algorithm>
+#include <atomic>
+#include <cstdint>
+#include <deque>
+#include <functional>
+#include <iterator>
+#include <list>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+#include <unordered_map>
+
+#include "curvefs/src/client/dentry_cache_manager.h"
+#include "curvefs/src/client/fuse_common.h"
+#include "curvefs/src/client/inode_cache_manager.h"
+#include "curvefs/src/client/kvclient/kvclient_manager.h"
+#include "curvefs/src/client/rpcclient/metaserver_client.h"
+#include "curvefs/src/client/s3/client_s3_adaptor.h"
+#include "curvefs/src/client/s3/client_s3_cache_manager.h"
+#include "src/common/concurrent/concurrent.h"
+#include "src/common/concurrent/rw_lock.h"
+#include "curvefs/src/common/task_thread_pool.h"
+#include "curvefs/src/client/metric/client_metric.h"
+
+namespace curvefs {
+namespace client {
+namespace warmup {
+
+using common::FuseClientOption;
+
+using ThreadPool = curvefs::common::TaskThreadPool2<bthread::Mutex,
+                                                    bthread::ConditionVariable>;
+using curve::common::BthreadRWLock;
+
+class WarmupFile {
+ public:
+    explicit WarmupFile(fuse_ino_t key = 0, uint64_t fileLen = 0)
+        : key_(key), fileLen_(fileLen) {}
+
+    fuse_ino_t GetKey() const { return key_; }
+    uint64_t GetFileLen() const { return fileLen_; }
+    bool operator==(const WarmupFile &other) const {
+        return key_ == other.key_;
+    }
+
+ private:
+    fuse_ino_t key_;
+    uint64_t fileLen_;
+};
+
+using WarmupFilelist = WarmupFile;
+
+class WarmupInodes {
+ public:
+    explicit WarmupInodes(fuse_ino_t key = 0,
+                          std::set<fuse_ino_t> list = std::set<fuse_ino_t>())
+        : key_(key), readAheadFiles_(std::move(list)) {}
+
+    fuse_ino_t GetKey() const { return key_; }
+    const std::set<fuse_ino_t> &GetReadAheadFiles() const {
+        return readAheadFiles_;
+    }
+
+    void AddFileInode(fuse_ino_t file) { readAheadFiles_.emplace(file); }
+
+ private:
+    fuse_ino_t key_;
+    std::set<fuse_ino_t> readAheadFiles_;
+};
+
+
+using FuseOpReadFunctionType =
+    std::function<CURVEFS_ERROR(fuse_req_t, fuse_ino_t, size_t, off_t,
+                                struct fuse_file_info *, char *, size_t *)>;
+
+class WarmupProgress {
+ public:
+    explicit WarmupProgress(uint64_t total = 0, uint64_t finished = 0)
+        : total_(total), finished_(finished) {}
+
+    WarmupProgress(const WarmupProgress &wp)
+        : total_(wp.total_), finished_(wp.finished_) {}
+
+    void AddTotal(uint64_t add) {
+        std::lock_guard<std::mutex> lock(totalMutex_);
+        total_ += add;
+    }
+
+    WarmupProgress &operator=(const WarmupProgress &wp) {
+        total_ = wp.total_;
+        finished_ = wp.finished_;
+        return *this;
+    }
+
+    void FinishedPlusOne() {
+        std::lock_guard<std::mutex> lock(finishedMutex_);
+        ++finished_;
+    }
+
+    uint64_t GetTotal() {
+        std::lock_guard<std::mutex> lock(totalMutex_);
+        return total_;
+    }
+
+    uint64_t GetFinished() {
+        std::lock_guard<std::mutex> lock(finishedMutex_);
+        return finished_;
+    }
+
+    std::string ToString() {
+        std::lock_guard<std::mutex> lockT(totalMutex_);
+        std::lock_guard<std::mutex> lockF(finishedMutex_);
+        return "total:" + std::to_string(total_) +
+               ",finished:" + std::to_string(finished_);
+    }
+
+ private:
+    uint64_t total_;
+    std::mutex totalMutex_;
+    uint64_t finished_;
+    std::mutex finishedMutex_;
+};
+
+class WarmupManager {
+ public:
+    WarmupManager()
+        : mounted_(false),
+          metaClient_(std::make_shared<MetaServerClientImpl>()),
+          inodeManager_(std::make_shared<InodeCacheManagerImpl>(metaClient_)),
+          dentryManager_(
+              std::make_shared<DentryCacheManagerImpl>(metaClient_)) {
+        kvClientManager_ = nullptr;
+    }
+
+    explicit WarmupManager(std::shared_ptr<MetaServerClient> metaClient,
+                           std::shared_ptr<InodeCacheManager> inodeManager,
+                           std::shared_ptr<DentryCacheManager> dentryManager,
+                           std::shared_ptr<FsInfo> fsInfo,
+                           FuseOpReadFunctionType readFunc,
+                           std::shared_ptr<KVClientManager> kvClientManager)
+        : mounted_(false), metaClient_(std::move(metaClient)),
+          inodeManager_(std::move(inodeManager)),
+          dentryManager_(std::move(dentryManager)), fsInfo_(std::move(fsInfo)),
+          fuseOpRead_(std::move(readFunc)),
+          kvClientManager_(std::move(kvClientManager)) {}
+
+    virtual void Init(const FuseClientOption &option) {
+        option_ = option;
+    }
+    virtual void UnInit() { ClearWarmupProcess(); }
+
+    virtual bool AddWarmupFilelist(fuse_ino_t key) = 0;
+    virtual bool AddWarmupFile(fuse_ino_t key, const std::string &path) = 0;
+
+    void SetMounted(bool mounted) {
+        mounted_.store(mounted, std::memory_order_release);
+    }
+
+    void SetFsInfo(const std::shared_ptr<FsInfo> &fsinfo) { fsInfo_ = fsinfo; }
+
+    void SetFuseOpRead(const FuseOpReadFunctionType &read) {
+        fuseOpRead_ = read;
+    }
+
+    void SetKVClientManager(std::shared_ptr<KVClientManager> kvClientManager) {
+        kvClientManager_ = std::move(kvClientManager);
+    }
+
+    /**
+     * @brief
+     *
+     * @param key
+     * @param progress
+     * @return true
+     * @return false no this warmup task or finished
+     */
+    bool QueryWarmupProgress(fuse_ino_t key, WarmupProgress *progress) {
+        bool ret = true;
+        ReadLockGuard lock(inode2ProgressMutex_);
+        auto iter = FindKeyWarmupProgressLocked(key);
+        if (iter != inode2Progress_.end()) {
+            *progress = iter->second;
+        } else {
+            ret = false;
+        }
+        return ret;
+    }
+
+    virtual void CollectMetrics(InterfaceMetric *interface, int count,
+                                uint64_t start);
+
+ protected:
+    /**
+     * @brief Add warmupProcess
+     *
+     * @return true
+     * @return false warmupProcess has been added
+     */
+    virtual bool AddWarmupProcess(fuse_ino_t key) {
+        WriteLockGuard lock(inode2ProgressMutex_);
+        auto ret = inode2Progress_.emplace(key, WarmupProgress());
+        return ret.second;
+    }
+
+    /**
+     * @brief
+     * Please use it with the lock inode2ProgressMutex_
+     * @param key
+     * @return std::unordered_map<fuse_ino_t, WarmupProgress>::iterator
+     */
+    std::unordered_map<fuse_ino_t, WarmupProgress>::iterator
+    FindKeyWarmupProgressLocked(fuse_ino_t key) {
+        return inode2Progress_.find(key);
+    }
+
+    virtual void ClearWarmupProcess() {
+        WriteLockGuard lock(inode2ProgressMutex_);
+        inode2Progress_.clear();
+    }
+
+ protected:
+    std::atomic<bool> mounted_;
+
+    // metaserver client
+    std::shared_ptr<MetaServerClient> metaClient_;
+
+    // inode cache manager
+    std::shared_ptr<InodeCacheManager> inodeManager_;
+
+    // dentry cache manager
+    std::shared_ptr<DentryCacheManager> dentryManager_;
+
+    // filesystem info
+    std::shared_ptr<FsInfo> fsInfo_;
+
+    // FuseOpRead
+    FuseOpReadFunctionType fuseOpRead_;
+
+    // warmup progress
+    std::unordered_map<fuse_ino_t, WarmupProgress> inode2Progress_;
+    BthreadRWLock inode2ProgressMutex_;
+
+    std::shared_ptr<KVClientManager> kvClientManager_ = nullptr;
+
+    FuseClientOption option_;
+};
+
+class WarmupManagerS3Impl : public WarmupManager {
+ public:
+    explicit WarmupManagerS3Impl(
+        std::shared_ptr<MetaServerClient> metaClient,
+        std::shared_ptr<InodeCacheManager> inodeManager,
+        std::shared_ptr<DentryCacheManager> dentryManager,
+        std::shared_ptr<FsInfo> fsInfo, FuseOpReadFunctionType readFunc,
+        std::shared_ptr<S3ClientAdaptor> s3Adaptor,
+        std::shared_ptr<KVClientManager> kvClientManager)
+        : WarmupManager(std::move(metaClient), std::move(inodeManager),
+                        std::move(dentryManager), std::move(fsInfo),
+                        std::move(readFunc), std::move(kvClientManager)),
+          s3Adaptor_(std::move(s3Adaptor)) {}
+
+    bool AddWarmupFilelist(fuse_ino_t key) override;
+    bool AddWarmupFile(fuse_ino_t key, const std::string &path) override;
+
+    void Init(const FuseClientOption &option) override;
+    void UnInit() override;
+
+ private:
+    void BackGroundFetch();
+
+    void GetWarmupList(const WarmupFilelist &filelist,
+                       std::vector<std::string> *list);
+
+    void FetchDentryEnqueue(fuse_ino_t key, const std::string &file);
+
+    void LookPath(fuse_ino_t key, std::string file);
+
+    void FetchDentry(fuse_ino_t key, fuse_ino_t ino, const std::string &file);
+
+    void FetchChildDentry(fuse_ino_t key, fuse_ino_t ino);
+
+    /**
+     * @brief
+     * Please use it with the lock warmupInodesDequeMutex_
+     * @param key
+     * @return std::deque<WarmupInodes>::iterator
+     */
+    std::deque<WarmupInodes>::iterator
+    FindKeyWarmupInodesLocked(fuse_ino_t key) {
+        return std::find_if(warmupInodesDeque_.begin(),
+                            warmupInodesDeque_.end(),
+                            [key](const WarmupInodes &inodes) {
+                                return key == inodes.GetKey();
+                            });
+    }
+
+    /**
+     * @brief
+     * Please use it with the lock warmupFilelistDequeMutex_
+     * @param key
+     * @return std::deque<WarmupFilelist>::iterator
+     */
+    std::deque<WarmupFilelist>::iterator
+    FindKeyWarmupFilelistLocked(fuse_ino_t key) {
+        return std::find_if(warmupFilelistDeque_.begin(),
+                            warmupFilelistDeque_.end(),
+                            [key](const WarmupFilelist &filelist_) {
+                                return key == filelist_.GetKey();
+                            });
+    }
+
+    /**
+     * @brief
+     * Please use it with the lock inode2FetchDentryPoolMutex_
+     * @param key
+     * @return std::unordered_map<fuse_ino_t,
+     * std::unique_ptr<ThreadPool>>::iterator
+     */
+    std::unordered_map<fuse_ino_t, std::unique_ptr<ThreadPool>>::iterator
+    FindKeyFetchDentryPoolLocked(fuse_ino_t key) {
+        return inode2FetchDentryPool_.find(key);
+    }
+
+    /**
+     * @brief
+     * Please use it with the lock inode2FetchS3ObjectsPoolMutex_
+     * @param key
+     * @return std::unordered_map<fuse_ino_t,
+     * std::unique_ptr<ThreadPool>>::iterator
+     */
+    std::unordered_map<fuse_ino_t, std::unique_ptr<ThreadPool>>::iterator
+    FindKeyFetchS3ObjectsPoolLocked(fuse_ino_t key) {
+        return inode2FetchS3ObjectsPool_.find(key);
+    }
+
+    void FetchDataEnqueue(fuse_ino_t key, fuse_ino_t ino);
+
+    using S3ChunkInfoMapType = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
+
+    // travel all chunks
+    void TravelChunks(fuse_ino_t key, fuse_ino_t ino,
+                      const S3ChunkInfoMapType &s3ChunkInfoMap);
+
+    using ObjectListType = std::list<std::pair<std::string, uint64_t>>;
+    // travel and download all objs belong to the chunk
+    void TravelChunk(fuse_ino_t ino, const S3ChunkInfoList &chunkInfo,
+                     ObjectListType *prefetchObjs);
+
+    // warmup all the prefetchObjs
+    void WarmUpAllObjs(
+        fuse_ino_t key,
+        const std::list<std::pair<std::string, uint64_t>> &prefetchObjs);
+
+    /**
+     * @brief Whether the warmup task[key] is completed (or terminated)
+     *
+     * @return true
+     * @return false
+     */
+    bool ProgressDone(fuse_ino_t key);
+
+    void ScanCleanFetchDentryPool();
+
+    void ScanCleanFetchS3ObjectsPool();
+
+    void ScanCleanWarmupProgress();
+
+    void ScanWarmupInodes();
+
+    void ScanWarmupFilelist();
+
+    void AddFetchDentryTask(fuse_ino_t key, std::function<void()> task);
+
+    void AddFetchS3objectsTask(fuse_ino_t key, std::function<void()> task);
+
+    void PutObjectToCache(const std::string &filename, const char *data,
+                          uint64_t len);
+
+ protected:
+    std::deque<WarmupFilelist> warmupFilelistDeque_;
+    mutable RWLock warmupFilelistDequeMutex_;
+
+    bool initbgFetchThread_ = false;
+    Thread bgFetchThread_;
+    std::atomic<bool> bgFetchStop_;
+
+    // TODO(chengyi01): limit thread nums
+    std::unordered_map<fuse_ino_t, std::unique_ptr<ThreadPool>>
+        inode2FetchDentryPool_;
+    mutable RWLock inode2FetchDentryPoolMutex_;
+
+    std::deque<WarmupInodes> warmupInodesDeque_;
+    mutable RWLock warmupInodesDequeMutex_;
+
+    // s3 adaptor
+    std::shared_ptr<S3ClientAdaptor> s3Adaptor_;
+
+    // TODO(chengyi01): limit thread nums
+    std::unordered_map<fuse_ino_t, std::unique_ptr<ThreadPool>>
+        inode2FetchS3ObjectsPool_;
+    mutable RWLock inode2FetchS3ObjectsPoolMutex_;
+
+    curvefs::client::metric::WarmupManagerS3Metric warmupS3Metric_;
+};
+
+}  // namespace warmup
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_WARMUP_WARMUP_MANAGER_H_

curvefs/src/client/xattr_manager.h

@@ -36,7 +36,7 @@
 
 #include "curvefs/src/common/define.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/client_operator.h"
 #include "src/common/interruptible_sleeper.h"
 

curvefs/src/common/task_thread_pool.h

@@ -0,0 +1,99 @@
+/*
+*  Copyright (c) 2023 NetEase Inc.
+*
+*  Licensed under the Apache License, Version 2.0 (the "License");
+*  you may not use this file except in compliance with the License.
+*  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+*  Unless required by applicable law or agreed to in writing, software
+*  distributed under the License is distributed on an "AS IS" BASIS,
+*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*  See the License for the specific language governing permissions and
+*  limitations under the License.
+*/
+/*
+ * Project: curve
+ * Created Date: 2023-02-14
+ * Author: chengyi01
+ */
+
+#ifndef CURVEFS_SRC_COMMON_TASK_THREAD_POOL_H_
+#define CURVEFS_SRC_COMMON_TASK_THREAD_POOL_H_
+
+#include "src/common/concurrent/task_thread_pool.h"
+
+#include <condition_variable>
+#include <mutex>
+#include <utility>
+
+namespace curvefs {
+namespace common {
+
+template <typename MutexT = std::mutex,
+          typename CondVarT = std::condition_variable>
+class TaskThreadPool2 : public curve::common::TaskThreadPool<MutexT, CondVarT> {
+    using Base = curve::common::TaskThreadPool<MutexT, CondVarT>;
+
+ public:
+    template <class F, class... Args> bool Enqueue(F &&f, Args &&...args) {
+        std::unique_lock<MutexT> guard(Base::mutex_);
+
+        if (!Base::running_.load(std::memory_order_acquire)) {
+            // When stopped, running_ false recovery
+            return false;
+        }
+
+        while (Base::IsFullUnlock()) {
+            Base::notFull_.wait(guard);
+        }
+        auto task = std::bind(std::forward<F>(f), std::forward<Args>(args)...);
+        Base::queue_.push_back(std::move(task));
+        Base::notEmpty_.notify_one();
+        return true;
+    }
+
+    curve::common::Task Take() {
+        std::unique_lock<MutexT> guard(Base::mutex_);
+        while (Base::queue_.empty() &&
+               Base::running_.load(std::memory_order_acquire)) {
+            Base::notEmpty_.wait(guard);
+        }
+        curve::common::Task task;
+        if (!Base::queue_.empty()) {
+            task = std::move(Base::queue_.front());
+            Base::queue_.pop_front();
+            Base::notFull_.notify_one();
+            std::unique_lock<MutexT> guard(executingMutex_);
+            ++executing_;
+        }
+        return task;
+    }
+
+    virtual void ThreadFunc() {
+        while (Base::running_.load(std::memory_order_acquire)) {
+            curve::common::Task task(Take());
+            if (task) {
+                task();
+                std::unique_lock<MutexT> guard(executingMutex_);
+                --executing_;
+            }
+        }
+    }
+
+    int QueueSize() const {
+        std::lock_guard<MutexT> guard(Base::mutex_);
+        std::lock_guard<MutexT> guardExecuting(executingMutex_);
+        return Base::queue_.size() + executing_;
+    }
+
+ protected:
+    int executing_ = 0;
+    mutable MutexT executingMutex_;
+};
+
+}  // namespace common
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_COMMON_TASK_THREAD_POOL_H_

curvefs/src/mds/fs_manager.cpp

@@ -263,11 +263,6 @@ FSStatusCode FsManager::CreateFs(const ::curvefs::mds::CreateFsRequest* request,
     const auto& fsType = request->fstype();
     const auto& detail = request->fsdetail();
 
-    // check fsname
-    if (!CheckFsName(fsName)) {
-        return FSStatusCode::FSNAME_INVALID;
-    }
-
     NameLockGuard lock(nameLock_, fsName);
     FsInfoWrapper wrapper;
     bool skipCreateNewFs = false;
@@ -299,6 +294,11 @@ FSStatusCode FsManager::CreateFs(const ::curvefs::mds::CreateFsRequest* request,
         }
     }
 
+    // check fsname
+    if (!CheckFsName(fsName)) {
+        return FSStatusCode::FSNAME_INVALID;
+    }
+
     // check s3info
     if (!skipCreateNewFs && detail.has_s3info()) {
         const auto& s3Info = detail.s3info();

curvefs/src/mds/topology/topology.cpp

@@ -1735,12 +1735,12 @@ TopoStatusCode TopologyImpl::AllocOrGetMemcacheCluster(
         int randId =
             static_cast<int>(butil::fast_rand()) % memcacheClusterMap_.size();
         auto iter = memcacheClusterMap_.cbegin();
-        for (int i = 0; i < randId; ++i, ++iter) continue;
-        *cluster = iter->second;
-        if (!storage_->StorageFs2MemcacheCluster(fsId, cluster->clusterid())) {
+        std::advance(iter, randId);
+        if (!storage_->StorageFs2MemcacheCluster(fsId, iter->first)) {
             ret = TopoStatusCode::TOPO_STORGE_FAIL;
         } else {
-            fs2MemcacheCluster_[fsId] = cluster->clusterid();
+            fs2MemcacheCluster_[fsId] = iter->first;
+            *cluster = iter->second;
         }
     }
     return ret;

curvefs/src/mds/topology/topology_item.h

@@ -823,6 +823,8 @@ class MemcacheServer {
     MemcacheServer() : port_(0) {}
     explicit MemcacheServer(const MemcacheServerInfo& info)
         : ip_(info.ip()), port_(info.port()) {}
+    explicit MemcacheServer(const std::string&& ip, uint32_t port)
+        : ip_(ip), port_(port) {}
 
     MemcacheServer& operator=(const MemcacheServerInfo& info) {
         ip_ = info.ip();
@@ -888,6 +890,19 @@ class MemcacheCluster {
         return info;
     }
 
+    bool operator==(const MemcacheCluster& rhs) const {
+        if (rhs.id_ != id_ || rhs.servers_.size() != servers_.size()) {
+            return false;
+        }
+        for (auto const& server : servers_) {
+            if (std::find(rhs.servers_.cbegin(), rhs.servers_.cend(), server) ==
+                rhs.servers_.cend()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     std::list<MemcacheServer> GetServers() const {
         return servers_;
     }
@@ -899,6 +914,10 @@ class MemcacheCluster {
     bool ParseFromString(const std::string& value);
     bool SerializeToString(std::string* value) const;
 
+    void SetId(MetaServerIdType id) {
+        id_ = id;
+    }
+
  private:
     MetaServerIdType id_;
     std::list<MemcacheServer> servers_;

curvefs/src/mds/topology/topology_manager.cpp

@@ -1327,6 +1327,20 @@ void TopologyManager::RegistMemcacheCluster(
     // register memcacheCluster as server
     WriteLockGuard lock(registMemcacheClusterMutex_);
 
+    // idempotence
+    std::list<MemcacheCluster> clusterList = topology_->ListMemcacheClusters();
+    MemcacheCluster mCluster(
+        0, std::list<MemcacheServer>(request->servers().begin(),
+                                     request->servers().end()));
+    for (auto const& cluster : clusterList) {
+        mCluster.SetId(cluster.GetId());
+        if (cluster == mCluster) {
+            // has registered memcache cluster
+            response->set_clusterid(cluster.GetId());
+            return;
+        }
+    }
+
     // Guarantee the uniqueness of memcacheServer
     std::list<MemcacheServer> serverRegisted = topology_->ListMemcacheServers();
     std::list<MemcacheServer> serverList;

curvefs/src/mds/topology/topology_storage_codec.cpp

@@ -22,7 +22,9 @@
 #include "curvefs/src/mds/topology/topology_storage_codec.h"
 
 #include <string>
+#include <utility>
 #include "curvefs/src/mds/common/storage_key.h"
+#include "src/common/encode.h"
 
 namespace curvefs {
 namespace mds {
@@ -178,6 +180,13 @@ std::string TopologyStorageCodec::EncodeFs2MemcacheClusterKey(FsIdType fsId) {
     return key;
 }
 
+bool TopologyStorageCodec::DecodeFs2MemcacheClusterKey(const std::string& value,
+                                                       FsIdType* data) {
+    size_t prefixLen = TOPOLOGY_PREFIX_LENGTH;
+    *data = curve::common::DecodeBigEndian(&(value[prefixLen]));
+    return true;
+}
+
 }  // namespace topology
 }  // namespace mds
 }  // namespace curvefs

curvefs/src/mds/topology/topology_storage_codec.h

@@ -89,6 +89,7 @@ class TopologyStorageCodec {
                                   MemcacheCluster* data);
 
     std::string EncodeFs2MemcacheClusterKey(FsIdType fsId);
+    bool DecodeFs2MemcacheClusterKey(const std::string& value, FsIdType* data);
 };
 
 

curvefs/src/mds/topology/topology_storge_etcd.cpp

@@ -21,6 +21,7 @@
  */
 #include "curvefs/src/mds/topology/topology_storge_etcd.h"
 
+#include <exception>
 #include <string>
 #include <vector>
 #include <map>
@@ -625,8 +626,18 @@ bool TopologyStorageEtcd::LoadFs2MemcacheCluster(
         return false;
     }
     for (auto const& data : out) {
-        fs2MemcacheCluster->emplace(
-            std::make_pair(std::stoul(data.first), std::stoul(data.second)));
+        FsIdType id;
+        codec_->DecodeFs2MemcacheClusterKey(data.first, &id);
+        if (fs2MemcacheCluster->find(id) != fs2MemcacheCluster->end()) {
+            // Duplicated id
+            return false;
+        }
+        try {
+            fs2MemcacheCluster->emplace(id, std::stoul(data.second));
+        } catch (std::exception) {
+            // for std::stoul(data.second) exceptions
+            return false;
+        }
     }
     return true;
 }

curvefs/test/client/chunk_cache_manager_test.cpp

@@ -56,12 +56,12 @@ class ChunkCacheManagerTest : public testing::Test {
         option.diskCacheOpt.diskCacheType = (DiskCacheType)0;
         s3ClientAdaptor_ = new S3ClientAdaptorImpl();
         auto fsCacheManager_ = std::make_shared<FsCacheManager>(
-            s3ClientAdaptor_, option.readCacheMaxByte,
-            option.writeCacheMaxByte);
+            s3ClientAdaptor_, option.readCacheMaxByte, option.writeCacheMaxByte,
+            nullptr);
         s3ClientAdaptor_->Init(option, nullptr, nullptr, nullptr,
-                               fsCacheManager_, nullptr);
-        chunkCacheManager_ =
-            std::make_shared<ChunkCacheManager>(index, s3ClientAdaptor_);
+                               fsCacheManager_, nullptr, nullptr);
+        chunkCacheManager_ = std::make_shared<ChunkCacheManager>(
+            index, s3ClientAdaptor_, nullptr);
     }
     void TearDown() override {
         delete s3ClientAdaptor_;
@@ -91,20 +91,20 @@ TEST_F(ChunkCacheManagerTest, test_add_read_data_cache) {
     uint64_t len = 1024 * 1024;
     char *buf = new char[len];
     auto dataCache = std::make_shared<DataCache>(
-        s3ClientAdaptor_, chunkCacheManager_, offset, len, buf);
+        s3ClientAdaptor_, chunkCacheManager_, offset, len, buf, nullptr);
     VLOG(0) << "wghs001";
     chunkCacheManager_->AddReadDataCache(dataCache);
     VLOG(0) << "wghs002";
     ASSERT_EQ(len, s3ClientAdaptor_->GetFsCacheManager()->GetLruByte());
     offset = 2 * 1024 * 1024;
     auto dataCache1 = std::make_shared<DataCache>(
-        s3ClientAdaptor_, chunkCacheManager_, offset, len, buf);
+        s3ClientAdaptor_, chunkCacheManager_, offset, len, buf, nullptr);
     chunkCacheManager_->AddReadDataCache(dataCache1);
     ASSERT_EQ(2 * len, s3ClientAdaptor_->GetFsCacheManager()->GetLruByte());
     offset = 0;
     len = 512 * 1024;
     auto dataCache2 = std::make_shared<DataCache>(
-        s3ClientAdaptor_, chunkCacheManager_, offset, len, buf);
+        s3ClientAdaptor_, chunkCacheManager_, offset, len, buf, nullptr);
     chunkCacheManager_->AddReadDataCache(dataCache2);
     ASSERT_EQ(1.5 * 1024 * 1024,
               s3ClientAdaptor_->GetFsCacheManager()->GetLruByte());
@@ -236,7 +236,7 @@ TEST_F(ChunkCacheManagerTest, test_read_by_read_cache) {
     requests.clear();
 
     auto dataCache = std::make_shared<DataCache>(
-        s3ClientAdaptor_, chunkCacheManager_, 1024, len, dataCacheBuf);
+        s3ClientAdaptor_, chunkCacheManager_, 1024, len, dataCacheBuf, nullptr);
     chunkCacheManager_->AddReadDataCache(dataCache);
     chunkCacheManager_->ReadByReadCache(offset, len, buf, 0, &requests);
     ASSERT_EQ(1, requests.size());
@@ -311,7 +311,7 @@ TEST_F(ChunkCacheManagerTest, test_flush) {
     uint64_t len = 1024 * 1024;
     char *buf = new char[len];
     auto dataCache = std::make_shared<MockDataCache>(
-        s3ClientAdaptor_, chunkCacheManager_, offset, len, buf);
+        s3ClientAdaptor_, chunkCacheManager_, offset, len, buf, nullptr);
     EXPECT_CALL(*dataCache, Flush(_, _))
         .WillOnce(Return(CURVEFS_ERROR::OK))
         .WillOnce(Return(CURVEFS_ERROR::INTERNAL))
@@ -341,7 +341,7 @@ TEST_F(ChunkCacheManagerTest, test_release_read_dataCache) {
     uint64_t len = 1024 * 1024;
     char *buf = new char[len];
     auto dataCache = std::make_shared<DataCache>(
-        s3ClientAdaptor_, chunkCacheManager_, offset, len, buf);
+        s3ClientAdaptor_, chunkCacheManager_, offset, len, buf, nullptr);
     chunkCacheManager_->AddReadDataCache(dataCache);
     chunkCacheManager_->ReleaseReadDataCache(offset);
     ASSERT_EQ(true, chunkCacheManager_->IsEmpty());
@@ -355,7 +355,7 @@ TEST_F(ChunkCacheManagerTest, test_truncate_cache) {
     uint64_t len = 1024 * 1024;
     char *buf = new char[len];
     auto dataCache = std::make_shared<DataCache>(
-        s3ClientAdaptor_, chunkCacheManager_, offset, len, buf);
+        s3ClientAdaptor_, chunkCacheManager_, offset, len, buf, nullptr);
     /*EXPECT_CALL(*dataCache, Truncate(_))
         .WillOnce(Return());*/
     chunkCacheManager_->AddWriteDataCacheForTest(dataCache);

curvefs/test/client/client_memcache_test.cpp

@@ -73,6 +73,7 @@ class MemCachedTest : public ::testing::Test {
             std::this_thread::sleep_for(std::chrono::milliseconds(100));
             retry++;
         } while (1);
+        LOG(INFO) << "=============== memcache start ok";
     }
 
     void TearDown() {
@@ -111,16 +112,24 @@ TEST_F(MemCachedTest, MultiThreadTask) {
         });
     }
     taskEnvent.Wait();
-    ASSERT_EQ(5, g_kvClientMetric->kvClientSet.latency.count());
+    ASSERT_EQ(
+        5, manager_.GetClientMetricForTesting()->kvClientSet.latency.count());
 
     // get
     for (int i = 0; i < 5; i++) {
         workers.emplace_back([&, i]() {
+            CountDownEvent taskEnvent(1);
             char *result = new char[4];
-            auto context = std::make_shared<GetKvCacheContext>(kvstr[i].first,
-                                                               result, 0, 4);
-            ASSERT_EQ(true, manager_.Get(context));
+            auto task =
+                std::make_shared<GetKVCacheTask>(kvstr[i].first, result, 0, 4);
+            task->done =
+                [&taskEnvent](const std::shared_ptr<GetKVCacheTask> &task) {
+                    taskEnvent.Signal();
+                };
+            manager_.Get(task);
+            taskEnvent.Wait();
             ASSERT_EQ(0, memcmp(result, kvstr[i].second.c_str(), 4));
+            ASSERT_TRUE(task->res);
         });
     }
     for (auto &iter : workers) {

curvefs/test/client/client_s3_adaptor_Integration.cpp

@@ -23,6 +23,7 @@
 #include <brpc/server.h>
 #include <google/protobuf/util/message_differencer.h>
 #include <gtest/gtest.h>
+#include <memory>
 
 #include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
@@ -123,6 +124,7 @@ class ClientS3IntegrationTest : public testing::Test {
  protected:
     void SetUp() override {
         Aws::InitAPI(awsOptions_);
+        InitKVClientManager();
         ASSERT_EQ(0, server_.AddService(&mockMetaServerService_,
                                         brpc::SERVER_DOESNT_OWN_SERVICE));
         ASSERT_EQ(0, server_.Start(addr_.c_str(), nullptr));
@@ -143,9 +145,10 @@ class ClientS3IntegrationTest : public testing::Test {
         s3ClientAdaptor_ = new S3ClientAdaptorImpl();
         auto fsCacheManager = std::make_shared<FsCacheManager>(
             s3ClientAdaptor_, option.readCacheMaxByte,
-            option.writeCacheMaxByte);
+            option.writeCacheMaxByte, kvClientManager_);
         s3ClientAdaptor_->Init(option, mockS3Client, mockInodeManager,
-                               mockMdsClient, fsCacheManager, nullptr);
+                               mockMdsClient, fsCacheManager, nullptr,
+                               kvClientManager_);
         s3ClientAdaptor_->SetFsId(2);
         curvefs::client::common::FLAGS_enableCto = false;
     }
@@ -157,11 +160,11 @@ class ClientS3IntegrationTest : public testing::Test {
     }
 
     void InitKVClientManager() {
-        g_kvClientManager = new KVClientManager();
+        kvClientManager_ = std::make_shared<KVClientManager>();
 
         KVClientManagerOpt opt;
         std::shared_ptr<MockKVClient> mockKVClient(&mockKVClient_);
-        g_kvClientManager->Init(opt, mockKVClient);
+        kvClientManager_->Init(opt, mockKVClient);
     }
 
  protected:
@@ -176,6 +179,8 @@ class ClientS3IntegrationTest : public testing::Test {
     Aws::SDKOptions awsOptions_;
 
     std::shared_ptr<InodeWrapper> inode{InitInodeForIntegration()};
+
+    std::shared_ptr<KVClientManager> kvClientManager_;
 };
 
 TEST_F(ClientS3IntegrationTest, test_first_write) {
@@ -2834,8 +2839,6 @@ TEST_F(ClientS3IntegrationTest, test_fssync_overlap_write) {
 }
 
 TEST_F(ClientS3IntegrationTest, test_write_read_remotekvcache) {
-    InitKVClientManager();
-
     curvefs::client::common::FLAGS_enableCto = true;
 
     uint64_t offset_0 = 0, offset_4M = (4 << 20), chunkId = 10;

curvefs/test/client/client_s3_adaptor_test.cpp

@@ -47,7 +47,7 @@ using ::testing::WithArg;
 
 using rpcclient::MockMdsClient;
 
-extern KVClientManager *g_kvClientManager;
+// extern KVClientManager *g_kvClientManager;
 
 class ClientS3AdaptorTest : public testing::Test {
  protected:
@@ -72,10 +72,10 @@ class ClientS3AdaptorTest : public testing::Test {
         option.fuseMaxSize = 131072;
         option.chunkFlushThreads = 5;
         option.diskCacheOpt.diskCacheType = (DiskCacheType)0;
+        kvClientManager_ = nullptr;
         s3ClientAdaptor_->Init(option, mockS3Client_, mockInodeManager_,
                                mockMdsClient_, mockFsCacheManager_,
-                               mockDiskcacheManagerImpl_);
-        g_kvClientManager = nullptr;
+                               mockDiskcacheManagerImpl_, kvClientManager_);
     }
 
     void TearDown() override {
@@ -88,6 +88,7 @@ class ClientS3AdaptorTest : public testing::Test {
     std::shared_ptr<MockFsCacheManager> mockFsCacheManager_;
     std::shared_ptr<MockS3Client> mockS3Client_;
     std::shared_ptr<MockMdsClient> mockMdsClient_;
+    std::shared_ptr<KVClientManager> kvClientManager_;
 };
 
 uint64_t gInodeId = 1;

curvefs/test/client/data_cache_test.cpp

@@ -54,15 +54,16 @@ class DataCacheTest : public testing::Test {
         s3ClientAdaptor_ = new S3ClientAdaptorImpl();
         auto fsCacheManager = std::make_shared<FsCacheManager>(
             s3ClientAdaptor_, option.readCacheMaxByte,
-            option.writeCacheMaxByte);
+            option.writeCacheMaxByte, nullptr);
         s3ClientAdaptor_->Init(option, nullptr, nullptr, nullptr,
-                               fsCacheManager, nullptr);
+                               fsCacheManager, nullptr, nullptr);
         mockChunkCacheManager_ = std::make_shared<MockChunkCacheManager>();
         uint64_t offset = 512 * 1024;
         uint64_t len = 1024 * 1024;
         char *buf = new char[len];
-        dataCache_ = std::make_shared<DataCache>(
-            s3ClientAdaptor_, mockChunkCacheManager_, offset, len, buf);
+        dataCache_ = std::make_shared<DataCache>(s3ClientAdaptor_,
+                                                 mockChunkCacheManager_, offset,
+                                                 len, buf, nullptr);
         delete buf;
     }
     void TearDown() override {}
@@ -102,7 +103,7 @@ TEST_F(DataCacheTest, test_write3) {
     std::vector<DataCachePtr> mergeDataCacheVer;
     uint64_t offset1 = len;
     auto dataCache = std::make_shared<DataCache>(
-        s3ClientAdaptor_, mockChunkCacheManager_, offset1, len, buf);
+        s3ClientAdaptor_, mockChunkCacheManager_, offset1, len, buf, nullptr);
     mergeDataCacheVer.push_back(dataCache);
     dataCache_->Write(offset, len, buf, mergeDataCacheVer);
     ASSERT_EQ(0, dataCache_->GetChunkPos());
@@ -139,7 +140,7 @@ TEST_F(DataCacheTest, test_write6) {
     std::vector<DataCachePtr> mergeDataCacheVer;
     uint64_t offset1 = offset + len;
     auto dataCache = std::make_shared<DataCache>(
-        s3ClientAdaptor_, mockChunkCacheManager_, offset1, len, buf);
+        s3ClientAdaptor_, mockChunkCacheManager_, offset1, len, buf, nullptr);
     mergeDataCacheVer.push_back(dataCache);
     dataCache_->Write(offset, len, buf, mergeDataCacheVer);
     ASSERT_EQ(512 * 1024, dataCache_->GetChunkPos());

curvefs/test/client/file_cache_manager_test.cpp

@@ -48,7 +48,7 @@ using ::testing::SetArgPointee;
 using ::testing::SetArgReferee;
 using ::testing::WithArg;
 
-extern KVClientManager *g_kvClientManager;
+// extern KVClientManager *g_kvClientManager;
 
 class FileCacheManagerTest : public testing::Test {
  protected:
@@ -69,18 +69,18 @@ class FileCacheManagerTest : public testing::Test {
         option.chunkFlushThreads = 5;
         s3ClientAdaptor_ = new S3ClientAdaptorImpl();
         auto fsCacheManager_ = std::make_shared<FsCacheManager>(
-            s3ClientAdaptor_, option.readCacheMaxByte,
-            option.writeCacheMaxByte);
+            s3ClientAdaptor_, option.readCacheMaxByte, option.writeCacheMaxByte,
+            nullptr);
         mockInodeManager_ = std::make_shared<MockInodeCacheManager>();
         mockS3Client_ = std::make_shared<MockS3Client>();
         s3ClientAdaptor_->Init(option, mockS3Client_, mockInodeManager_,
-                               nullptr, fsCacheManager_, nullptr);
+                               nullptr, fsCacheManager_, nullptr, nullptr);
         s3ClientAdaptor_->SetFsId(fsId);
-        fileCacheManager_ =
-            std::make_shared<FileCacheManager>(fsId, inodeId, s3ClientAdaptor_);
+        fileCacheManager_ = std::make_shared<FileCacheManager>(
+            fsId, inodeId, s3ClientAdaptor_, nullptr);
         mockChunkCacheManager_ = std::make_shared<MockChunkCacheManager>();
         curvefs::client::common::FLAGS_enableCto = false;
-        g_kvClientManager = nullptr;
+        kvClientManager_ = nullptr;
     }
 
     void TearDown() override {
@@ -95,6 +95,7 @@ class FileCacheManagerTest : public testing::Test {
     std::shared_ptr<MockChunkCacheManager> mockChunkCacheManager_;
     std::shared_ptr<MockInodeCacheManager> mockInodeManager_;
     std::shared_ptr<MockS3Client> mockS3Client_;
+    std::shared_ptr<KVClientManager> kvClientManager_;
 };
 
 TEST_F(FileCacheManagerTest, test_FindOrCreateChunkCacheManager) {
@@ -159,8 +160,8 @@ TEST_F(FileCacheManagerTest, test_old_write) {
     uint64_t len = 1024;
     char buf[len] = {0};
 
-    auto dataCache = std::make_shared<MockDataCache>(s3ClientAdaptor_, nullptr,
-                                                     offset, 0, nullptr);
+    auto dataCache = std::make_shared<MockDataCache>(
+        s3ClientAdaptor_, nullptr, offset, 0, nullptr, nullptr);
     EXPECT_CALL(*dataCache, Write(_, _, _, _)).WillOnce(Return());
     EXPECT_CALL(*mockChunkCacheManager_, FindWriteableDataCache(_, _, _, _))
         .WillOnce(Return(dataCache));

curvefs/test/client/fs_cache_manager_test.cpp

@@ -56,9 +56,9 @@ class FsCacheManagerTest : public testing::Test {
         option.chunkFlushThreads = 5;
         s3ClientAdaptor_ = new S3ClientAdaptorImpl();
         fsCacheManager_ = std::make_shared<FsCacheManager>(
-            s3ClientAdaptor_, maxReadCacheByte_, maxWriteCacheByte);
+            s3ClientAdaptor_, maxReadCacheByte_, maxWriteCacheByte, nullptr);
         s3ClientAdaptor_->Init(option, nullptr, nullptr, nullptr,
-                               fsCacheManager_, nullptr);
+                               fsCacheManager_, nullptr, nullptr);
         s3ClientAdaptor_->SetFsId(2);
 
         mockChunkCacheManager_ = std::make_shared<MockChunkCacheManager>();
@@ -118,7 +118,7 @@ TEST_F(FsCacheManagerTest, test_lru_set_and_delete) {
         for (size_t i = 0; i < maxReadCacheByte_ / smallDataCacheByte; ++i) {
             fsCacheManager_->Set(std::make_shared<DataCache>(
                                      s3ClientAdaptor_, mockChunkCacheManager_,
-                                     0, smallDataCacheByte, buf),
+                                     0, smallDataCacheByte, buf, nullptr),
                                  &outIter);
         }
     }
@@ -130,9 +130,9 @@ TEST_F(FsCacheManagerTest, test_lru_set_and_delete) {
         EXPECT_CALL(*mockChunkCacheManager_, ReleaseReadDataCache(_))
             .Times(expectCallTimes)
             .WillRepeatedly(Invoke([&counter](uint64_t) { counter.Signal(); }));
-        fsCacheManager_->Set(std::make_shared<DataCache>(s3ClientAdaptor_,
-                                                         mockChunkCacheManager_,
-                                                         0, dataCacheByte, buf),
+        fsCacheManager_->Set(std::make_shared<DataCache>(
+                                 s3ClientAdaptor_, mockChunkCacheManager_, 0,
+                                 dataCacheByte, buf, nullptr),
                              &outIter);
 
         counter.Wait();
@@ -146,9 +146,9 @@ TEST_F(FsCacheManagerTest, test_lru_set_and_delete) {
             .Times(expectCallTimes)
             .WillRepeatedly(Invoke([&counter](uint64_t) { counter.Signal(); }));
 
-        fsCacheManager_->Set(std::make_shared<DataCache>(s3ClientAdaptor_,
-                                                         mockChunkCacheManager_,
-                                                         0, dataCacheByte, buf),
+        fsCacheManager_->Set(std::make_shared<DataCache>(
+                                 s3ClientAdaptor_, mockChunkCacheManager_, 0,
+                                 dataCacheByte, buf, nullptr),
                              &outIter);
         counter.Wait();
     }

curvefs/test/client/mock_client_s3_adaptor.h

@@ -23,6 +23,8 @@
 #ifndef CURVEFS_TEST_CLIENT_MOCK_CLIENT_S3_ADAPTOR_H_
 #define CURVEFS_TEST_CLIENT_MOCK_CLIENT_S3_ADAPTOR_H_
 
+#include <gmock/gmock.h>
+
 #include <memory>
 #include <string>
 
@@ -34,7 +36,7 @@ namespace client {
 
 class MockS3ClientAdaptor : public S3ClientAdaptor {
  public:
-    MOCK_METHOD7(Init,
+    MOCK_METHOD8(Init,
                  CURVEFS_ERROR(const S3ClientAdaptorOption &option,
                                std::shared_ptr<S3Client> client,
                                std::shared_ptr<InodeCacheManager> inodeManager,
@@ -42,6 +44,8 @@ class MockS3ClientAdaptor : public S3ClientAdaptor {
                                std::shared_ptr<FsCacheManager> fsCacheManager,
                                std::shared_ptr<DiskCacheManagerImpl>
                                    diskCacheManagerImpl,
+                                std::shared_ptr<KVClientManager>
+                                    kvClientManager,
                                bool startBackGround));
 
     MOCK_METHOD4(Write, int(uint64_t inodeId, uint64_t offset, uint64_t length,

curvefs/test/client/mock_client_s3_cache_manager.h

@@ -24,6 +24,7 @@
 #define CURVEFS_TEST_CLIENT_MOCK_CLIENT_S3_CACHE_MANAGER_H_
 
 #include <gmock/gmock.h>
+#include <utility>
 #include <vector>
 #include "curvefs/src/client/s3/client_s3_cache_manager.h"
 
@@ -56,7 +57,7 @@ class MockFileCacheManager : public FileCacheManager {
 
 class MockChunkCacheManager : public ChunkCacheManager {
  public:
-    MockChunkCacheManager() : ChunkCacheManager(0, nullptr) {}
+    MockChunkCacheManager() : ChunkCacheManager(0, nullptr, nullptr) {}
     ~MockChunkCacheManager() = default;
 
     MOCK_METHOD1(ReleaseReadDataCache, void(uint64_t));
@@ -82,8 +83,10 @@ class MockDataCache : public DataCache {
  public:
     MockDataCache(S3ClientAdaptorImpl *s3ClientAdaptor,
                   ChunkCacheManagerPtr chunkCacheManager, uint64_t chunkPos,
-                  uint64_t len, const char *data)
-        : DataCache(s3ClientAdaptor, chunkCacheManager, chunkPos, len, data) {}
+                  uint64_t len, const char *data,
+                  std::shared_ptr<KVClientManager> kvClientManager)
+        : DataCache(s3ClientAdaptor, chunkCacheManager, chunkPos, len, data,
+                    std::move(kvClientManager)) {}
     ~MockDataCache() = default;
 
     MOCK_METHOD4(Write,

curvefs/test/client/mock_disk_cache_manager.h

@@ -26,6 +26,7 @@
 #include <gmock/gmock.h>
 
 #include <string>
+#include <utility>
 #include <vector>
 #include <set>
 #include <memory>
@@ -68,15 +69,21 @@ class MockDiskCacheManager2 : public DiskCacheManager {
     MOCK_METHOD0(IsDiskCacheFull, bool());
     MOCK_METHOD3(WriteReadDirect, int(const std::string fileName,
                                       const char *buf, uint64_t length));
+    MOCK_METHOD1(IsCached, bool(const std::string));
 };
 
 class MockDiskCacheManagerImpl : public DiskCacheManagerImpl {
  public:
     MockDiskCacheManagerImpl() : DiskCacheManagerImpl() {}
+    MockDiskCacheManagerImpl(std::shared_ptr<DiskCacheManager> diskCacheManager,
+                             std::shared_ptr<S3Client> client)
+        : DiskCacheManagerImpl(std::move(diskCacheManager), std::move(client)) {
+    }
     ~MockDiskCacheManagerImpl() {}
 
     MOCK_METHOD1(UploadWriteCacheByInode, int(const std::string &inode));
     MOCK_METHOD1(ClearReadCache, int(const std::list<std::string> &files));
+    MOCK_METHOD1(IsCached, bool(const std::string));
 };
 
 }  // namespace client

curvefs/test/client/rpcclient/base_client_test.cpp

@@ -419,6 +419,30 @@ TEST_F(BaseClientTest, test_RefreshSession) {
         << response.ShortDebugString();
 }
 
+TEST_F(BaseClientTest, test_AllocOrGetMemcacheCluster) {
+    AllocOrGetMemcacheClusterResponse response;
+    AllocOrGetMemcacheClusterResponse resp;
+    response.set_statuscode(mds::topology::TOPO_OK);
+    EXPECT_CALL(mockTopologyService_, AllocOrGetMemcacheCluster(_, _, _, _))
+        .WillOnce(DoAll(SetArgPointee<2>(response),
+                        Invoke(RpcService<AllocOrGetMemcacheClusterRequest,
+                                          AllocOrGetMemcacheClusterResponse>)));
+
+    brpc::Controller cntl;
+    cntl.set_timeout_ms(1000);
+    brpc::Channel ch;
+    ASSERT_EQ(0, ch.Init(addr_.c_str(), nullptr));
+
+    mdsbasecli_.AllocOrGetMemcacheCluster(1, &resp, &cntl, &ch);
+
+    ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText();
+    ASSERT_TRUE(
+        google::protobuf::util::MessageDifferencer::Equals(resp, response))
+        << "resp:\n"
+        << resp.ShortDebugString() << "response:\n"
+        << response.ShortDebugString();
+}
+
 }  // namespace rpcclient
 }  // namespace client
 }  // namespace curvefs

curvefs/test/client/rpcclient/mds_client_test.cpp

@@ -24,6 +24,7 @@
 #include <gmock/gmock-more-actions.h>
 #include <google/protobuf/util/message_differencer.h>
 #include <gtest/gtest.h>
+#include <cstdint>
 
 #include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/test/client/rpcclient/mock_mds_base_client.h"
@@ -57,6 +58,12 @@ void UmountFsRpcFailed(const std::string &fsName, const Mountpoint &mountPt,
     cntl->SetFailed(112, "Not connected to");
 }
 
+void AllocOrGetMemcacheClusterRpcFailed(
+    uint32_t fsId, AllocOrGetMemcacheClusterResponse* response,
+    brpc::Controller* cntl, brpc::Channel* channel) {
+    cntl->SetFailed(112, "Not connected to");
+}
+
 void GetFsInfoByFsnameRpcFailed(const std::string &fsName,
                                 GetFsInfoResponse *response,
                                 brpc::Controller *cntl,
@@ -998,6 +1005,41 @@ TEST_F(MdsClientImplTest, TestReleaseVolumeBlockGroup) {
     }
 }
 
+TEST_F(MdsClientImplTest, test_AllocOrGetMemcacheCluster) {
+    AllocOrGetMemcacheClusterResponse response;
+    MemcacheClusterInfo cluster1;
+    cluster1.set_clusterid(1);
+    mds::topology::MemcacheServerInfo server;
+    server.set_ip("127.0.0.1");
+    server.set_port(1);
+    *cluster1.add_servers() = server;
+    response.set_allocated_cluster(new MemcacheClusterInfo(cluster1));
+
+    // 1. ok
+    response.set_statuscode(curvefs::mds::topology::TOPO_OK);
+    EXPECT_CALL(mockmdsbasecli_, AllocOrGetMemcacheCluster(_, _, _, _))
+        .WillOnce(SetArgPointee<1>(response));
+    MemcacheClusterInfo cluster2;
+    ASSERT_EQ(true,
+              mdsclient_.AllocOrGetMemcacheCluster(1, &cluster2));
+
+    // 2. no memcached
+    response.set_statuscode(
+        curvefs::mds::topology::TOPO_MEMCACHECLUSTER_NOT_FOUND);
+    EXPECT_CALL(mockmdsbasecli_, AllocOrGetMemcacheCluster(_, _, _, _))
+        .WillOnce(SetArgPointee<1>(response));
+    ASSERT_EQ(false,
+              mdsclient_.AllocOrGetMemcacheCluster(1, &cluster2));
+
+    // 3. rpc error
+    brpc::Controller cntl;
+    cntl.SetFailed(ECONNRESET, "error connect reset");
+    EXPECT_CALL(mockmdsbasecli_, AllocOrGetMemcacheCluster(_, _, _, _))
+        .WillRepeatedly(Invoke(AllocOrGetMemcacheClusterRpcFailed));
+    ASSERT_EQ(false,
+              mdsclient_.AllocOrGetMemcacheCluster(1, &cluster2));
+}
+
 }  // namespace rpcclient
 }  // namespace client
 }  // namespace curvefs

curvefs/test/client/rpcclient/mock_mds_base_client.h

@@ -118,6 +118,11 @@ class MockMDSBaseClient : public MDSBaseClient {
              ReleaseBlockGroupResponse *response,
              brpc::Controller *cntl,
              brpc::Channel *channel));
+
+    MOCK_METHOD4(AllocOrGetMemcacheCluster,
+                 void(uint32_t fsId,
+                      AllocOrGetMemcacheClusterResponse* response,
+                      brpc::Controller* cntl, brpc::Channel* channel));
 };
 }  // namespace rpcclient
 }  // namespace client

curvefs/test/client/rpcclient/mock_topology_service.h

@@ -67,6 +67,14 @@ class MockTopologyService : public curvefs::mds::topology::TopologyService {
                  request,
              ::curvefs::mds::topology::GetCopysetOfPartitionResponse* response,
              ::google::protobuf::Closure* done));
+    MOCK_METHOD4(
+        AllocOrGetMemcacheCluster,
+        void(::google::protobuf::RpcController* controller,
+             const ::curvefs::mds::topology::AllocOrGetMemcacheClusterRequest*
+                 request,
+             ::curvefs::mds::topology::AllocOrGetMemcacheClusterResponse*
+                 response,
+             ::google::protobuf::Closure* done));
 };
 }  // namespace rpcclient
 }  // namespace client

curvefs/test/client/test_disk_cache_manager_impl.cpp

@@ -97,6 +97,7 @@ class TestDiskCacheManagerImpl : public ::testing::Test {
 
 TEST_F(TestDiskCacheManagerImpl, Init) {
     S3ClientAdaptorOption s3AdaptorOption;
+    s3AdaptorOption.diskCacheOpt.threads = 10;
     EXPECT_CALL(*diskCacheManager_, Init(_, _)).WillOnce(Return(-1));
     int ret = diskCacheManagerImpl_->Init(s3AdaptorOption);
     ASSERT_EQ(-1, ret);

curvefs/test/client/test_fuse_volume_client.cpp

@@ -25,7 +25,7 @@
 
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/fuse_volume_client.h"
 #include "curvefs/src/common/define.h"
 #include "curvefs/test/client/mock_dentry_cache_mamager.h"
@@ -97,6 +97,7 @@ class TestFuseVolumeClient : public ::testing::Test {
         fuseClientOption_.listDentryLimit = listDentryLimit_;
         fuseClientOption_.listDentryThreads = listDentryThreads_;
         fuseClientOption_.maxNameLength = 20u;
+        fuseClientOption_.warmupThreadsNum = 10;
 
         spaceManager_ = new MockSpaceManager();
         volumeStorage_ = new MockVolumeStorage();

curvefs/test/client/volume/default_volume_storage_test.cpp

@@ -25,7 +25,7 @@
 #include <string>
 #include <vector>
 
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/test/client/mock_inode_cache_manager.h"
 #include "curvefs/test/client/mock_metaserver_client.h"
 #include "curvefs/test/volume/mock/mock_block_device_client.h"

curvefs/test/common/BUILD

@@ -0,0 +1,32 @@
+#
+#  Copyright (c) 2023 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+load("//:copts.bzl", "CURVE_TEST_COPTS")
+
+cc_test(
+    name = "curvefs-common-test",
+    srcs = glob([
+        "*.cpp",
+    ]),
+    deps = [
+        "//external:bthread",
+        "//curvefs/src/common:curvefs_common",
+        "@com_google_googletest//:gtest_main",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/utility",
+    ],
+    copts = CURVE_TEST_COPTS,
+)

curvefs/test/common/task_thread_pool_test.cpp

@@ -0,0 +1,247 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+/*
+ * Project: curve
+ * Created Date: 2023-02-15
+ * Author: chengyi01
+ */
+
+#include "curvefs/src/common/task_thread_pool.h"
+
+#include <gtest/gtest.h>
+#include <bthread/condition_variable.h>
+#include <glog/logging.h>
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <utility>
+
+#include "src/common/concurrent/concurrent.h"
+#include "src/common/concurrent/rw_lock.h"
+#include "absl/memory/memory.h"
+
+namespace curvefs {
+namespace common {
+
+using curve::common::BthreadRWLock;
+using curve::common::ReadLockGuard;
+using curve::common::Thread;
+using curve::common::WriteLockGuard;
+using ThreadPool = TaskThreadPool2<bthread::Mutex, bthread::ConditionVariable>;
+
+const uint64_t TIME = 1000 * 1000;
+const uint64_t THREAD = 10;
+
+class SumTask {
+ public:
+    SumTask(uint64_t start, uint64_t end, uint64_t sum = 0)
+        : start_(start), end_(end), sum_(sum) {}
+    uint64_t start_ = 0;
+    uint64_t end_ = 0;
+    uint64_t sum_ = 0;
+};
+
+class SumManager {
+ public:
+    void AddSumTask(uint64_t key, std::function<void()> task) {
+        std::unique_ptr<ThreadPool> tp = absl::make_unique<ThreadPool>();
+        tp->Start(THREAD);
+        sumThreadPoolMutex_.WRLock();
+        auto iter = sumThreadPool_.emplace(key, std::move(tp));
+        sumThreadPoolMutex_.Unlock();
+        if (!iter.first->second->Enqueue(task)) {
+            LOG(ERROR) << "add task " << key << " fail!";
+        }
+    }
+
+    void AddSum(uint64_t key, uint64_t start, uint64_t end) {
+        WriteLockGuard lock(sumMutex_);
+        auto iter = sum_.emplace(key, SumTask(start, end));
+        iter.first->second.sum_ += start;
+
+        auto leftTask = [key, start, end, this]() {
+            uint64_t left = Left(start);
+            if (left <= end) {
+                WriteLockGuard lock(sumMutex_);
+                sum_.emplace(key, SumTask(start, end));
+                auto task = [this, key, left, end]() {
+                    AddSum(key, left, end);
+                };
+                AddSumTask(key, task);
+            }
+        };
+        AddSumTask(key, leftTask);
+
+        auto rightTask = [key, start, end, this]() {
+            uint64_t right = Right(start);
+            if (right <= end) {
+                WriteLockGuard lock(sumMutex_);
+                sum_.emplace(key, SumTask(start, end));
+                auto task = [this, key, right, end]() {
+                    AddSum(key, right, end);
+                };
+                AddSumTask(key, task);
+            }
+        };
+        AddSumTask(key, rightTask);
+
+        if (iter.second) {
+            LOG(INFO) << "add task:" << key << " start:" << start
+                      << " success!";
+        }
+    }
+
+    void Init() {
+        bgFetchStop_.store(false, std::memory_order_release);
+        bgFetchThread_ = Thread(&SumManager::backGroundFetch, this);
+        initbgFetchThread_ = true;
+    }
+    void Unit() {
+        bgFetchStop_.store(true, std::memory_order_release);
+        if (initbgFetchThread_) {
+            bgFetchThread_.join();
+        }
+
+        {
+            for (auto &iter : sumThreadPool_) {
+                LOG(INFO) << "Stop task:" << iter.first;
+                iter.second->Stop();
+            }
+            WriteLockGuard lock(sumThreadPoolMutex_);
+            sumThreadPool_.clear();
+        }
+
+        {
+            WriteLockGuard lock(sumMutex_);
+            for (auto iter : sum_) {
+                uint64_t result = Sum(iter.second.start_, iter.second.end_);
+                LOG(INFO) << "Sum Done:" << iter.first
+                          << " sum:" << iter.second.sum_
+                          << " result:" << result;
+                ASSERT_EQ(iter.second.sum_, result);
+            }
+        }
+    }
+
+    void WaitAllDone() {
+        bool working = true;
+        while (working) {
+            working = false;
+            usleep(TIME);
+            ReadLockGuard lock(sumMutex_);
+            for (auto const &iter : sum_) {
+                if (!SumDone(iter.first)) {
+                    working = true;
+                }
+            }
+        }
+        LOG(INFO) << "all done";
+    }
+
+ private:
+    bool SumDone(uint64_t key) {
+        ReadLockGuard lock(sumThreadPoolMutex_);
+        return sumThreadPool_.find(key) == sumThreadPool_.end();
+    }
+
+    uint64_t Sum(uint64_t start, uint64_t end) {
+        return (start + end) * (end - start + 1) / 2;
+    }
+    void backGroundFetch() {
+        while (!bgFetchStop_.load(std::memory_order_acquire)) {
+            usleep(TIME);
+            {
+                sumThreadPoolMutex_.RDLock();
+                for (auto iter = sumThreadPool_.begin();
+                     iter != sumThreadPool_.end();) {
+                    if (iter->second->QueueSize() == 0) {
+                        LOG(INFO) << "Stop task:" << iter->first;
+                        sumThreadPoolMutex_.Unlock();
+                        iter->second->Stop();
+                        sumThreadPoolMutex_.WRLock();
+                        iter = sumThreadPool_.erase(iter);
+                        sumThreadPoolMutex_.Unlock();
+                        sumThreadPoolMutex_.RDLock();
+                    } else {
+                        ++iter;
+                    }
+                }
+                sumThreadPoolMutex_.Unlock();
+            }
+            {
+                WriteLockGuard lock(sumMutex_);
+                for (auto iter = sum_.begin(); iter != sum_.end();) {
+                    if (SumDone(iter->first)) {
+                        uint64_t result =
+                            Sum(iter->second.start_, iter->second.end_);
+                        LOG(INFO) << "rm Sum Done:" << iter->first
+                                  << " sum:" << iter->second.sum_
+                                  << " result:" << result;
+                        ASSERT_EQ(iter->second.sum_, result);
+                        iter = sum_.erase(iter);
+                    } else {
+                        ++iter;
+                    }
+                }
+            }
+        }
+    }
+
+
+    static uint64_t Left(uint64_t n) { return 2 * n; }
+
+    static uint64_t Right(uint64_t n) { return 2 * n + 1; }
+
+ protected:
+    std::map<uint64_t, std::unique_ptr<ThreadPool>> sumThreadPool_;
+    BthreadRWLock sumThreadPoolMutex_;
+
+    std::map<uint64_t, SumTask> sum_;
+    BthreadRWLock sumMutex_;
+
+    Thread bgFetchThread_;
+    std::atomic<bool> bgFetchStop_;
+    bool initbgFetchThread_;
+};
+
+class TaskThreadPool2Test : public testing::Test {
+ protected:
+    void SetUp() override {
+        sumManager_ = std::make_shared<SumManager>();
+        sumManager_->Init();
+    }
+    void TearDown() override { sumManager_->Unit(); }
+
+    std::shared_ptr<SumManager> sumManager_;
+};
+
+TEST_F(TaskThreadPool2Test, test) {
+    sumManager_->AddSum(1, 1, 10);
+    sumManager_->AddSum(2, 1, 100);
+    sumManager_->AddSum(3, 1, 1000);
+    sumManager_->AddSum(4, 1, 10000);
+    sumManager_->AddSum(5, 1, 100000);
+    sumManager_->WaitAllDone();
+}
+
+TEST_F(TaskThreadPool2Test, test2) {
+    sumManager_->AddSum(4, 1, 10000);
+    sumManager_->WaitAllDone();
+}
+
+}  // namespace common
+}  // namespace curvefs

curvefs/test/mds/mock/mock_topology.h

@@ -446,6 +446,12 @@ class MockTopologyManager : public TopologyManager {
     MOCK_METHOD2(GetLatestPartitionsTxId,
                  void(const std::vector<PartitionTxId> &txIds,
                       std::vector<PartitionTxId> *needUpdate));
+
+    MOCK_METHOD2(RegistMemcacheCluster,
+                 void(const RegistMemcacheClusterRequest*,
+                      RegistMemcacheClusterResponse*));
+
+    MOCK_METHOD1(ListMemcacheCluster, void(ListMemcacheClusterResponse*));
 };
 
 }  // namespace topology

curvefs/test/mds/topology/test_topology.cpp

@@ -2343,6 +2343,35 @@ TEST_F(TestTopology, GenCopysetAddrBatch_2pool) {
     ASSERT_EQ(pool1Count, 3);
     ASSERT_EQ(pool2Count, 12);
 }
+
+TEST_F(TestTopology, test_AddMemcacheCluster_success) {
+    MemcacheCluster data(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)});
+
+    EXPECT_CALL(*storage_, StorageMemcacheCluster(_)).WillOnce(Return(true));
+
+    int ret = topology_->AddMemcacheCluster(data);
+
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, ret);
+
+    ASSERT_EQ(*topology_->ListMemcacheClusters().begin(), data);
+}
+
+TEST_F(TestTopology, test_AddMemcacheCluster_fail) {
+    MemcacheCluster data(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)});
+
+    EXPECT_CALL(*storage_, StorageMemcacheCluster(_)).WillOnce(Return(false));
+
+    int ret = topology_->AddMemcacheCluster(data);
+
+    ASSERT_EQ(TopoStatusCode::TOPO_STORGE_FAIL, ret);
+}
+
 }  // namespace topology
 }  // namespace mds
 }  // namespace curvefs

curvefs/test/mds/topology/test_topology_manager.cpp

@@ -23,12 +23,17 @@
 #include <brpc/channel.h>
 #include <brpc/controller.h>
 #include <brpc/server.h>
+#include <gmock/gmock-matchers.h>
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include <google/protobuf/util/message_differencer.h>
+#include <algorithm>
+#include <iostream>
+#include <vector>
 
 #include "curvefs/proto/topology.pb.h"
 #include "curvefs/src/mds/common/mds_define.h"
+#include "curvefs/src/mds/topology/topology_item.h"
 #include "curvefs/src/mds/topology/topology_manager.h"
 #include "curvefs/test/mds/mock/mock_metaserver.h"
 #include "curvefs/test/mds/mock/mock_metaserver_client.h"
@@ -218,6 +223,14 @@ class TestTopologyManager : public ::testing::Test {
         PrepareAddPartition(fsId + 1, poolId, copysetId, pId3, 1, 100, 2);
     }
 
+    void PrepareAddMemcacheCluster(MemcacheClusterIdType id,
+                                   const std::list<MemcacheServer>& servers) {
+        MemcacheCluster cluster(id, servers);
+        EXPECT_CALL(*storage_, StorageMemcacheCluster(_))
+            .WillOnce(Return(true));
+        topology_->AddMemcacheCluster(cluster);
+    }
+
  protected:
     std::shared_ptr<MockIdGenerator> idGenerator_;
     std::shared_ptr<MockTokenGenerator> tokenGenerator_;
@@ -2842,6 +2855,175 @@ TEST_F(TestTopologyManager, test_GetCopysetMembers_Success) {
     ASSERT_THAT(*addrs.begin(), AnyOf("ip2:8887", "ip2:8888", "ip2:8889"));
 }
 
+TEST_F(TestTopologyManager, test_RegistMemcacheCluster_Success) {
+    RegistMemcacheClusterRequest request;
+    RegistMemcacheClusterResponse response;
+    MemcacheServerInfo server;
+    server.set_ip("127.0.0.1");
+    server.set_port(1);
+    *request.add_servers() = server;
+
+    MemcacheClusterIdType mcCId(1);
+
+    EXPECT_CALL(*idGenerator_, GenMemCacheClusterId()).WillOnce(Return(mcCId));
+    EXPECT_CALL(*storage_, StorageMemcacheCluster(_)).WillOnce(Return(true));
+
+    serviceManager_->RegistMemcacheCluster(&request, &response);
+
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response.statuscode());
+    ASSERT_TRUE(response.has_clusterid());
+    ASSERT_EQ(mcCId, response.clusterid());
+
+    // test register same cluster
+    serviceManager_->RegistMemcacheCluster(&request, &response);
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response.statuscode());
+
+    // test register register server
+    server.set_port(2);
+    *request.add_servers() = server;
+    serviceManager_->RegistMemcacheCluster(&request, &response);
+    ASSERT_EQ(TopoStatusCode::TOPO_IP_PORT_DUPLICATED, response.statuscode());
+}
+
+TEST_F(TestTopologyManager, test_RegistMemcacheCluster_AllocateIdFail) {
+    RegistMemcacheClusterRequest request;
+    RegistMemcacheClusterResponse response;
+    MemcacheServerInfo server;
+    server.set_ip("127.0.0.1");
+    server.set_port(1);
+    *request.add_servers() = server;
+
+    MemcacheClusterIdType mcCId(1);
+
+    EXPECT_CALL(*idGenerator_, GenMemCacheClusterId())
+        .WillOnce(Return(UNINITIALIZE_ID));
+
+    serviceManager_->RegistMemcacheCluster(&request, &response);
+
+    ASSERT_EQ(TopoStatusCode::TOPO_ALLOCATE_ID_FAIL, response.statuscode());
+}
+
+TEST_F(TestTopologyManager, test_ListMemcacheCluster_Success) {
+    std::vector<MemcacheCluster> clusters;
+    clusters.emplace_back(MemcacheCluster(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)}));
+    clusters.emplace_back(MemcacheCluster(
+        2, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 4),
+                                     MemcacheServer("127.0.0.1", 5),
+                                     MemcacheServer("127.0.0.1", 6)}));
+    for (auto const & cluster : clusters) {
+        PrepareAddMemcacheCluster(cluster.GetId(), cluster.GetServers());
+    }
+
+    ListMemcacheClusterResponse response;
+    serviceManager_->ListMemcacheCluster(&response);
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response.statuscode());
+    auto clusterList = response.memcacheclusters();
+    ASSERT_EQ(clusterList.size(), clusters.size());
+    for (auto const& cluster : clusterList) {
+        ASSERT_TRUE(std::find(clusters.cbegin(), clusters.cend(),
+                              static_cast<MemcacheCluster>(cluster)) !=
+                    clusters.cend());
+    }
+}
+
+TEST_F(TestTopologyManager, test_ListMemcacheCluster_NotFound) {
+    ListMemcacheClusterResponse response;
+    serviceManager_->ListMemcacheCluster(&response);
+    ASSERT_EQ(TopoStatusCode::TOPO_MEMCACHECLUSTER_NOT_FOUND,
+              response.statuscode());
+}
+
+TEST_F(TestTopologyManager, test_AllocOrGetMemcacheCluster_1_Success) {
+    std::vector<MemcacheCluster> clusters;
+    clusters.emplace_back(MemcacheCluster(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)}));
+    for (auto const& cluster : clusters) {
+        PrepareAddMemcacheCluster(cluster.GetId(), cluster.GetServers());
+    }
+
+    EXPECT_CALL(*storage_, StorageFs2MemcacheCluster(_, _))
+        .WillOnce(Return(true));
+
+    AllocOrGetMemcacheClusterRequest request;
+    AllocOrGetMemcacheClusterResponse response;
+    serviceManager_->AllocOrGetMemcacheCluster(&request, &response);
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response.statuscode());
+    ASSERT_TRUE(response.has_cluster());
+    ASSERT_EQ(clusters[0],
+              static_cast<MemcacheCluster>(response.cluster()));
+}
+
+TEST_F(TestTopologyManager, test_AllocOrGetMemcacheCluster_Success) {
+    std::vector<MemcacheCluster> clusters;
+    clusters.emplace_back(MemcacheCluster(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)}));
+    clusters.emplace_back(MemcacheCluster(
+        2, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 4),
+                                     MemcacheServer("127.0.0.1", 5),
+                                     MemcacheServer("127.0.0.1", 6)}));
+    for (auto const& cluster : clusters) {
+        PrepareAddMemcacheCluster(cluster.GetId(), cluster.GetServers());
+    }
+
+    EXPECT_CALL(*storage_, StorageFs2MemcacheCluster(_, _))
+        .WillOnce(Return(true));
+
+    AllocOrGetMemcacheClusterRequest request1;
+    request1.set_fsid(1);
+    AllocOrGetMemcacheClusterResponse response1;
+    serviceManager_->AllocOrGetMemcacheCluster(&request1, &response1);
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response1.statuscode());
+    ASSERT_TRUE(response1.has_cluster());
+
+
+    // same fsid
+    AllocOrGetMemcacheClusterResponse response2;
+    serviceManager_->AllocOrGetMemcacheCluster(&request1, &response2);
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response2.statuscode());
+    ASSERT_TRUE(response2.has_cluster());
+    ASSERT_EQ(static_cast<MemcacheCluster>(response1.cluster()),
+              static_cast<MemcacheCluster>(response2.cluster()));
+}
+
+TEST_F(TestTopologyManager, test_AllocOrGetMemcacheCluster_NotFound) {
+    AllocOrGetMemcacheClusterRequest request;
+    AllocOrGetMemcacheClusterResponse response;
+    serviceManager_->AllocOrGetMemcacheCluster(&request, &response);
+    ASSERT_EQ(TopoStatusCode::TOPO_MEMCACHECLUSTER_NOT_FOUND,
+              response.statuscode());
+}
+
+TEST_F(TestTopologyManager, test_AllocOrGetMemcacheCluster_StorageFail) {
+    std::vector<MemcacheCluster> clusters;
+    clusters.emplace_back(MemcacheCluster(
+        1, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 1),
+                                     MemcacheServer("127.0.0.1", 2),
+                                     MemcacheServer("127.0.0.1", 3)}));
+    clusters.emplace_back(MemcacheCluster(
+        2, std::list<MemcacheServer>{MemcacheServer("127.0.0.1", 4),
+                                     MemcacheServer("127.0.0.1", 5),
+                                     MemcacheServer("127.0.0.1", 6)}));
+    for (auto const& cluster : clusters) {
+        PrepareAddMemcacheCluster(cluster.GetId(), cluster.GetServers());
+    }
+
+    AllocOrGetMemcacheClusterRequest request;
+    request.set_fsid(1);
+    AllocOrGetMemcacheClusterResponse response;
+
+    EXPECT_CALL(*storage_, StorageFs2MemcacheCluster(_, _))
+        .WillOnce(Return(false));
+    serviceManager_->AllocOrGetMemcacheCluster(&request, &response);
+    ASSERT_EQ(TopoStatusCode::TOPO_STORGE_FAIL, response.statuscode());
+}
+
 }  // namespace topology
 }  // namespace mds
 }  // namespace curvefs